boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git@gitorious.org:yacy/rc1.git

Browse Source
pull/1/head
orbiter 11 years ago
parent d93325a578 b31db00010
commit f3a12801f0
49 changed files with 353 additions and 363 deletions
Show Stats Download Patch File Download Diff File

2
.classpath
Unescape View File

@ -71,7 +71,7 @@
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="lib" path="lib/icu4j-core.jar"/>
<classpathentry kind="lib" path="lib/htmllexer.jar"/>
<classpathentry kind="lib" path="lib/jsoup-1.6.3.jar"/>
<classpathentry kind="lib" path="lib/jsoup-1.8.1.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.3.v20140905.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.3.v20140905.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.3.v20140905.jar"/>

2
addon/YaCy.app/Contents/Info.plist
Unescape View File

@ -78,7 +78,7 @@
<string>$JAVAROOT/lib/jetty-xml-9.2.3.v20140905.jar</string>
<string>$JAVAROOT/lib/jsch-0.1.50.jar</string>
<string>$JAVAROOT/lib/json-simple-1.1.1.jar</string>
<string>$JAVAROOT/lib/jsoup-1.6.3.jar</string>
<string>$JAVAROOT/lib/jsoup-1.8.1.jar</string>
<string>$JAVAROOT/lib/log4j-over-slf4j-1.7.2.jar</string>
<string>$JAVAROOT/lib/lucene-analyzers-common-4.9.0.jar</string>
<string>$JAVAROOT/lib/lucene-analyzers-phonetic-4.9.0.jar</string>

2
build.xml
Unescape View File

@ -202,7 +202,7 @@
<pathelement location="${lib}/jetty-xml-9.2.3.v20140905.jar" />
<pathelement location="${lib}/jsch-0.1.50.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.6.3.jar" />
<pathelement location="${lib}/jsoup-1.8.1.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.7.2.jar" />
<pathelement location="${lib}/lucene-analyzers-common-4.9.0.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-4.9.0.jar" />

1
htroot/CrawlProfileEditor_p.xml
Unescape View File

@ -4,7 +4,6 @@
<crawlProfile>
<name>#[name]#</name>
<status>#(status)#terminated::active#(/status)#</status>
<starturl>#[startURL]#</starturl>
<depth>#[depth]#</depth>
<mustmatch>#[mustmatch]#</mustmatch>
<mustnotmatch>#[mustnotmatch]#</mustnotmatch>

2
htroot/IndexCreateParserErrors_p.html
Unescape View File

@ -38,7 +38,7 @@
#{list}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[time]#</td>
<td><a href="#[url]#">#[url]#</a></td>
<td><a href="#[url]#" target="_blank">#[url]#</a></td>
<td>#[failreason]#</td>
</tr>
#{/list}#

4
htroot/IndexCreateQueues_p.java
Unescape View File

@ -14,6 +14,7 @@ import java.util.regex.PatternSyntaxException;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.crawler.CrawlSwitchboard;
@ -141,8 +142,7 @@ public class IndexCreateQueues_p {
int hc = 0;
for (Map.Entry<String, Integer[]> host: hosts.entrySet()) {
String hostnameport = host.getKey();
int p = hostnameport.lastIndexOf(':');
String hostname = p < 0 ? hostnameport : hostnameport.substring(0, p);
String hostname = Domains.stripToHostName(hostnameport);
prop.putHTML("crawler_host_" + hc + "_hostnameport", hostnameport);
prop.putHTML("crawler_host_" + hc + "_hostname", hostname);
prop.put("crawler_host_" + hc + "_embed", embed ? 1 : 0);

3
htroot/Load_RSS_p.java
Unescape View File

@ -36,6 +36,7 @@ import net.yacy.cora.document.feed.RSSFeed;
import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.document.feed.RSSReader;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.RequestHeader;
@ -212,7 +213,7 @@ public class Load_RSS_p {
final Date date_next_exec = r.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, (Date) null);
prop.put("showscheduledfeeds_list_" + apic + "_pk", UTF8.String(row.getPK()));
prop.put("showscheduledfeeds_list_" + apic + "_count", apic);
prop.putXML("showscheduledfeeds_list_" + apic + "_rss", messageurl);
prop.put("showscheduledfeeds_list_" + apic + "_rss", MultiProtocolURL.escape(messageurl).toString());
prop.putXML("showscheduledfeeds_list_" + apic + "_title", row.get("title", ""));
prop.putXML("showscheduledfeeds_list_" + apic + "_referrer", referrer == null ? "#" : referrer.toNormalform(true));
prop.put("showscheduledfeeds_list_" + apic + "_recording", DateFormat.getDateTimeInstance().format(row.get("recording_date", new Date())));

1
htroot/Network.csv
Unescape View File

@ -1 +0,0 @@
#(table)#:: ::#[active-count]#;#[active-links]#;#[active-words]#;#[passive-count]#;#[passive-links]#;#[passive-words]#;#[potential-count]#;#[potential-links]#;#[potential-words]#;#[all-count]#;#[all-links]#;#[all-words]#;#[my-links]#;#[my-words]##(/table)#
1 #(table)#:: ::#[active-count]# #[active-links]# #[active-words]# #[passive-count]# #[passive-links]# #[passive-words]# #[potential-count]# #[potential-links]# #[potential-words]# #[all-count]# #[all-links]# #[all-words]# #[my-links]# #[my-words]##(/table)#

65
htroot/Network.html
Unescape View File

@ -104,54 +104,43 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window.
</form>
<table class="sortable" border="0">
<tr class="TableHeader" valign="bottom">
<td>send&nbsp;<strong>M</strong>essage/<br/>show&nbsp;<strong>P</strong>rofile/<br/>edit&nbsp;<strong>W</strong>iki/<br/>browse&nbsp;<strong>B</strong>log</td>
<td><strong>Hash</strong></td>
<td><strong>Name</strong><br/></td>
<td width="34"><strong>Info</strong></td>
<td><strong>Release</strong></td>
<td><strong>Age</strong></td>
<td><strong>#Seeds</strong></td>
<td><strong>con/h<br/></strong></td>
<td><strong>PPM</strong></td>
<td><strong>QPH</strong></td>
<td><strong>Last<br/>Seen</strong></td>
<td><strong>UTC</strong><br/>Offset</td>
<td><strong>Location</strong><br/></td>
<td style="width:70px;"><strong>Uptime</strong></td>
<td><strong>Links</strong></td>
<td><strong>RWIs</strong></td>
<td><strong>URLs for<br/>Remote<br/>Crawl</strong></td>
<td><strong>Sent<br/>DHT Word Chunks</strong></td>
<td><strong>URLs<br/>for<br/>Remote<br/>Crawl</strong></td>
<td><strong>Sent DHT<br/>Word Chunks</strong></td>
<td><strong>Sent<br/>URLs</strong></td>
<td><strong>Received<br/>DHT Word Chunks</strong></td>
<td><strong>Received DHT<br/>Word Chunks</strong></td>
<td><strong>Received<br/>URLs</strong></td>
#(complete)#::
<td><strong>Address</strong></td>
<td><strong>Hash</strong></td>
<td><strong>Age</strong></td>
<td><strong>#Seeds</strong></td>
<td><strong>con/h<br/></strong></td>
<td><strong>user agent<br/></strong></td>
<td><strong>Peer<br/>Ping</strong></td>
#(/complete)#
<td><strong>Location</strong><br/></td>
#(c)#::<td><strong>user agent<br/></strong></td>#(/c)#
<td>send&nbsp;<strong>M</strong>essage/<br/>show&nbsp;<strong>P</strong>rofile/<br/>edit&nbsp;<strong>W</strong>iki/<br/>browse&nbsp;<strong>B</strong>log</td>
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td><nobr>
<a href="MessageSend_p.html?hash=#[hash]#" title="Send message to peer #[fullname]#">m</a>&nbsp;
<a href="ViewProfile.html?hash=#[hash]#&amp;display=1" title="View profile of peer #[fullname]#">p</a>&nbsp;
<a href="goto_p.html?hash=#[hash]#&path=/Wiki.html" title="Read and edit wiki on peer #[fullname]#">w</a>&nbsp;
<a href="goto_p.html?hash=#[hash]#&path=/Blog.html" title="Browse blog of peer #[fullname]#">b</a>&nbsp;
#(updatedProfile)#::<a href="ViewProfile.html?hash=#[hash]#&amp;display=1"><img src="env/grafics/profile.gif" width="54" height="11" alt="Profile updated" /></a>#(/updatedProfile)#
#(updatedWiki)#::<a href="http://#[address]#/Wiki.html?page=#[page]#" title="#[page]#"><img src="env/grafics/wiki.gif" width="54" height="11" alt="Wiki updated" /></a>#(/updatedWiki)#
#(updatedBlog)#::<a href="http://#[address]#/Blog.html?page=#[page]#" title="#[subject]#"><img src="env/grafics/blog.gif" width="54" height="11" alt="Blog updated" /></a>#(/updatedBlog)#
#(isCrawling)#::<a href="#[page]#"><img src="env/grafics/crawl.gif" width="54" height="11" alt="Crawl" /></a>#(/isCrawling)#
</nobr></td>
<td #(special)#::class="TableCellActive"#(/special)#><a href="goto_p.html?hash=#[hash]#">#[shortname]#</a>#(ssl)#::<img src="env/grafics/lock.gif" width="11" height="15" title="https supported" alt="https supported" />#(/ssl)#</td>
<td >#[hash]#</td>
<td #(special)#::class="TableCellActive"#(/special)#>#[shortname]##(ssl)#::<img src="env/grafics/lock.gif" width="11" height="15" title="https supported" alt="https supported" />#(/ssl)#</td>
<td nowrap>
#(type)##(direct)#<img src="env/grafics/JuniorPassive.gif" width="11" height="11" title="Type: Junior | Contact: passive" alt="Junior passive" />::<img src="env/grafics/JuniorDirect.gif" width="11" height="11" title="Type: Junior | Contact: direct" alt="Junior direct" />::<img src="env/grafics/JuniorOffline.gif" width="11" height="11" title="Type: Junior | Contact: offline" alt="Junior offline" />#(/direct)#::#(direct)#<img src="env/grafics/SeniorPassive.gif" width="11" height="11" title="Type: Senior | Contact: passive" alt="senior passive" />::<img src="env/grafics/SeniorDirect.gif" width="11" height="11" title="Type: Senior | Contact: direct" alt="Senior direct" />::<img src="env/grafics/SeniorOffline.gif" width="11" height="11" title="Type: Senior | Contact: offline" alt="Senior offline" />#(/direct)#::<a href="#[url]#">#(direct)#<img src="env/grafics/PrincipalPassive.gif" width="11" height="11" title="Type: Principal | Contact: passive | Seed download: possible" alt="Principal passive" />::<img src="env/grafics/PrincipalDirect.gif" width="11" height="11" title="Type: Principal | Contact: direct | Seed download: possible" alt="Principal active" />::<img src="env/grafics/PrincipalOffline.gif" width="11" height="11" title="Type: Principal | Contact: offline | Seed download: ?" alt="Principal offline" />#(/direct)#</a>#(/type)##(acceptcrawl)#<img src="env/grafics/CrawlNo.gif" width="11" height="11" title="Accept Crawl: no" alt="no crawl" />::<img src="env/grafics/CrawlYes.gif" width="11" height="11" title="Accept Crawl: yes" alt="crawl possible" />::<img src="env/grafics/CrawlYesOffline.gif" width="11" height="11" title="Accept Crawl: yes" alt="crawl possible" />#(/acceptcrawl)##(dhtreceive)#<img src="env/grafics/DHTReceiveNo.gif" width="11" height="11" title="DHT Receive: no; #[peertags]#" alt="no DHT receive" />::<img src="env/grafics/DHTReceiveYes.gif" width="11" height="11" title="DHT Receive: yes" alt="DHT receive enabled" />::<img src="env/grafics/DHTReceiveYesOffline.gif" width="11" height="11" title="DHT Receive: yes" alt="DHT receive enabled" />#(/dhtreceive)##(nodestate)#<img src="env/grafics/RootStateNo.gif" width="11" height="11" title="Not a Node Candidate" alt="no node candidate" />::<a href="http://#[ip]#:#[port]#/Network.html?page=1"><img src="env/grafics/RootStateYes.gif" width="11" height="11" title="Node Candidate" alt="node candidate" /></a>#(/nodestate)#</td>
#(type)##(direct)#<img src="env/grafics/JuniorPassive.gif" width="11" height="11" title="Type: Junior | Contact: passive" alt="Junior passive" />::<img src="env/grafics/JuniorDirect.gif" width="11" height="11" title="Type: Junior | Contact: direct" alt="Junior direct" />::<img src="env/grafics/JuniorOffline.gif" width="11" height="11" title="Type: Junior | Contact: offline" alt="Junior offline" />#(/direct)#::#(direct)#<img src="env/grafics/SeniorPassive.gif" width="11" height="11" title="Type: Senior | Contact: passive" alt="senior passive" />::<img src="env/grafics/SeniorDirect.gif" width="11" height="11" title="Type: Senior | Contact: direct" alt="Senior direct" />::<img src="env/grafics/SeniorOffline.gif" width="11" height="11" title="Type: Senior | Contact: offline" alt="Senior offline" />#(/direct)#::<a href="#[url]#" class="forceNoExternalIcon">#(direct)#<img src="env/grafics/PrincipalPassive.gif" width="11" height="11" title="Type: Principal | Contact: passive | Seed download: possible" alt="Principal passive" />::<img src="env/grafics/PrincipalDirect.gif" width="11" height="11" title="Type: Principal | Contact: direct | Seed download: possible" alt="Principal active" />::<img src="env/grafics/PrincipalOffline.gif" width="11" height="11" title="Type: Principal | Contact: offline | Seed download: ?" alt="Principal offline" />#(/direct)#</a>#(/type)##(acceptcrawl)#<img src="env/grafics/CrawlNo.gif" width="11" height="11" title="Accept Crawl: no" alt="no crawl" />::<img src="env/grafics/CrawlYes.gif" width="11" height="11" title="Accept Crawl: yes" alt="crawl possible" />::<img src="env/grafics/CrawlYesOffline.gif" width="11" height="11" title="Accept Crawl: yes" alt="crawl possible" />#(/acceptcrawl)##(dhtreceive)#<img src="env/grafics/DHTReceiveNo.gif" width="11" height="11" title="DHT Receive: no; #[peertags]#" alt="no DHT receive" />::<img src="env/grafics/DHTReceiveYes.gif" width="11" height="11" title="DHT Receive: yes" alt="DHT receive enabled" />::<img src="env/grafics/DHTReceiveYesOffline.gif" width="11" height="11" title="DHT Receive: yes" alt="DHT receive enabled" />#(/dhtreceive)##{ips}#<a href="#(c)#http://#(ipv6)#::[#(/ipv6)##[ip]##(ipv6)#::]#(/ipv6)#:#[port]#/Network.html?page=1::Network.html?page=4&amp;peerHash=#[hash]#&amp;peerIP=#[ip]#&amp;peerPort=#[port]#&amp;addPeer=add+Peer#(/c)#"#(c)#:: target="_blank"#(/c)#class="forceNoExternalIcon"><img src="env/grafics/#(nodestate)#NodeDisqualified::NodeQualified#(/nodestate)##(c)##(ipv6)#IPv4::IPv6#(/ipv6)#::#(ipv6)#IPv4::IPv6#(/ipv6)##(/c)#.gif" width="11" height="11" title="#(c)##(ipv6)#IPv4::IPv6#(/ipv6)#::#(ipv6)#IPv4::IPv6#(/ipv6)##(/c)# #(nodestate)#Peer::Node Peer#(/nodestate)#" /></a>#{/ips}#</td>
<td align="right">#[version]#</td>
<td align="right">#[age]#</td>
<td align="right">#[seeds]#</td>
<td align="right">#[connects]#</td>
<td align="right">#[ppm]#</td>
<td align="right">#[qph]#</td>
<td align="right">#[lastSeen]#</td>
<td align="right">#[utc]#</td>
<td align="right">#[location]#</td>
<td align="right" nowrap>#[uptime]#</td>
<td align="right">#[LCount]#</td>
<td align="right">#[ICount]#</td>
@ -160,16 +149,18 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window.
<td align="right">#[sU]#</td>
<td align="right">#[rI]#</td>
<td align="right">#[rU]#</td>
#(complete)#
::
<td><a href="http://#[ip]#:#[port]#/Network.html?page=1&amp;ip=">http://#[ip]#:#[port]#</a> #[ips]#</td>
<td >#[hash]#</td>
<td >#[age]#</td>
<td align="right">#[seeds]#</td>
<td align="right">#[connects]#</td>
<td align="right">#[userAgent]#</td>
<td align="right">[<a href="Network.html?page=4&amp;peerHash=#[hash]#&amp;peerIP=#[ip]#&amp;peerPort=#[port]#&amp;addPeer=add+Peer">Ping</a>]</td>
#(/complete)#
<td align="right">#[location]#</td>
#(c)#::<td align="right">#[userAgent]#</td>#(/c)#
<td><nobr>
<a href="MessageSend_p.html?hash=#[hash]#" title="Send message to peer #[fullname]#">m</a>&nbsp;
<a href="ViewProfile.html?hash=#[hash]#&amp;display=1" title="View profile of peer #[fullname]#">p</a>&nbsp;
<a href="goto_p.html?hash=#[hash]#&path=/Wiki.html" title="Read and edit wiki on peer #[fullname]#">w</a>&nbsp;
<a href="goto_p.html?hash=#[hash]#&path=/Blog.html" title="Browse blog of peer #[fullname]#">b</a>&nbsp;
#(updatedProfile)#::<a href="ViewProfile.html?hash=#[hash]#&amp;display=1"><img src="env/grafics/profile.gif" width="54" height="11" alt="Profile updated" /></a>#(/updatedProfile)#
#(updatedWiki)#::<a href="http://#[address]#/Wiki.html?page=#[page]#" title="#[page]#"><img src="env/grafics/wiki.gif" width="54" height="11" alt="Wiki updated" /></a>#(/updatedWiki)#
#(updatedBlog)#::<a href="http://#[address]#/Blog.html?page=#[page]#" title="#[subject]#"><img src="env/grafics/blog.gif" width="54" height="11" alt="Blog updated" /></a>#(/updatedBlog)#
#(isCrawling)#::<a href="#[page]#"><img src="env/grafics/crawl.gif" width="54" height="11" alt="Crawl" /></a>#(/isCrawling)#
</nobr></td>
</tr>
#{/list}#
</table>

74
htroot/Network.java
Unescape View File

@ -32,6 +32,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
@ -205,24 +206,30 @@ public class Network {
}
final ConcurrentMap<String, String> map = new ConcurrentHashMap<String, String>();
map.put(Seed.IP, post.get("peerIP"));
map.put(Seed.PORT, post.get("peerPort"));
Seed peer = new Seed(post.get("peerHash"), map);
String challengeIP = post.get("peerIP");
String challengePort = post.get("peerPort");
map.put(Seed.IP, challengeIP);
map.put(Seed.PORT, challengePort);
Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map);
String challengeAddress = peer.getPublicAddress(challengeIP);
sb.updateMySeed();
final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer);
Seed mySeed = sb.peers.mySeed();
final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeAddress, peer.hash);
if (response == null) {
if (added <= 0) {
prop.put("table_comment",1);
prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getIPs());
prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>");
} else {
String yourtype = response.get("yourtype");
String yourip = response.get("yourip");
peer = sb.peers.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
prop.putHTML("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
} else {
prop.put("table_comment",2);
prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getIPs());
prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
prop.putHTML("table_comment_details",peer.toString());
}
}
@ -285,7 +292,7 @@ public class Network {
boolean dark = true;
Seed seed;
final boolean complete = (post != null && post.containsKey("ip"));
final boolean c = (post != null && post.containsKey("ip"));
final boolean onlyIncomingDHT = (post != null && post.containsKey("onlydhtin"));
final boolean onlyNode = (post != null && post.containsKey("onlynode"));
final long onlyAgeOverDays = post == null ? 0 : post.getLong("onlyageoverdays", 0);
@ -341,6 +348,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_updatedWikiPage", 0);
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0);
prop.put(STR_TABLE_LIST + conCount + "_isCrawling", 0);
String ip = seed.getIP();
if (conCount >= maxCount) { break; }
if (sb.peers != null && sb.peers.mySeed() != null && seed.hash != null && seed.hash.equals(sb.peers.mySeed().hash)) {
prop.put(STR_TABLE_LIST + conCount + "_dark", 2);
@ -356,7 +364,7 @@ public class Network {
} else {
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki", 1);
prop.putHTML(STR_TABLE_LIST + conCount + "_updatedWiki_page", wikiMap.get("page"));
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_address", seed.getPublicAddress());
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_address", seed.getPublicAddress(ip));
}
if ((blogMap = updatedBlog.get(seed.hash)) == null) {
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0);
@ -364,7 +372,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 1);
prop.putHTML(STR_TABLE_LIST + conCount + "_updatedBlog_page", blogMap.get("page"));
prop.putHTML(STR_TABLE_LIST + conCount + "_updatedBlog_subject", blogMap.get("subject"));
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_address", seed.getPublicAddress());
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_address", seed.getPublicAddress(ip));
}
PPM = seed.getPPM();
QPM = seed.getQPM();
@ -386,30 +394,39 @@ public class Network {
userAgent = ClientIdentification.yacyInternetCrawlerAgent.userAgent;
location = ClientIdentification.generateLocation();
} else {
userAgent = sb.peers.peerActions.getUserAgent(seed.getIP());
userAgent = sb.peers.peerActions.getUserAgent(ip);
location = ClientIdentification.parseLocationInUserAgent(userAgent);
}
if (location.length() > 10) location = location.substring(0, 10);
if (location.length() == 0) {
Locale l = Domains.getLocale(seed.getIP());
Locale l = Domains.getLocale(ip);
if (l != null) location = l.toString();
}
prop.putHTML(STR_TABLE_LIST + conCount + "_location", location);
if (complete) {
prop.put(STR_TABLE_LIST + conCount + "_complete", 1);
prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() );
prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ips", seed.getIPs().toString() );
prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get(Seed.PORT, "-") );
prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash);
prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge());
prop.putNum(STR_TABLE_LIST + conCount + "_complete_seeds", seed.getLong(Seed.SCOUNT, 0L));
prop.putNum(STR_TABLE_LIST + conCount + "_complete_connects", seed.getFloat(Seed.CCOUNT, 0F));
prop.putHTML(STR_TABLE_LIST + conCount + "_complete_userAgent", userAgent);
String port = seed.get(Seed.PORT, "-");
Set<String> ips = seed.getIPs();
int ipsc = 0;
for (String s: ips) {
prop.put(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_nodestate", seed.getFlagRootNode() ? 1 : 0);
prop.put(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_c", c ? 1 : 0);
prop.putHTML(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_c_hash", seed.hash);
prop.putHTML(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_c_ip", s);
prop.putHTML(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_c_port", port);
prop.put(STR_TABLE_LIST + conCount + "_ips_" + ipsc++ + "_c_ipv6", s.indexOf(':') >= 0 ? 1 : 0);
}
prop.put(STR_TABLE_LIST + conCount + "_ips", ipsc);
prop.put(STR_TABLE_LIST + conCount + "_port", port);
prop.put(STR_TABLE_LIST + conCount + "_hash", seed.hash);
prop.put(STR_TABLE_LIST + conCount + "_age", seed.getAge());
prop.putNum(STR_TABLE_LIST + conCount + "_seeds", seed.getLong(Seed.SCOUNT, 0L));
prop.putNum(STR_TABLE_LIST + conCount + "_connects", seed.getFloat(Seed.CCOUNT, 0F));
if (c) {
prop.put(STR_TABLE_LIST + conCount + "_c", 1);
prop.putHTML(STR_TABLE_LIST + conCount + "_c_userAgent", userAgent);
} else {
prop.put(STR_TABLE_LIST + conCount + "_complete", 0);
prop.put(STR_TABLE_LIST + conCount + "_c", 0);
}
if (seed.isJunior()) {
prop.put(STR_TABLE_LIST + conCount + "_type", 0);
} else if(seed.isSenior()){
@ -447,9 +464,6 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive", 0); // red/red; offline was off
}
}
prop.put(STR_TABLE_LIST + conCount + "_nodestate", seed.getFlagRootNode() ? 1 : 0);
prop.put(STR_TABLE_LIST + conCount + "_nodestate_ip", seed.getIP());
prop.put(STR_TABLE_LIST + conCount + "_nodestate_port", seed.get(Seed.PORT, "-") );
if (seed.getFlagAcceptRemoteIndex()) {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", "");
} else {
@ -479,7 +493,7 @@ public class Network {
prop.put("table", 1);
prop.putNum("table_num", conCount);
prop.putNum("table_total", ((page == 1) && (iAmActive)) ? (size + 1) : size );
prop.put("table_complete", ((complete)? 1 : 0) );
prop.put("table_c", ((c)? 1 : 0) );
}
}
prop.put("page", page);

11
htroot/Network.xml
Unescape View File

@ -21,17 +21,16 @@
<direct>#(type_direct)#direct::passive#(/type_direct)#</direct>
<acceptcrawl>#[acceptcrawl]#</acceptcrawl>
<dhtreceive>#[dhtreceive]#</dhtreceive>
<nodestate>#[nodestate]#</nodestate>
<nodestate>#(nodestate)#false::true#(/nodestate)#</nodestate>
<location>#[location]#</location>
<seedurl>#[type_url]#</seedurl>
#(complete)#
::
<age>#[age]#</age>
<seeds>#[seeds]#</seeds>
<connects>#[connects]#</connects>
<address>#[ip]#:#[port]#</address>
<useragent>#[userAgent]#</useragent>
#(/complete)#
#{ips}#<ip>#(c)##[ip]#::#[ip]##(/c)#</ip>#{/ips}#
#{ips}#<address>#(c)##(ipv6)#::[#(/ipv6)##[ip]##(ipv6)#::]#(/ipv6)#:#[port]#::#(/c)#</address>#{/ips}#
<port>#[port]#</port>
#(c)#::<useragent>#[userAgent]#</useragent>#(/c)#
</peer>
#{/list}#
::

2
htroot/Settings_p.java
Unescape View File

@ -108,7 +108,7 @@ public final class Settings_p {
} else {
prop.put("use_proxyAccounts", "1"); //checked
/*s = env.getConfig("proxyAccount", "proxy:void");
pos = s.indexOf(":");
pos = s.indexOf(':');
if (pos < 0) {
prop.put("proxyuser","proxy");
} else {

0
htroot/env/grafics/RootStateNo.gif → htroot/env/grafics/NodeDisqualified.gif vendored
Unescape View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 875 B

After

Width:  |  Height:  |  Size: 875 B

BIN
htroot/env/grafics/NodeDisqualifiedIPv4.gif vendored
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 95 B

BIN
htroot/env/grafics/NodeDisqualifiedIPv6.gif vendored
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 94 B

0
htroot/env/grafics/RootStateYes.gif → htroot/env/grafics/NodeQualified.gif vendored
Unescape View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 130 B

After

Width:  |  Height:  |  Size: 130 B

BIN
htroot/env/grafics/NodeQualifiedIPv4.gif vendored
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 124 B

BIN
htroot/env/grafics/NodeQualifiedIPv6.gif vendored
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 94 B

2
htroot/env/templates/submenuComputation.template vendored
Unescape View File

@ -11,8 +11,8 @@
<div class="SubMenugroup">
<h3>Processes</h3>
<ul class="SubMenu">
<li><a href="Threaddump_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Thread Dump</a></li>
<li><a href="ViewLog_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Server Log</a></li>
<li><a href="Threaddump_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Thread Dump</a></li>
<li><a href="PerformanceConcurrency_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Concurrent Indexing</a></li>
<li><a href="PerformanceMemory_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Memory Usage</a></li>
<li><a href="PerformanceSearch_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Search Sequence</a></li>

2
htroot/index.html
Unescape View File

@ -56,7 +56,7 @@
<form class="search form-inline" action="yacysearch.html" method="get" id="searchform" accept-charset="UTF-8">
<fieldset class="maininput">
<div class="input-group">
<input name="query" id="search" type="text" size="40" maxlength="80" value="#[former]#" onFocus="this.select()" class="form-control searchinput typeahead" />
<input name="query" id="search" type="text" size="40" maxlength="80" value="#[former]#" #(focus)#::autofocus="autofocus"#(/focus)# onFocus="this.select()" class="form-control searchinput typeahead" />
<div class="input-group-btn">
<button type="submit" id="Enter" class="btn btn-primary">Search</button>
</div>

2
htroot/index.java
Unescape View File

@ -79,7 +79,6 @@ public class index {
final int maximumRecords = sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10);
final String prefermaskfilter = (post == null) ? "" : post.get("prefermaskfilter", "");
final String constraint = (post == null) ? "" : post.get("constraint", "");
final String cat = (post == null) ? "href" : post.get("cat", "href");
final int type = (post == null) ? 0 : post.getInt("type", 0);
//final boolean indexDistributeGranted = sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW, true);
@ -118,7 +117,6 @@ public class index {
prop.putHTML("searchoptions_prefermaskoptions_prefermaskfilter", prefermaskfilter);
prop.put("searchoptions_indexofChecked", "");
prop.put("results", "");
prop.putHTML("cat", cat);
prop.put("type", type);
prop.put("depth", "0");
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);

4
htroot/js/highslide/highslide.js
Unescape View File

@ -2,7 +2,7 @@
Name: Highslide JS
Version: 3.3.9 (February 15 2008)
Config: default
Author: Torstein Hønsi
Author: Torstein H<EFBFBD>nsi
Support: http://vikjavev.no/highslide/forum
Licence:
@ -1197,7 +1197,7 @@ writeCaption : function() {
null, null, true); // to get height
this.caption.innerHTML = '';
this.caption.appendChild(temp);
height = this.caption.childNodes[0].offsetHeight;
height = this.caption.offsetHeight;
this.caption.innerHTML = this.caption.childNodes[0].innerHTML;
}
hs.setStyles(this.caption, { overflow: 'hidden', height: 0, zIndex: 2, marginTop: 0 });

4
htroot/yacyinteractive.java
Unescape View File

@ -53,8 +53,8 @@ public class yacyinteractive {
final String maximumRecords = (post == null) ? sb.getConfig(SwitchboardConstants.SEARCH_ITEMS, "10") : post.get("maximumRecords", "");
final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1");
prop.putHTML("query", query);
prop.put("startRecord", startRecord);
prop.put("maximumRecords", maximumRecords);
prop.putHTML("startRecord", startRecord);
prop.putHTML("maximumRecords", maximumRecords);
prop.putHTML("querys", query.replaceAll(" ", "+"));
prop.put("serverlist", query.isEmpty() ? 1 : 0);
prop.put("focus", focus ? 1 : 0);

3
htroot/yacysearch.html
Unescape View File

@ -92,7 +92,7 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<form class="search small" name="searchform" action="" method="get" accept-charset="UTF-8" style="position:fixed;top:8px;z-index:1052;max-width:500px;">
<div class="input-group">
<input type="text" class="form-control searchinput typeahead" size="40" maxlength="200" placeholder="#[promoteSearchPageGreeting]#" name="query" value="#[former]#" onFocus="this.select()" id="search" onclick="document.getElementById('Enter').innerHTML = 'search'"/>
<input type="text" class="form-control searchinput typeahead" size="40" maxlength="200" placeholder="#[promoteSearchPageGreeting]#" name="query" value="#[former]#" autofocus="autofocus" onFocus="this.select()" id="search" onclick="document.getElementById('Enter').innerHTML = 'search'"/>
<div class="input-group-btn">
<button id="Enter" class="btn btn-default" type="submit">search</button>
</div>
@ -106,7 +106,6 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<input type="hidden" name="nav" value="#[search.navigation]#" />
<input type="hidden" name="prefermaskfilter" value="#[prefermaskfilter]#" />
<input type="hidden" name="depth" value="#[depth]#" />
<input type="hidden" name="cat" value="#[cat]#" />
<input type="hidden" name="constraint" value="#[constraint]#" />
<input type="hidden" name="meanCount" value="#[meanCount]#" />
</form>

20
htroot/yacysearch.java
Unescape View File

@ -28,7 +28,6 @@
// if the shell's current path is HTROOT
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.ConcurrentModificationException;
@ -168,10 +167,8 @@ public class yacysearch {
prop.put("resource", "global");
prop.put("urlmaskfilter", (post == null) ? ".*" : post.get("urlmaskfilter", ".*"));
prop.put("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", ""));
prop.put("tenant", (post == null) ? "" : post.get("tenant", ""));
prop.put("indexof", "off");
prop.put("constraint", "");
prop.put("cat", "href");
prop.put("depth", "0");
prop.put(
"search.verify",
@ -348,7 +345,7 @@ public class yacysearch {
}
}
if ( !block && (post == null || post.get("cat", "href").equals("href")) ) {
if ( !block ) {
String urlmask = null;
String tld = null;
String inlink = null;
@ -858,7 +855,6 @@ public class yacysearch {
prop.put("num-results", "3");
}
prop.put("cat", "href");
prop.put("depth", "0");
}
@ -871,13 +867,9 @@ public class yacysearch {
prop.putHTML("prefermaskfilter", prefermask);
prop.put("indexof", (indexof) ? "on" : "off");
prop.put("constraint", (constraint == null) ? "" : constraint.exportB64());
prop.put("search.verify", snippetFetchStrategy == null
? sb.getConfig("search.verify", "iffresh")
: snippetFetchStrategy.toName());
prop.put(
"search.navigation",
(post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
prop.put("contentdom", (post == null ? "text" : post.get("contentdom", "text")));
prop.put("search.verify", snippetFetchStrategy == null ? sb.getConfig("search.verify", "iffresh") : snippetFetchStrategy.toName());
prop.put("search.navigation", (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
prop.putHTML("contentdom", (post == null ? "text" : post.get("contentdom", "text")));
// for RSS: don't HTML encode some elements
prop.putXML("rss_query", originalquerystring);
@ -886,8 +878,8 @@ public class yacysearch {
sb.localSearchLastAccess = System.currentTimeMillis();
// hostname and port (assume locahost if nothing helps)
final InetAddress hostIP = Domains.myPublicLocalIP();
prop.put("myhost", hostIP != null ? hostIP.getHostAddress() : Domains.LOCALHOST);
final String hostIP = sb.peers.mySeed().getIP();
prop.put("myhost", hostIP != null ? hostIP : Domains.LOCALHOST);
prop.put("myport", sb.getConfig("port", "8090"));
// return rewrite properties

2
htroot/yacysearch_location.html
Unescape View File

@ -172,7 +172,7 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<h2>#[promoteSearchPageGreeting]#</h2>
<div class="yacylogo"><a href="#[promoteSearchPageGreeting.homepage]#" class="yacylogo"><img src="#[promoteSearchPageGreeting.smallImage]#" alt="yacysearch" /></a></div>
<fieldset class="yacys">
<input type="text" value="#(initsearch)#::#[query]##(/initsearch)#" id="query" size="42" onFocus="this.select()" />
<input type="text" value="#(initsearch)#::#[query]##(/initsearch)#" id="query" size="42" autofocus="autofocus" onFocus="this.select()" />
<input type="submit" onclick="search(); return false;" value="search" onsubmit="search(); return false;" />
</fieldset>
<div id="resultline"></div>

2
htroot/yacysearchitem.html
Unescape View File

@ -29,7 +29,7 @@
#(showMetadata)#::&nbsp;|&nbsp;<a href="solr/select?q=id:%22#[urlhash]#%22&defType=edismax&start=0&rows=1&core=collection1&wt=html" target="_blank" onclick="return hs.htmlExpand(this, { objectType: 'ajax'} )">Metadata</a>#(/showMetadata)#
#(showParser)#::&nbsp;|&nbsp;<a href="ViewFile.html?urlHash=#[urlhash]#&amp;words=#[words]#" target="_blank">Parser</a>#(/showParser)#
#(showCitation)#::&nbsp;|&nbsp;<a href="api/citation.html?hash=#[urlhash]#" target="_blank">Citations</a>#(/showCitation)#
#(showPictures)#::&nbsp;|&nbsp;<a href="yacysearch.html?cat=image&amp;url=#[link]#&amp;query=#[former]#">Pictures</a>#(/showPictures)#
#(showPictures)#::&nbsp;|&nbsp;<a href="yacysearch.html?contentdom=image&amp;url=#[link]#&amp;query=#[former]# inurl:#[link]#" target="_blank">Pictures</a>#(/showPictures)#
#(showCache)#::&nbsp;|&nbsp;<a href="CacheResource_p.html?url=#[link]#" target="_blank">Cache</a>#(/showCache)#
#(showProxy)#::&nbsp;|&nbsp;<a href="proxy.html?url=#[link]#" target="_blank">Augmented Browsing</a>#(/showProxy)#
#(showHostBrowser)#::&nbsp;|&nbsp;<a href="HostBrowser.html?path=#[link]#"><img src="env/grafics/minitree.png" width="15" height="8"/></a>#(/showHostBrowser)#

2
htroot/yacysearchtrailer.html
Unescape View File

@ -30,7 +30,7 @@
data-content="You can switch to 'Peer-to-Peer Mode' which will cause that your search is done using the other peers in the YaCy network."
>
<input
data-label-text="Privacy"
data-label-text="Peer-to-Peer"
data-on-text="<span>&nbsp;&nbsp;&nbsp;Peer-to-Peer&nbsp;&nbsp;&nbsp;&nbsp;</span>"
data-off-text="<span>&nbsp;&nbsp;&nbsp;&nbsp;Stealth&nbsp;Mode&nbsp;&nbsp;&nbsp;</span>"
data-on-color="warning" data-off-color="success"

4
htroot/yacysearchtrailer.java
Unescape View File

@ -525,5 +525,5 @@ public class yacysearchtrailer {
}
}
//http://localhost:8090/yacysearch.html?query=java+&amp;maximumRecords=10&amp;resource=local&amp;verify=cacheonly&amp;nav=hosts,authors,namespace,topics,filetype,protocol&amp;urlmaskfilter=ftp://.*&amp;prefermaskfilter=&amp;cat=href&amp;constraint=&amp;contentdom=text&amp;former=java+%2Fftp&amp;startRecord=0
//http://localhost:8090/yacysearch.html?query=java+&amp;maximumRecords=10&amp;resource=local&amp;verify=cacheonly&amp;nav=hosts,authors,namespace,topics,filetype,protocol&amp;urlmaskfilter=.*&amp;prefermaskfilter=&amp;cat=href&amp;constraint=&amp;contentdom=text&amp;former=java+%2Fvocabulary%2FGewerke%2FTore&amp;startRecord=0
//http://localhost:8090/yacysearch.html?query=java+&amp;maximumRecords=10&amp;resource=local&amp;verify=cacheonly&amp;nav=hosts,authors,namespace,topics,filetype,protocol&amp;urlmaskfilter=ftp://.*&amp;prefermaskfilter=&amp;constraint=&amp;contentdom=text&amp;former=java+%2Fftp&amp;startRecord=0
//http://localhost:8090/yacysearch.html?query=java+&amp;maximumRecords=10&amp;resource=local&amp;verify=cacheonly&amp;nav=hosts,authors,namespace,topics,filetype,protocol&amp;urlmaskfilter=.*&amp;prefermaskfilter=&amp;constraint=&amp;contentdom=text&amp;former=java+%2Fvocabulary%2FGewerke%2FTore&amp;startRecord=0

BIN
lib/jsoup-1.6.3.jar
View File

Binary file not shown.

BIN
lib/jsoup-1.8.1.jar
View File

Binary file not shown.

2
nbproject/project.xml
Unescape View File

@ -83,7 +83,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.8.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-io-2.3.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.8.6.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-16.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.3.5.jar;lib/httpcore-4.3.2.jar;lib/httpmime-4.3.5.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.8.6.jar;lib/jetty-client-9.2.2.v20140723.jar;lib/jetty-continuation-9.2.2.v20140723.jar;lib/jetty-deploy-9.2.2.v20140723.jar;lib/jetty-http-9.2.2.v20140723.jar;lib/jetty-io-9.2.2.v20140723.jar;lib/jetty-jmx-9.2.2.v20140723.jar;lib/jetty-proxy-9.2.2.v20140723.jar;lib/jetty-security-9.2.2.v20140723.jar;lib/jetty-server-9.2.2.v20140723.jar;lib/jetty-servlet-9.2.2.v20140723.jar;lib/jetty-servlets-9.2.2.v20140723.jar;lib/jetty-util-9.2.2.v20140723.jar;lib/jetty-webapp-9.2.2.v20140723.jar;lib/jetty-xml-9.2.2.v20140723.jar;lib/jsch-0.1.50.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.9.0.jar;lib/lucene-analyzers-phonetic-4.9.0.jar;lib/lucene-classification-4.9.0.jar;lib/lucene-codecs-4.9.0.jar;lib/lucene-core-4.9.0.jar;lib/lucene-facet-4.9.0.jar;lib/lucene-grouping-4.9.0.jar;lib/lucene-highlighter-4.9.0.jar;lib/lucene-join-4.9.0.jar;lib/lucene-memory-4.9.0.jar;lib/lucene-misc-4.9.0.jar;lib/lucene-queries-4.9.0.jar;lib/lucene-queryparser-4.9.0.jar;lib/lucene-spatial-4.9.0.jar;lib/lucene-suggest-4.9.0.jar;lib/metadata-extractor-2.6.2.jar;lib/noggit-0.5.jar;lib/org.restlet.jar;lib/pdfbox-1.8.6.jar;lib/poi-3.10-FINAL-20140208.jar;lib/poi-scratchpad-3.10-FINAL-20140208.jar;lib/slf4j-api-1.7.6.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.9.0.jar;lib/solr-solrj-4.9.0.jar;lib/spatial4j-0.4.1.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.6.jar</classpath>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.8.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-io-2.3.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.8.7.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-16.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.3.5.jar;lib/httpcore-4.3.2.jar;lib/httpmime-4.3.5.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.8.7.jar;lib/jetty-client-9.2.3.v20140905.jar;lib/jetty-continuation-9.2.3.v20140905.jar;lib/jetty-deploy-9.2.3.v20140905.jar;lib/jetty-http-9.2.3.v20140905.jar;lib/jetty-io-9.2.3.v20140905.jar;lib/jetty-jmx-9.2.3.v20140905.jar;lib/jetty-proxy-9.2.3.v20140905.jar;lib/jetty-security-9.2.3.v20140905.jar;lib/jetty-server-9.2.3.v20140905.jar;lib/jetty-servlet-9.2.3.v20140905.jar;lib/jetty-servlets-9.2.3.v20140905.jar;lib/jetty-util-9.2.3.v20140905.jar;lib/jetty-webapp-9.2.3.v20140905.jar;lib/jetty-xml-9.2.3.v20140905.jar;lib/jsch-0.1.50.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.1.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.9.0.jar;lib/lucene-analyzers-phonetic-4.9.0.jar;lib/lucene-classification-4.9.0.jar;lib/lucene-codecs-4.9.0.jar;lib/lucene-core-4.9.0.jar;lib/lucene-facet-4.9.0.jar;lib/lucene-grouping-4.9.0.jar;lib/lucene-highlighter-4.9.0.jar;lib/lucene-join-4.9.0.jar;lib/lucene-memory-4.9.0.jar;lib/lucene-misc-4.9.0.jar;lib/lucene-queries-4.9.0.jar;lib/lucene-queryparser-4.9.0.jar;lib/lucene-spatial-4.9.0.jar;lib/lucene-suggest-4.9.0.jar;lib/metadata-extractor-2.6.2.jar;lib/noggit-0.5.jar;lib/org.restlet.jar;lib/pdfbox-1.8.7.jar;lib/poi-3.10-FINAL-20140208.jar;lib/poi-scratchpad-3.10-FINAL-20140208.jar;lib/slf4j-api-1.7.6.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.9.0.jar;lib/solr-solrj-4.9.0.jar;lib/spatial4j-0.4.1.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.6.jar</classpath>
<built-to>lib/yacycore.jar</built-to>
<source-level>1.7</source-level>
</compilation-unit>

2
pom.xml
Unescape View File

@ -408,7 +408,7 @@
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.3</version>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>log4j</groupId>

12
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -66,8 +66,6 @@ import net.yacy.crawler.retrieval.Response;
*/
public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolURL> {
public static final MultiProtocolURL POISON = new MultiProtocolURL(); // poison pill for concurrent link generators
private static final long serialVersionUID = -1173233022912141884L;
private static final long SMB_TIMEOUT = 5000;
@ -155,7 +153,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
url = "file://" + url;
}
int p = url.indexOf(':');
int p = url.indexOf("://");
if (p < 0) {
url = "http://" + url;
p = 4;
@ -373,6 +371,11 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
escape();
}
/**
* creates MultiProtocolURL
* if path contains '?' search part is automatically created by splitting input into path and searchpart
* dto for anchor's ('#')
*/
public MultiProtocolURL(final String protocol, String host, final int port, final String path) throws MalformedURLException {
if (protocol == null) throw new MalformedURLException("protocol is null");
if (host.indexOf(':') >= 0 && host.charAt(0) != '[') host = '[' + host + ']'; // IPv6 host must be enclosed in square brackets
@ -521,9 +524,8 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
if (i < len - 6 && "amp;".equals(s.substring(i + 1, i + 5).toLowerCase())) {
sbuf.append((char)ch); // leave it that way, it is used the right way
} else {
sbuf.append("&amp;"); // this must be urlencoded
sbuf.append("%26"); // this must be urlencoded
}
sbuf.append((char)ch);
} else if (ch == '#') { // RFC 1738 2.2 unsafe char is _not_ encoded because it may already be used for encoding
sbuf.append((char)ch);
} else if (ch == '!' || ch == ':' // unreserved

106
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -40,6 +40,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@ -152,28 +153,43 @@ public class Domains {
// fill a cache of local host names
for (final InetAddress a: myHostAddresses) {
String hostaddressP = a.getHostAddress();
int p = hostaddressP.indexOf('%');
if (p > 0) hostaddressP = hostaddressP.substring(0, p);
Set<String> hns = new LinkedHashSet<>();
// generate alternative representations of IPv6 addresses which are needed to check access on the interface (i.e. localhost check)
if (hostaddressP.indexOf("::") < 0) {
hns.add(hostaddressP.replaceFirst(":0:0:0:0:0:0:", "::"));
hns.add(hostaddressP.replaceFirst(":0:0:0:0:0:", "::"));
hns.add(hostaddressP.replaceFirst(":0:0:0:0:", "::"));
hns.add(hostaddressP.replaceFirst(":0:0:0:", "::"));
hns.add(hostaddressP.replaceFirst(":0:0:", "::"));
hns.add(hostaddressP.replaceFirst(":0:", "::"));
}
hns.add(hostaddressP);
final String hostname = getHostName(a);
if (hostname != null) {
myHostNames.add(hostname);
myHostNames.add(a.getHostAddress());
}
if (hostname != null) myHostNames.add(hostname);
myHostNames.addAll(hns);
for (String hostaddress: hns) {
if (hostaddress.contains("::0:") || hostaddress.contains(":0::")) continue; // not common (but possible); we skip that
// we write the local tests into variables to be able to debug these values
boolean isAnyLocalAddress = a.isAnyLocalAddress();
boolean isLinkLocalAddress = a.isLinkLocalAddress(); // true i.e. for localhost/fe80:0:0:0:0:0:0:1%1, myhost.local/fe80:0:0:0:223:dfff:fedf:30ce%7
boolean isLoopbackAddress = a.isLoopbackAddress(); // true i.e. for localhost/0:0:0:0:0:0:0:1, localhost/127.0.0.1
boolean isSiteLocalAddress = a.isSiteLocalAddress(); // true i.e. for myhost.local/192.168.1.33
if (isAnyLocalAddress || isLinkLocalAddress || isLoopbackAddress || isSiteLocalAddress) {
ConcurrentLog.info("Domain Init", "local host address: " + a + " (local)");
ConcurrentLog.info("Domain Init", "local host address: " + hostaddress + " (local)");
localHostAddresses.add(a);
if (hostname != null) {localHostNames.add(hostname); localHostNames.add(a.getHostAddress());}
if (hostname != null) {localHostNames.add(hostname); localHostNames.add(hostaddress);}
} else {
ConcurrentLog.info("Domain Init", "local host address: " + a + " (public)");
ConcurrentLog.info("Domain Init", "local host address: " + hostaddress + " (public)");
if (a instanceof Inet4Address) {
publicIPv4HostAddresses.add(a);
if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(a.getHostAddress());}
if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(hostaddress);}
} else {
publicIPv6HostAddresses.add(a);
if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(a.getHostAddress());}
if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(hostaddress);}
}
}
}
}
@ -791,12 +807,84 @@ public class Domains {
final private static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newFixedThreadPool(20));
/**
* strip off any parts of an url, address string (containing host/ip:port) or raw IPs/Hosts,
* considering that the host may also be an (IPv4) IP or a IPv6 IP in brackets.
* @param target
* @return a host name or IP string
*/
public static String stripToHostName(String target) {
// normalize
if (target == null || target.isEmpty()) return null;
target = target.toLowerCase().trim(); // we can lowercase this because host names are case-insensitive
// extract the address (host:port) part (applies if this is an url)
int p = target.indexOf("://");
if (p > 0) target = target.substring(p + 3);
p = target.indexOf('/');
if (p > 0) target = target.substring(0, p);
// IPv4 / host heuristics
p = target.lastIndexOf(':');
if ( p < 0 ) {
// may be IPv4 or IPv6, we chop off brackets if exist
if (target.charAt(0) == '[') target = target.substring(1);
if (target.charAt(target.length() - 1) == ']') target = target.substring(0, target.length() - 1);
p = target.lastIndexOf('%');
if (p > 0) target = target.substring(0, p);
return target;
}
// the ':' at pos p may be either a port divider or a part of an IPv6 address
if (target.charAt(p - 1) == ']') {
target = target.substring(1, p - 1);
p = target.lastIndexOf('%');
if (p > 0) target = target.substring(0, p);
return target;
}
// the ':' must be a port divider
target = target.substring(0, p);
p = target.lastIndexOf('%');
if (p > 0) target = target.substring(0, p);
return target;
}
public static int stripToPort(String target) {
int port = 80; // default port
// normalize
if (target == null || target.isEmpty()) return port;
target = target.toLowerCase().trim(); // we can lowercase this because host names are case-insensitive
// extract the address (host:port) part (applies if this is an url)
int p = target.indexOf("://");
if (p > 0) {
String protocol = target.substring(0, p);
target = target.substring(p + 3);
if ("https".equals(protocol)) port = 443;
if ("ftp".equals(protocol)) port = 21;
if ("smb".equals(protocol)) port = 445;
}
p = target.indexOf('/');
if (p > 0) target = target.substring(0, p);
// IPv4 / host heuristics
p = target.lastIndexOf(':');
if ( p < 0 ) return port;
// the ':' must be a port divider
port = Integer.parseInt(target.substring(p + 1));
return port;
}
/**
* resolve a host address using a local DNS cache and a DNS lookup if necessary
* @param clienthost
* @return the hosts InetAddress or null if the address cannot be resolved
*/
public static InetAddress dnsResolve(final String host0) {
// consider to call stripToHostName() before calling this
if (host0 == null || host0.isEmpty()) return null;
final String host = host0.toLowerCase().trim();

16
source/net/yacy/cora/protocol/HeaderFramework.java
Unescape View File

@ -574,7 +574,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
//String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
int port, pos;
if ((pos = host.indexOf(':')) < 0) {
if ((pos = host.lastIndexOf(':')) < 0 || host.charAt(pos - 1) != ']') {
port = 80;
} else {
port = NumberTools.parseIntDecSubstring(host, pos + 1);
@ -675,11 +675,11 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
*
* @param name Cookie name
* @param value Cookie value
* @param expires when should this cookie be autmatically deleted. If <b>null</b> - cookie will stay forever
* @param expires when should this cookie be automatically deleted. If <b>null</b> - cookie will stay forever
* @param path Path the cookie belongs to. Default - "/". Can be <b>null</b>.
* @param domain Domain this cookie belongs to. Default - domain name. Can be <b>null</b>.
*
* Note: this cookie will be sent over each connection independend if it is safe connection or not.
* Note: this cookie will be sent over each connection independent if it is safe connection or not.
* @see further documentation: <a href="http://docs.sun.com/source/816-6408-10/cookies.htm">docs.sun.com</a>
*/
public void setCookie(final String name, final String value, final String expires, final String path, final String domain)
@ -691,10 +691,10 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
*
* @param name Cookie name
* @param value Cookie value
* @param expires when should this cookie be autmatically deleted. If <b>null</b> - cookie will stay forever
* @param expires when should this cookie be automatically deleted. If <b>null</b> - cookie will stay forever
* @param path Path the cookie belongs to. Default - "/". Can be <b>null</b>.
*
* Note: this cookie will be sent over each connection independend if it is safe connection or not.
* Note: this cookie will be sent over each connection independent if it is safe connection or not.
* @see further documentation: <a href="http://docs.sun.com/source/816-6408-10/cookies.htm">docs.sun.com</a>
*/
public void setCookie(final String name, final String value, final String expires, final String path)
@ -706,9 +706,9 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
*
* @param name Cookie name
* @param value Cookie value
* @param expires when should this cookie be autmatically deleted. If <b>null</b> - cookie will stay forever
* @param expires when should this cookie be automatically deleted. If <b>null</b> - cookie will stay forever
*
* Note: this cookie will be sent over each connection independend if it is safe connection or not.
* Note: this cookie will be sent over each connection independent if it is safe connection or not.
* @see further documentation: <a href="http://docs.sun.com/source/816-6408-10/cookies.htm">docs.sun.com</a>
*/
public void setCookie(final String name, final String value, final String expires)
@ -721,7 +721,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
* @param name Cookie name
* @param value Cookie value
*
* Note: this cookie will be sent over each connection independend if it is safe connection or not. This cookie never expires
* Note: this cookie will be sent over each connection independent if it is safe connection or not. This cookie never expires
* @see further documentation: <a href="http://docs.sun.com/source/816-6408-10/cookies.htm">docs.sun.com</a>
*/
public void setCookie(final String name, final String value )

1
source/net/yacy/crawler/data/Cache.java
Unescape View File

@ -201,6 +201,7 @@ public final class Cache {
public static void store(final DigestURL url, final ResponseHeader responseHeader, final byte[] file) throws IOException {
if (maxCacheSize == 0) return;
if (responseHeader.getXRobotsTag().contains("noarchive")) return; // don't cache, see http://noarchive.net/
if (responseHeader == null) throw new IOException("Cache.store of url " + url.toNormalform(false) + " not possible: responseHeader == null");
if (file == null) throw new IOException("Cache.store of url " + url.toNormalform(false) + " not possible: file == null");
log.info("storing content of url " + url.toNormalform(false) + ", " + file.length + " bytes");

20
source/net/yacy/data/WorkTables.java
Unescape View File

@ -28,6 +28,7 @@ package net.yacy.data;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -42,6 +43,7 @@ import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.http.HTTPClient;
@ -222,7 +224,6 @@ public class WorkTables extends Tables {
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setTimout(120000);
Tables.Row row;
String url;
LinkedHashMap<String, Integer> l = new LinkedHashMap<String, Integer>();
for (final String pk: pks) {
row = null;
@ -234,15 +235,20 @@ public class WorkTables extends Tables {
ConcurrentLog.logException(e);
}
if (row == null) continue;
url = "http://" + host + ":" + port + UTF8.String(row.get(WorkTables.TABLE_API_COL_URL));
url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());
ConcurrentLog.info("WorkTables", "executing url: " + url);
String theapicall = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)) + "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());
try {
client.GETbytes(url, username, pass, false);
l.put(url, client.getStatusCode());
// use 4 param MultiProtocolURL to allow api_row_url with searchpart (like url?p=a&p2=b ) in client.GETbytes()
MultiProtocolURL url = new MultiProtocolURL("http", host, port, theapicall);
ConcurrentLog.info("WorkTables", "executing url: " + url.toNormalform(true));
try {
client.GETbytes(url, username, pass, false); // use GETbytes(MultiProtocolURL,..) form to allow url in parameter (&url=path%
l.put(url.toNormalform(true), client.getStatusCode());
} catch (final IOException e) {
ConcurrentLog.logException(e);
l.put(url, -1);
l.put(url.toString(), -1);
}
} catch (MalformedURLException ex) {
ConcurrentLog.warn("APICALL", "wrong url in apicall " + theapicall);
}
}
return l;

2
source/net/yacy/document/parser/images/metadataImageParser.java
Unescape View File

@ -157,7 +157,7 @@ public class metadataImageParser extends AbstractParser implements Parser {
return new Document[]{new Document(
location,
mimeType,
"UTF-8",
documentCharset,
this,
new HashSet<String>(0), // languages
keywords == null ? new String[]{} : keywords.split(keywords.indexOf(',') > 0 ? "," : " "), // keywords

10
source/net/yacy/http/AbstractRemoteHandler.java
Unescape View File

@ -93,19 +93,17 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha
String host = request.getHeader("Host");
if (host == null) return; // no proxy request, continue processing by handlers
int hostSplitPos = host.indexOf(':');
String hostOnly = hostSplitPos < 0 ? host : host.substring(0, hostSplitPos);
String hostOnly = Domains.stripToHostName(host);
if (localVirtualHostNames.contains(hostOnly)) return; // no proxy request (quick check), continue processing by handlers
if (Domains.isLocal(hostOnly, null)) return; // no proxy, continue processing by handlers
if (hostOnly.startsWith(sb.peers.myIP())) { // remote access to my external IP, continue processing by handlers
localVirtualHostNames.add(sb.peers.myIP()); // not available on init, add it now for quickcheck
if (sb.peers.myIPs().contains(hostOnly)) { // remote access to my external IP, continue processing by handlers
localVirtualHostNames.addAll(sb.peers.myIPs()); // not available on init, add it now for quickcheck
return;
}
InetAddress resolvedIP = Domains.dnsResolve(hostOnly); // during testing isLocal() failed to resolve domain against publicIP
if (resolvedIP != null && sb.myPublicIP().equals(resolvedIP.getHostAddress())) {
if (resolvedIP != null && sb.myPublicIPs().contains(resolvedIP.getHostAddress())) {
localVirtualHostNames.add(resolvedIP.getHostName()); // remember resolved hostname
//localVirtualHostNames.add(resolved.getHostAddress()); // might change ?
return;

12
source/net/yacy/http/YacyDomainHandler.java
Unescape View File

@ -72,16 +72,8 @@ public class YacyDomainHandler extends AbstractHandler implements Handler {
path = "";
hostPort = resolved;
}
int posPort = hostPort.lastIndexOf(':');
String newHost;
int newPort;
if (posPort >= 0) {
newHost = hostPort.substring(0, posPort);
newPort = Integer.parseInt(hostPort.substring(posPort + 1));
} else {
newHost = hostPort;
newPort = 80;
}
int newPort = Domains.stripToPort(hostPort);
String newHost = Domains.stripToHostName(hostPort);
if (alternativeResolvers.myIPs().contains(newHost)) return;
if (Domains.isLocal(newHost, null)) return;
RequestDispatcher dispatcher = request.getRequestDispatcher(path + target);

4
source/net/yacy/http/servlets/UrlProxyServlet.java
Unescape View File

@ -206,9 +206,9 @@ public class UrlProxyServlet extends ProxyServlet implements Servlet {
final String servletstub = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + request.getServletPath() + "?url=";
Document doc;
try {
doc = Jsoup.parse(proxyout, UTF8.charset.name(), proxyurl.toString());
doc = Jsoup.parse(proxyout, proxyResponseHeader.getCharacterEncoding(), proxyurl.toString());
} catch (IOException eio) {
response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,"Proxy: parser error on " + proxyurl.toString());
response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,"Proxy: parser error on " + proxyurl.toString() +"\n\n"+ eio.getMessage());
return;
}

95
source/net/yacy/peers/Network.java
Unescape View File

@ -185,14 +185,7 @@ public class Network
+ this.sb.peers.sizeConnected()
+ " new peer(s)");
}
final int newSeeds = publishMySeed(false);
if ( newSeeds > 0 ) {
log.info("received "
+ newSeeds
+ " new peer(s), know a total of "
+ this.sb.peers.sizeConnected()
+ " different peers");
}
publishMySeed(false);
}
// use our own formatter to prevent concurrency locks with other processes
@ -202,7 +195,7 @@ public class Network
protected class publishThread extends Thread
{
int added;
private Map<String, String> result;
private final Seed seed;
private final Semaphore sync;
private final List<Thread> syncList;
@ -219,14 +212,17 @@ public class Network
this.syncList = syncList;
this.seed = seed;
this.added = 0;
this.result = null;
}
@Override
public final void run() {
try {
this.added = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed);
if ( this.added < 0 ) {
for (String ip: this.seed.getIPs()) {
this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash);
if (this.result != null) break;
}
if ( this.result == null ) {
// no or wrong response, delete that address
final String cause = "peer ping to peer resulted in error response (added < 0)";
log.info("publish: disconnected "
@ -324,7 +320,7 @@ public class Network
}
}
private int publishMySeed(final boolean force) {
private boolean publishMySeed(final boolean force) {
try {
// call this after the httpd was started up
@ -349,9 +345,7 @@ public class Network
// getting a list of peers to contact
if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN).equals(Seed.PEERTYPE_VIRGIN) ) {
if ( attempts > PING_INITIAL ) {
attempts = PING_INITIAL;
}
if (attempts > PING_INITIAL) attempts = PING_INITIAL;
final Set<byte[]> ch = Switchboard.getSwitchboard().clusterhashes;
seeds = DHTSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
@ -363,9 +357,7 @@ public class Network
seed = seeds.get(hash);
if (seed == null) {
seed = this.sb.peers.get(hash);
if ( seed == null ) {
continue;
}
if (seed == null) continue;
}
seeds.put(hash, seed);
}
@ -385,12 +377,8 @@ public class Network
seeds = DHTSelection.seedsByAge(this.sb.peers, false, attempts); // best for seed list maintenance/cleaning
}
if ( seeds == null || seeds.isEmpty() ) {
return 0;
}
if ( seeds.size() < attempts ) {
attempts = seeds.size();
}
if (seeds == null || seeds.isEmpty()) return false;
if (seeds.size() < attempts) attempts = seeds.size();
// This will try to get Peers that are not currently in amIAccessibleDB
final Iterator<Seed> si = seeds.values().iterator();
@ -408,14 +396,13 @@ public class Network
log.severe("publishMySeed: problem with news encoding", e);
}
this.sb.peers.mySeed().setUnusedFlags();
int newSeeds = -1;
//if (seeds.length > 1) {
// holding a reference to all started threads
int contactedSeedCount = 0;
final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads
final Semaphore sync = new Semaphore(attempts);
// going through the peer list and starting a new publisher thread for each peer
// go through the peer list and starting a new publisher thread for each peer
int i = 0;
while ( si.hasNext() ) {
seed = si.next();
@ -426,9 +413,7 @@ public class Network
i++;
final String address = seed.getPublicAddress(seed.getIP());
if ( log.isFine() ) {
log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug
}
if ( log.isFine() ) log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug
final String seederror = seed.isProper(false);
if ( (address == null) || (seederror != null) ) {
// we don't like that address, delete it
@ -456,15 +441,6 @@ public class Network
// getting a reference to the finished thread
final publishThread t = (publishThread) syncList.remove(0);
// getting the amount of new reported seeds
if ( t.added >= 0 ) {
if ( newSeeds == -1 ) {
newSeeds = t.added;
} else {
newSeeds += t.added;
}
}
}
int accessible = 0;
@ -486,19 +462,9 @@ public class Network
}
}
}
if ( log.isFine() ) {
log
.fine("DBSize before -> after Cleanup: "
+ dbSize
+ " -> "
+ amIAccessibleDB.size());
}
if (log.isFine()) log.fine("DBSize before -> after Cleanup: " + dbSize + " -> " + amIAccessibleDB.size());
}
log.info("PeerPing: I am accessible for "
+ accessible
+ " peer(s), not accessible for "
+ notaccessible
+ " peer(s).");
log.info("PeerPing: I am accessible for " + accessible + " peer(s), not accessible for " + notaccessible + " peer(s).");
if ( (accessible + notaccessible) > 0 ) {
final String newPeerType;
@ -517,11 +483,7 @@ public class Network
if ( this.sb.peers.mySeed().orVirgin().equals(newPeerType) ) {
log.info("PeerPing: myType is " + this.sb.peers.mySeed().orVirgin());
} else {
log.info("PeerPing: changing myType from '"
+ this.sb.peers.mySeed().orVirgin()
+ "' to '"
+ newPeerType
+ "'");
log.info("PeerPing: changing myType from '" + this.sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'");
this.sb.peers.mySeed().put(Seed.PEERTYPE, newPeerType);
}
} else {
@ -534,20 +496,11 @@ public class Network
this.sb.peers.saveMySeed();
// if we have an address, we do nothing
if ( this.sb.peers.mySeed().isProper(true) == null && !force ) {
return 0;
}
if ( newSeeds > 0 ) {
return newSeeds;
}
if (this.sb.peers.mySeed().isProper(true) == null) return true;
// still no success: ask own NAT or internet responder
//final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true");
//final String DI604pw = switchboard.getConfig("DI604pw", "");
// still no success
final String ip = this.sb.getConfig("staticIP", "");
//if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
// yacyCore.log.logDebug("DEBUG: new IP=" + ip);
if (Seed.isProperIP(ip)) {
this.sb.peers.mySeed().setIP(ip);
}
@ -556,7 +509,7 @@ public class Network
}
log.info("publish: no recipient found, our address is " + this.sb.peers.mySeed().getIPs());
this.sb.peers.saveMySeed();
return 0;
return false;
} catch (final InterruptedException e ) {
try {
log.info("publish: Interruption detected while publishing my seed.");
@ -606,12 +559,10 @@ public class Network
log.info("publish: Shutdown off all remaining publishing thread finished.");
} catch (final Exception ee ) {
log.warn(
"publish: Unexpected error while trying to shutdown all remaining publishing threads.",
e);
log.warn("publish: Unexpected error while trying to shutdown all remaining publishing threads.", e);
}
return 0;
return false;
}
}

78
source/net/yacy/peers/Protocol.java
Unescape View File

@ -167,11 +167,11 @@ public final class Protocol {
*
* @return the number of new seeds
*/
public static int hello(
public static Map<String, String> hello(
final Seed mySeed,
final PeerActions peerActions,
final Seed otherSeed) {
final String address = otherSeed.getPublicAddress(otherSeed.getIP());
final String targetAddress,
final String targetHash) {
Map<String, String> result = null;
final String salt = crypt.randomSalt();
@ -190,8 +190,8 @@ public final class Protocol {
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000);
content =
httpClient.POSTbytes(
new MultiProtocolURL("http://" + address + "/yacy/hello.html"),
Seed.b64Hash2hexHash(otherSeed.hash) + ".yacyh",
new MultiProtocolURL("http://" + targetAddress + "/yacy/hello.html"),
Seed.b64Hash2hexHash(targetHash) + ".yacyh",
parts,
false, true);
responseTime = System.currentTimeMillis() - start;
@ -201,9 +201,9 @@ public final class Protocol {
Network.log.info("yacyClient.hello thread '"
+ Thread.currentThread().getName()
+ "' interrupted.");
return -1;
return null;
}
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + address + "; exception: " + e.getMessage());
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetAddress + "; exception: " + e.getMessage());
// try again (go into loop)
result = null;
}
@ -211,41 +211,29 @@ public final class Protocol {
if (result == null || result.size() == 0) {
Network.log.info("yacyClient.hello result error: "
+ ((result == null) ? "result null" : ("result=" + result.toString())));
return -1;
return null;
}
Network.log.info("yacyClient.hello thread '"
+ Thread.currentThread().getName()
+ "' contacted peer at "
+ address
+ ", received "
+ ((content == null) ? "null" : content.length)
+ " bytes, time = "
+ responseTime
+ " milliseconds");
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetAddress + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");
// check consistency with expectation
Seed otherPeer = null;
String seed;
if ( (otherSeed.hash != null) && (otherSeed.hash.length() > 0) && ((seed = result.get("seed0")) != null) ) {
if ( (targetHash != null) && (targetHash.length() > 0) && ((seed = result.get("seed0")) != null) ) {
if ( seed.length() > Seed.maxsize ) {
Network.log.info("hello/client 0: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ")");
} else {
try {
// patch the remote peer address to avoid that remote peers spoof the network with wrong addresses
final int p = address.lastIndexOf(':');
if ( p < 0 ) return -1;
String h = address.substring(0, p);
if (h.charAt(0) == '[') h = h.substring(1);
if (h.charAt(h.length() - 1) == ']') h = h.substring(0, h.length() - 1);
InetAddress ie = Domains.dnsResolve(h);
String host = Domains.stripToHostName(targetAddress);
InetAddress ie = Domains.dnsResolve(host);
otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());
if ( !otherPeer.hash.equals(otherSeed.hash) ) {
Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " + otherPeer.hash + ", otherHash = " + otherSeed.hash);
return -1; // no success
if ( !otherPeer.hash.equals(targetHash) ) {
Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " + otherPeer.hash + ", otherHash = " + targetHash);
return null; // no success
}
} catch (final IOException e ) {
Network.log.info("yacyClient.hello: consistency error: other seed bad:" + e.getMessage() + ", seed=" + seed);
return -1; // no success
return null; // no success
}
}
}
@ -285,7 +273,7 @@ public final class Protocol {
accessible.IWasAccessed = false;
}
accessible.lastUpdated = System.currentTimeMillis();
Network.amIAccessibleDB.put(otherSeed.hash, accessible);
Network.amIAccessibleDB.put(targetHash, accessible);
/*
* If we were reported as junior we have to check if your port forwarding channel is broken
@ -313,7 +301,7 @@ public final class Protocol {
+ mytype
+ ", rejecting other peer.");
}
return -1;
return null;
}
if ( mySeed.orVirgin().equals(Seed.PEERTYPE_VIRGIN) ) {
mySeed.put(Seed.PEERTYPE, mytype);
@ -322,14 +310,13 @@ public final class Protocol {
final String error = mySeed.isProper(true);
if ( error != null ) {
Network.log.warn("yacyClient.hello mySeed error - not proper: " + error);
return -1;
return null;
}
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
// read the seeds that the peer returned and integrate them into own database
int i = 0;
int count = 0;
String seedStr;
Seed s;
final int connectedBefore = peerActions.sizeConnected();
@ -337,41 +324,30 @@ public final class Protocol {
// integrate new seed into own database
// the first seed, "seed0" is the seed of the responding peer
if ( seedStr.length() > Seed.maxsize ) {
Network.log.info("hello/client: rejected contacting seed; too large ("
+ seedStr.length()
+ " > "
+ Seed.maxsize
+ ")");
Network.log.info("hello/client: rejected contacting seed; too large ("+ seedStr.length() + " > " + Seed.maxsize + ")");
} else {
try {
if ( i == 1 ) {
final int p = address.indexOf(':');
if ( p < 0 ) {
return -1;
}
InetAddress ia = Domains.dnsResolve(address.substring(0, p));
String host = Domains.stripToHostName(targetAddress);
InetAddress ia = Domains.dnsResolve(host);
if (ia == null) continue;
final String host = ia.getHostAddress();
host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here
s = Seed.genRemoteSeed(seedStr, false, host);
} else {
s = Seed.genRemoteSeed(seedStr, false, null);
}
if ( peerActions.peerArrival(s, (i == 1)) ) {
count++;
}
peerActions.peerArrival(s, (i == 1));
} catch (final IOException e ) {
Network.log.info("hello/client: rejected contacting seed; bad ("
+ e.getMessage()
+ ")");
Network.log.info("hello/client: rejected contacting seed; bad (" + e.getMessage() + ")");
}
}
}
final int connectedAfter = peerActions.sizeConnected();
// update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), otherSeed.getName(), true, connectedAfter - connectedBefore), false);
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), targetHash, true, connectedAfter - connectedBefore), false);
return count;
return result;
}
public static Seed querySeed(final Seed target, final String seedHash) {

42
source/net/yacy/search/schema/CollectionConfiguration.java
Unescape View File

@ -47,6 +47,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.analysis.EnhancedTextProfileSignature;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.AnchorURL;
@ -200,7 +201,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
* @param doctype
* @return the normalized url
*/
public String addURIAttributes(final SolrInputDocument doc, final boolean allAttr, final DigestURL digestURL, final char doctype) {
public String addURIAttributes(final SolrInputDocument doc, final boolean allAttr, final DigestURL digestURL) {
add(doc, CollectionSchema.id, ASCII.String(digestURL.hash()));
if (allAttr || contains(CollectionSchema.host_id_s)) add(doc, CollectionSchema.host_id_s, digestURL.hosthash());
String us = digestURL.toNormalform(true);
@ -237,8 +238,6 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
if (allAttr || contains(CollectionSchema.url_file_name_s)) add(doc, CollectionSchema.url_file_name_s, filenameStub);
if (allAttr || contains(CollectionSchema.url_file_name_tokens_t)) add(doc, CollectionSchema.url_file_name_tokens_t, MultiProtocolURL.toTokens(filenameStub));
if (allAttr || contains(CollectionSchema.url_file_ext_s)) add(doc, CollectionSchema.url_file_ext_s, extension);
if (allAttr || contains(CollectionSchema.content_type)) add(doc, CollectionSchema.content_type, Response.doctype2mime(extension, doctype));
Map<String, String> searchpart = digestURL.getSearchpartMap();
if (searchpart == null) {
@ -253,13 +252,12 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
public SolrInputDocument metadata2solr(final URIMetadataNode md) {
final SolrInputDocument doc = new SolrInputDocument();
boolean allAttr = this.isEmpty();
SolrInputDocument doc = toSolrInputDocument(md); //urimetadatanode stores some values in private fields, add now to sorldocument
addURIAttributes(doc, allAttr, md.url(), md.doctype());
boolean allAttr = this.isEmpty();
addURIAttributes(doc, allAttr, md.url());
String title = md.dc_title();
if (allAttr || contains(CollectionSchema.title)) add(doc, CollectionSchema.title, new String[]{title});
if (allAttr || contains(CollectionSchema.title_count_i)) add(doc, CollectionSchema.title_count_i, 1);
if (allAttr || contains(CollectionSchema.title_chars_val)) {
Integer[] cv = new Integer[]{new Integer(title.length())};
@ -282,10 +280,6 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
add(doc, CollectionSchema.description_words_val, description_exist ? new Integer[]{new Integer(description.length() == 0 ? 0 : CommonPattern.SPACE.split(description).length)} : new Integer[0]);
}
if (allAttr || contains(CollectionSchema.author)) add(doc, CollectionSchema.author, md.dc_creator());
if (allAttr || contains(CollectionSchema.last_modified)) add(doc, CollectionSchema.last_modified, md.moddate());
if (allAttr || contains(CollectionSchema.wordcount_i)) add(doc, CollectionSchema.wordcount_i, md.wordCount());
String keywords = md.dc_subject();
Bitfield flags = md.flags();
if (flags.get(Condenser.flag_cat_indexof)) {
@ -310,13 +304,6 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
if (allAttr || contains(CollectionSchema.httpstatus_i)) add(doc, CollectionSchema.httpstatus_i, 200);
// fields that are in URIMetadataRow additional to yacy2solr basic requirement
if (allAttr || contains(CollectionSchema.load_date_dt)) add(doc, CollectionSchema.load_date_dt, md.loaddate());
if (allAttr || contains(CollectionSchema.fresh_date_dt)) add(doc, CollectionSchema.fresh_date_dt, md.freshdate());
if ((allAttr || contains(CollectionSchema.referrer_id_s)) && md.referrerHash() != null) add(doc, CollectionSchema.referrer_id_s, ASCII.String(md.referrerHash()));
if (allAttr || contains(CollectionSchema.md5_s)) add(doc, CollectionSchema.md5_s, md.md5());
if (allAttr || contains(CollectionSchema.publisher_t)) add(doc, CollectionSchema.publisher_t, md.dc_publisher());
if (allAttr || contains(CollectionSchema.language_s)) add(doc, CollectionSchema.language_s, md.language());
if (allAttr || contains(CollectionSchema.size_i)) add(doc, CollectionSchema.size_i, md.size());
if (allAttr || contains(CollectionSchema.audiolinkscount_i)) add(doc, CollectionSchema.audiolinkscount_i, md.laudio());
if (allAttr || contains(CollectionSchema.videolinkscount_i)) add(doc, CollectionSchema.videolinkscount_i, md.lvideo());
if (allAttr || contains(CollectionSchema.applinkscount_i)) add(doc, CollectionSchema.applinkscount_i, md.lapp());
@ -404,7 +391,8 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
SolrVector doc = new SolrVector();
final DigestURL digestURL = document.dc_source();
boolean allAttr = this.isEmpty();
String url = addURIAttributes(doc, allAttr, digestURL, Response.docType(digestURL));
String url = addURIAttributes(doc, allAttr, digestURL);
if (allAttr || contains(CollectionSchema.content_type)) add(doc, CollectionSchema.content_type, new String[]{document.dc_format()});
Set<ProcessType> processTypes = new LinkedHashSet<ProcessType>();
String host = digestURL.getHost();
@ -476,7 +464,6 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
if (author == null || author.length() == 0) author = document.dc_publisher();
add(doc, CollectionSchema.author, author);
}
if (allAttr || contains(CollectionSchema.content_type)) add(doc, CollectionSchema.content_type, new String[]{document.dc_format()});
if (allAttr || contains(CollectionSchema.last_modified)) {
Date lastModified = responseHeader == null ? new Date() : responseHeader.lastModified();
if (lastModified == null) lastModified = new Date();
@ -557,6 +544,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
// bit 2: "follow" contained in html header meta
// bit 3: "noindex" contained in html header meta
// bit 4: "nofollow" contained in html header meta
// bit 5: "noarchive" contained in html header meta
// bit 8: "all" contained in http header X-Robots-Tag
// bit 9: "noindex" contained in http header X-Robots-Tag
// bit 10: "nofollow" contained in http header X-Robots-Tag
@ -576,6 +564,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
if (robots_meta.indexOf("follow",0) == 0 || robots_meta.indexOf(" follow",0) >= 0 || robots_meta.indexOf(",follow",0) >= 0 ) b += 4; // set bit 2
if (robots_meta.indexOf("noindex",0) >= 0) b += 8; // set bit 3
if (robots_meta.indexOf("nofollow",0) >= 0) b += 16; // set bit 4
if (robots_meta.indexOf("noarchive",0) >= 0) b += 32; // set bit 5
}
String x_robots_tag = responseHeader == null ? "" : responseHeader.getXRobotsTag();
if (!x_robots_tag.isEmpty()) {
@ -1494,10 +1483,10 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
try {
String doccountquery =
CollectionSchema.host_id_s.getSolrFieldName() + ":\"" + hostid + "\" AND " +
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":8 AND " + // bit 3
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":24 AND " + // bit 3 + 4
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":512 AND " + // bit 9
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":1536 AND " + // bit 9 + 10
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":8 AND " + // bit 3 (noindex)
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":24 AND " + // bit 3 + 4 (noindex + nofollow)
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":512 AND " + // bit 9 (noindex)
"-" + CollectionSchema.robots_i.getSolrFieldName() + ":1536 AND " + // bit 9 + 10 (noindex + nofollow)
"((-" + CollectionSchema.canonical_equal_sku_b.getSolrFieldName() + ":" + AbstractSolrConnector.CATCHALL_TERM + ") OR (" + CollectionSchema.canonical_equal_sku_b.getSolrFieldName() + ":true)) AND " +
CollectionSchema.httpstatus_i.getSolrFieldName() + ":200 AND " +
"-" + CollectionSchema.id.getSolrFieldName() + ":\"" + urlhash + "\" AND " +
@ -1856,7 +1845,10 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
assert allAttr || configuration.contains(CollectionSchema.failreason_s);
final SolrInputDocument doc = new SolrInputDocument();
String url = configuration.addURIAttributes(doc, allAttr, this.getDigestURL(), Response.docType(this.getDigestURL()));
String url = configuration.addURIAttributes(doc, allAttr, this.getDigestURL());
if (allAttr || configuration.contains(CollectionSchema.content_type)) configuration.add(doc, CollectionSchema.content_type, new String[]{Classification.url2mime(this.digestURL)});
if (allAttr || configuration.contains(CollectionSchema.load_date_dt)) configuration.add(doc, CollectionSchema.load_date_dt, getFailDate());
if (allAttr || configuration.contains(CollectionSchema.crawldepth_i)) configuration.add(doc, CollectionSchema.crawldepth_i, this.crawldepth);

20
source/net/yacy/search/schema/CollectionSchema.java
Unescape View File

@ -113,13 +113,19 @@ public enum CollectionSchema implements SchemaDeclaration {
// encoded as binary value into an integer:
// bit 0: "all" contained in html header meta
// bit 1: "index" contained in html header meta
// bit 2: "noindex" contained in html header meta
// bit 3: "nofollow" contained in html header meta
// bit 8: "noarchive" contained in http header properties
// bit 9: "nosnippet" contained in http header properties
// bit 10: "noindex" contained in http header properties
// bit 11: "nofollow" contained in http header properties
// bit 12: "unavailable_after" contained in http header properties
// bit 2: "follow" contained in html header meta
// bit 3: "noindex" contained in html header meta
// bit 4: "nofollow" contained in html header meta
// bit 5: "noarchive" contained in html header meta
// bit 8: "all" contained in http header X-Robots-Tag
// bit 9: "noindex" contained in http header X-Robots-Tag
// bit 10: "nofollow" contained in http header X-Robots-Tag
// bit 11: "noarchive" contained in http header X-Robots-Tag
// bit 12: "nosnippet" contained in http header X-Robots-Tag
// bit 13: "noodp" contained in http header X-Robots-Tag
// bit 14: "notranslate" contained in http header X-Robots-Tag
// bit 15: "noimageindex" contained in http header X-Robots-Tag
// bit 16: "unavailable_after" contained in http header X-Robots-Tag
robots_i(SolrType.num_integer, true, true, false, false, false, "content of <meta name=\"robots\" content=#content#> tag and the \"X-Robots-Tag\" HTTP property"),
metagenerator_t(SolrType.text_general, true, true, false, false, false, "content of <meta name=\"generator\" content=#content#> tag"),
inboundlinks_protocol_sxt(SolrType.string, true, true, true, false, false, "internal links, only the protocol"),

20
source/net/yacy/server/http/HTTPDProxyHandler.java
Unescape View File

@ -423,13 +423,8 @@ public final class HTTPDProxyHandler {
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the ip from the connecting peer
int port, pos;
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
host = host.substring(0, pos);
}
int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
// resolve yacy and yacyh domains
String yAddress = resolveYacyDomains(host);
@ -440,7 +435,7 @@ public final class HTTPDProxyHandler {
// remove yacy-subdomain-path, when accessing /env
if ((yAddress != null)
&& (remotePath.startsWith("/env"))
&& ((pos = yAddress.indexOf('/')) != -1)
&& (yAddress.indexOf('/') != -1)
) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
modifyProxyHeaders(requestHeader, httpVer);
@ -1050,11 +1045,8 @@ public final class HTTPDProxyHandler {
String orgHostName = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase();
int pos = orgHostName.indexOf(':');
if (pos != -1) {
orgHostPort = orgHostName.substring(pos+1);
orgHostName = orgHostName.substring(0,pos);
}
orgHostPort = Integer.toString(Domains.stripToPort(orgHostName));
orgHostName = Domains.stripToHostName(orgHostName);
String orgHostPath = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); if (orgHostArgs == null) orgHostArgs = "";
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
@ -1078,7 +1070,7 @@ public final class HTTPDProxyHandler {
if (addr != null) if (addr != null) testHostNames.add(testHostName);
}
pos = orgHostName.lastIndexOf('.');
int pos = orgHostName.lastIndexOf('.');
if (pos != -1) {
final Iterator<String> iter = topLevelDomains.iterator();
while (iter.hasNext()) {

10
source/net/yacy/server/http/HTTPDemon.java
Unescape View File

@ -144,14 +144,8 @@ public final class HTTPDemon {
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
final int port;
final int pos = host.indexOf(':');
if (pos != -1) {
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
} else {
port = 80;
}
final int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
String urlString;
try {

2
source/net/yacy/yacy.java
Unescape View File

@ -136,7 +136,7 @@ public final class yacy {
try {
"a".isEmpty(); // needs at least Java 1.6
} catch (final NoSuchMethodError e) {
System.err.println("STARTUP: Java Version too low. You need at least Java 1.6 to run YaCy");
System.err.println("STARTUP: Java Version too low. You need at least Java 1.7 to run YaCy");
System.exit(-1);
}

Write Preview
Loading…
Cancel
Save