From 71bcf02d3a7e4dd5894bd178c919dbb79df0e398 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 9 Jan 2008 23:05:52 +0000 Subject: [PATCH] - removed pro-version (is the same as standard version, use the standard instead) - changed yacy logo - removed crawlOrder protocol (unused) - removed file index in kelondroFlex (will not work, it takes too long to maintain) - fixed remoted crawl for clusters (now denies remote crawls from peers outside cluster) - 0.562 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4317 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- build.properties | 2 +- build.xml | 29 +- htroot/env/grafics/YaCyLogo_120ppi.png | Bin 7329 -> 7286 bytes htroot/env/grafics/YaCyLogo_60ppi.png | Bin 3893 -> 3786 bytes htroot/env/grafics/yacy.gif | Bin 1318 -> 1303 bytes htroot/yacy/crawlOrder.html | 13 - htroot/yacy/crawlOrder.java | 281 ------------------ .../de/anomic/kelondro/kelondroFlexTable.java | 57 ++-- .../plasma/crawler/plasmaCrawlQueues.java | 35 ++- source/de/anomic/yacy/yacyDHTAction.java | 6 +- 10 files changed, 45 insertions(+), 378 deletions(-) delete mode 100644 htroot/yacy/crawlOrder.html delete mode 100644 htroot/yacy/crawlOrder.java diff --git a/build.properties b/build.properties index 851ad22db..21a9f29b9 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5 # Release Configuration -releaseVersion=0.561 +releaseVersion=0.562 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz diff --git a/build.xml b/build.xml index 0c625bdbf..04faf3308 100644 --- a/build.xml +++ b/build.xml @@ -537,7 +537,7 @@ - + @@ -561,31 +561,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - @@ -767,7 +742,7 @@ This needs nsis-ant-1.2.jar in the yacyfolder. --> - + diff --git a/htroot/env/grafics/YaCyLogo_120ppi.png b/htroot/env/grafics/YaCyLogo_120ppi.png index 9dc8ae5e8d4880a3c5354ce704882ba9743e41f7..5672fcfb71c8b2a0cb8ca9bfe523c992b89718fa 100644 GIT binary patch literal 7286 zcmV-+9EszJP)KHC;&V000~DNklMkrGWOiEz!+#YM?ja*51@j2SW& z%9undGK9>Da7_(DC6Y06i_9rXrFriATgTzF_S$E^Yn{E%>G(bF`+Ob`kF(~_Uh93= zde^&}{qMJz`uh6ByHD5ph7NOFY~E#D^LE3u2c!0*`V5;rf0)3SBq7QJU$x z0=kr0gjr)>MWYIW>|(5AzfMV>n|UcJ|KH^}Ap<-ygMJ67HQ)(Hb}7+4JV*&bG1 zQ>)wF_#JsrS68=c!(N>;0EeB^lAO<9b=W7r9S;x*T-`S4eBk-$*n85)>@T|NmfwyC z%hv7D`H%~tpPgHANtsT%<@e&jxr;Y-KG-m(xNX!)&-_L__>xy(YU`-;k*4DE^`xfn z91ZiJs=B(Mu=s1iH*KJ#w7kB)L3G``w;a&<7_*@N@VOcs5k`^9gX-#2A_-jJEp-Sy^2MOhuFDzObkd*`L|#Sy5S~?L8t5;>PU<630bd5zm9#+PaW~=PbH9 z>#DQ#&FrVm^T~RX%~IQQm*|>-p80S?dk>T8{w6a6w1OEsTaR9)XElmilj-w)MURU( z5y^vuWZTBM^9Zv%Y?WNvFv;PrfYH5-OPoKYPB^#mh!G__vKA2x~)RjPz zR)gJ%5gL8+;leBF8i&*AM|df%N2~y9b?DeXNJW^x%ujls7_}d@VcVfH@R(Z-9ypBm z(D^X4;Qpf*M9tUu>;ryp?7W`f!L^rp4O)=Ed++d442LcPYIQu~y5rB2d89@Gzi;cdhv31>uVvcc!MuN-5Uiprou|Oz^Ho=M2b@kfd!Jmgy8VlNWv$-y8E~HB$ zZAl~#-hIfW_rB%G)}uVz&JAj}=zzAcAk_Hpjrtww+k(R!7tLL|O#!zh+YmE=AIo;c z;5-;}>Y+ABP*hoimtwvyLRa63O)r-(`GsP5aQw`5IO`T&CR+GhY#KFNwkzJyZmFg^ zMZc3Xk^idLOl|Ps;iqqSDQGd$6%U%LS~{6{u+lpS>dFp|O|uh+PpU2ZX=eFs_16L- z^YACx+TcON^BlYsqp@ps$b;tcBaBqFc`(`qvT)i=-qNHbP|T}dXEo9t#uI&sobP8B zX@dt_60-18TK1hOy3YF6V~r>Hm@W-NXydYpWufH5$#s?5Jb?W<&DWl3dLAI?+m4sc z1EhA~dA1KqD6Xo}1`lRkeAZY{;B7tGNO2W5wD*wt$KiFB?Tka3nVcUsj)-0cJ*G4_ zA5JHenf(XM4UW!({I5k&j;409fE#$VkXF3$)+;K89KDaHS{0W`O)~oALp+H=@9ENc zVBvcS&vVd`6m5_I$w16lu-J4-Ty0zSnPIj2mg)%-`x-6%qK%xqnuFRgLhyk|HF$t5 zP9QI21QQkVy77{wU&ftj++6Fj?@m2~r8fK0GrXV{T_#8*0s1J3%A>DV>VT_~;>w!# zSSe^e7gu4mrrx%50^!_x51pmFOGD?u?tN!~ybSG^62+fN`$X**H0oXUCMlCRpLxX6 zoBa;mYn%)N;;DS60?jBy<+mohmOSVxBqh5DFQv`Y?dnz4)V^iE*+yg6sJnC;4qXiA zylzt{D-Yn4`!-Xy6J+z>%wx*q7a)wO;S)^&6x(*oR&xegw`XL6O~6=J+&q| zC6!Nei+M%+k~7aZ_IKOayIT%$YuSG`9|lNg?mb-vdQLUy?ux^7c{tH=EFZTsrrFp8Csx+d zc~6WMi)&hQ`4;U6%Wc;M$9PY|qnzT`1?64$-BJ9r`~Kwp4?dtj8-KHl?v2;N%s%^5 z9)$-_p1lOJ(6@0SYBu+u=PW#!7L|sl`uMGanq)KA{?-PNN>8l@QfHO@j$nT*m71iw zw!TAfB3=;sc$mQ5@a7}GthzzI=zK(f_l(IXs;EKh6#fI*j#%_zP^oYz{1Sx+5m7gQ z^yrHuYPRFf806BPy+Zkcw2MOAZ0!*shZ!g$LaSL7J`E%42w8XsF7n zYpMNm!qL|Iod1y>W)7hQhZBWUhXuT>2VNMT$Noxv~Bd@Zyo z+@Y5X#DfzN&}4#C527%ntg_SYhwwGK*P`49oQ@%k%Kd6qp*!j}Y1 z_|zn?3RD;iFY*Le6Ev$Uf9B6%eBpvP)c4R`l}m~71Rr21hmTtd@xXI^Fp!w<=8HO) zTGvm-9AjZALsz0t$~GhkPc@8f0RjX~n?&UgJfQDs622s%sq`e5GQJQ6!K(>wj{t7d zXlXn8ScSjLYVOA6kIBI6EPTwpdgES*2M*4VZ)!XDy+nDRR9P~F#Bo2wkd*h-ByS4I zIeZ@5ZWSP&F=T(mzoC~Y7%1Sl)9SW_Q?Ap!R<9=lkXHsNYYHKDP7#(x|I1RuAK+QM;^0cLzO4JMclSPU_t3+S zxusRO{wv!RPxXqn>OP^Ws#=f-?>|C5%6b+KR)M$i=1pYKvyjZ?`tT!tjhX~azeHEE zn3{m#x|jl8_9n5os^h*SNSS9TtJi^KqVj08fZz8$_+x<|9|85J*NvCxH&6ZamEp7KE)i8AbO5Plv zCe2tN_J$Gu20L9L#9kKS#K8w2V5r#T-IH+VeW6ehd1Y0~pRI|5`ctuQxaoFQfAHv8 z7H|We54sJSjwS<$2SNKGM^HoFdFbIs5X}`*Si!NooX;>#A;cWsKIq4~C|dZ7=!ToG zsHyI2XfF#e{qKG#RQ@as?9ZOxry1%VE7IZM$sq35eZ_DK@?hZ#e;_^IM>&B(X(GDP zJD%>rCih(8&f(TUiNnHEsfbL^70Q#?8%BtDMt46s%G=8lfqsYX0Yw~p(pUu0pSdMf zEM-~xMYjN5d0l3DD~DT<2Sdj~w=fAeUxZH#o3f|UKH*8uF!Vlu4rjzoVf~-*bF^Yx zXjZ8*$8-XI$$G>dsPhsN=nKpD1Y>%e8#)qH2M&SV}tZgSK>Vn0FD7;k0vtyrI zuhi{*;(U)CAB24x2zRay!HJa&yHXU-A2y&K0E(FO^Ry5dUqE+;+<#Aw`j#uL2D`VJ zw7K1aeQoCj8;xBHfwtad>7 z>Yn>$x-ZE~t}h8Zy({G2M-_Ty%16P=QaPAg#c{R|dJMN#2(+|t$mGWR zSIq-B2vX}&o<#G`Gq|7NMv;~LP>SCBNC<0~(P{^T&`}Tq)b}OXo#itzAQEG@{bj;Gq|~VE452U8oJ%($33`WaYt6d!P!G7cA5$vP@>Fr{+nptVm9{^T)Sb6J+p=9tDgKpa8s5= zig*hRi?LA4^(A2mn8NpSvHE@-AoP2D1zZ^ec|y2xH$ND!Lg<^>t@YUl#AsE)gK#yY z%(1xn5}Wgkv1f?$J%lf%RV)dGB>4kr%XwUfPX82=kza}@S;mkgj(0|i@TO33)lbcC zTmfNE2>f%h*nFAFfR>gH)8_j;PJ1cD06`w88)c4VrGJ#e(Qe>F6r~J0!tqt_bwH}I z5XNo8!@D`g6{4QeGf>gXEo_=v|uLGY@t z+-dC!x}Tu$2hURx21N+lcOMhagQ9PxP?R}jEcnObRT;@h4rQvldq zQ5s#=!rp(CHFd;2!@h^;3r#)Ou@+eYR3jPvcel_xMCC(Mq18m`UpguB3Gg$tp5l4J zCrKBs+!oJ+q`ObmG8SN(D+PByVe~R3+6;e)cE?f`zX7(Z|II5UGK9DB6_Uffg$GaX z3d!dB;wpeNN1@;6mo*-FYyx-ai9HFu_E7xo7nyIx^I-3x^FY>yLl+axV{<>@7z-@d zN%?K=ENrz`);50VfHF@6T|n3fp8hF>xP`ZtLBr)c#Us2GHWm`FB7h5`->*V6?nzjJ z#sV@E;FmloJecPJ1^DK>pWG8ZI8h2=t?f*HLrp@o8DXrjsfiQ<<LP=e>1*Eg58y*b-;nZWfd0SeYC4f83yHlj-w*_(u~U-}JEvAO54=TJe{5RA%187o z&Ynb@Nt=L?a$mkfJP*)JLb=e$jhp&s802N86p!m1YmMb@D6-9nt^}<1t%;3xU4gLv z?{J(n761!VDU-092;?vpdN@iv57IN<0$J;| zA44>6#s&H@e8cx+f}oUlO0Zl>+sAXc0j&0^Q9q0>!21sww>Dg3f$<3YO6C{k?NXMR zOup5_yLk@8kSnRGQT`s@sa(eWoM7L>J1J@6c@T9O3aiwz-z=hejP55Qgumef+;r%5 z;D^yQElN=TPv;p6GcRZ*RN^yCSyAGNa9D`P*iI|FQ;B8^Gm;Z5QZ% zGI|x!JQhb&aY7giz`}_eU4Zu=aa?P~7Q#n_T*~-N6eK^=tQMRCk+G9oTBQIGY7W^- zw%9#(v(=}7k?PcEvRWh{c`)7G7s%aYhCk85)NH!aBtnp8jo;n30h>)wi`$oWr zj-OoQ3%IbN73B}^MsH{E^iPWLk1(X)%5C7?n{G8|J}^?pu!{+5^Wgc5*J>FHZknLE zO4)hm4N(?gwJQS_0p5Qfi^q=VTmC}72@7BY%HWaNEgq8Q;{){ zu`pl`(Y)OZj;7*dHrM?Gw%VV4E8l|r6)HiNym^RExpvq zk4k~eNTtIZ0QhmRjokae2EC>MgLUa@qIe$MOneArZP0Bp(L6@?6A^+R8Y#f?CITS* zXaRi>-94H1N#i*3m~?_=#|k#DH8}z)7aHHb&w+;=;z8))C?IRYVM|1}5L+pk)II*x z%5t;~$W4Z50Xhfl&VxrqwAcuZl!?t4^n39<@L01O$lcgwlk}#t+!R1EnCq*~@+JZ- z{AdBcAOKH@uiT6J?G_&ZMrz2Ig(67M$b%8%mjSuATM#O}sVrC0N_mI2j81HT@V!Es z45LmyK&@s$sqh9#5;u-Qo5@>&ky6+FgyeyF7iY*=@IEKKQ&=J*^DnmA9ZOwu?S&L{ z-%m!aYcjx2r;KuddnLwjIP}E^hDM0zLGI_Tkg;H9FWVK5SLKg=cUU4-;cz(qfaOj! z;h(Q@SZ=w}c76x}iLcPh-`Djxj6ehywW38oVHCof%b@2}V6dWZ+!N0ON~px{q_Myf z1wxzrC%Mo~6&te2n2XV&24<7gjL9hI^b|4=) zWQhVHPMml#12hP8??^oh=s;B>2q#`EHvdOh%4=#BuQ4eA8wxSKVkF6r<@^LrdOM(SAv9r1uFC`nA=-xj-Gl)ptK> zZXWt`epzpj!n!2G0~t0yz&R&V-iXwzC6?+OR}F5h29e7-pU) zbUw6dq1B-I+U5bG8c+dOw+)hb&~4y!Aa}D>C-AB;hNKoDZ;Lp_Lfbik+8#MQQ@c$& z4kz7v%7|-mccBbT?ajR-H8tdX$O7V9be;fIV9U;CzkLK;Ye z-L=Gk&iz~iLnBH`%0zUQ=lT#h^A@(_+s+M=?TFFfG+^p+n7aRAVl$?^yiyVmfM`f) zA|k3a=t#r1y@PKhJrce1iUhW?IiFDb(a7GB4(e-b z3anfoEQJJe9$0i0K5;g)m-)wOgxBhQ1Uq8j_Gi8MSXflTh~g5t{U))Sl;Q*i71Rlg ztw-m4&X?G(X7Kwve0TLG02~(3s{7v>8BuYP^)}m325D4Hk4-zoI4Bd+Y=u`4P=T!Ofp=Oe#dcy^{73VdK^K@ zDO{h3s5t4(mLh1CnYD1U&Kc1B`_-i!wy?QWX;x8L1*r%>VbE(@>rtLYj%&4q)+1LM zbe~MEhFNE)@(K;mU!v61)XrYCMYkld={-rU2#K)B&z%E8p!;&a6udU=*Hp_TN_~C9 z)_`z*Gw8ml_AtzGao*P=4K_#4gNll(&i$w8e2l4RXX}{zr9gv?mo9hjKZ6x*jl=Sf z@gWBz)ryhHB0v0a*GUqS1B0)IFgQ>IT4Kl*W6m zK7L_3A87!~*6qq QxBvhE07*qoM6N<$fkrG&YbIm#qa$#SKGrk0crjHrmo zw}fKy?Qr`NBK$sne|+|NY>)T*{rS9(=kxV^y~gnJa1s-d6#;=jVq|BM4{&b)t~enW z@cF@szXsfd6P*1|f%@uA^ZavUWx+$WrJ?Cgm*;g%qfyds%odoXz<^cY6a5Nzi%{~u8clqW{>3n?<4 z10jP!lY$HLbI(opaHX&`jMHUlcT@}%seu6hum(NdTb3*>UaJQaV)00>Ujzar+}ZYG zG?#IKkjYfjJ@TSRVl|`GoN+}-b_r%bET=?tmkH^xG96YBK@#1n$;S=b?ZEYYsi2>g z>+SPiG?1)2XXs9V7@e7p1ECwYLr!Cfph8^0+fg=*+Z)J2oq=*-G9@4^t<34NqC>TS z9R1E3QFyI;ZS4#^d~^2&e_`51_F$GkBi^j~gVhll(4V(|BZ>&ae8e8uw_FGjL>Sa# zcECO=z{)F?h@-dUXp#?^kR2-wc&`W*@#F;COawS3s>n|cLx0_7t_{9y*&@CYXhM-SPU*rub;C{J~ipsGT^Yuu8&Tp1ja6{XoWg z{CW4{9REOZ2OMLZmxUh$L2{XS!p2J1LImj{42afQhGGzWBgv-8D7UQ1I4huD!zFV= z!rgm^fSOA;RWduV(_Pxa>uj3oa_d#nJsOt$GD_l~>q2dMO-|@%prJLU6jKmMEI8=G zUcTV~S}%5N<&Yva zM@~~YOKMIYd)p8ugstNVy{oZf#Z?G-yR6gomot2iFos&b!za_InX0!> zHh}*50dikq@jE0~-WY9n(dR%2{WP7~+g$B-6^$`LVr;Jc&3d74r6{h}aUJ!yEX|&L zO=G6T5oh%&8!B0(&h|W2_>aWlfE}k9Hsu$qUEpZy)_eBO_Q|2aysU~Efp^5#a+^g> z^niHLAHhz;e!$T=y#nt%T^7y?L7-mUC=3nV^hPoabOF2S^U>4&qOGqRGJbSc$&k05 zgY$`9Nk>_p{O{NP0+MU=9GAFRZsSEs0AuEbvmH+YXN`Tgg+RkE3)+JXy#_6jUd3Hn z;;!r!0SW7n<|AaxhE@n`;oxrw^$}VFB!o>w$AP(vaER$)OUb<%!`I-UWAe~R=Di1B z!*aK_7o+qpq(V*!?cEP7ExryP@DSnEYF}c5vP$m>W)}>`h*;f+386SI zg3fy|8O$=q>#uKq(`ZtKLxqX=`gB6BpMwLhH^L$HKe^{8r&s2btj7kcif-zOHBuCM zDYAm1WA=8r6l{&ar4lM7R@lmj-mRfa2=aTDQ=+K*rVpzqU!hk4rpnUVVnXN+Fq`0) zidv5q>hFZmthHfC;*_C8?p;M&_4{Yl)Y`>y*s!{8)_02Q0gGg{CXE0G@J(Rrb1rgo zX|m6Y%n=#BRs{lg+TA0?{9Nd16W5~bNW3ia;+TZDoLSt%tz58)3bk(JH)c>3$GQiF zH5Sl79gvG6f z|2r8naaBs=ajL7~1agY2)u`6yx>MKwNvTYI&UN|<=dZ^{ge7hR<8E_$uONNF_qhJ~ zv8Kw5LyjW6o7*c4{`hTOwKKWQ3++G})R0_fdj;j#=#JMppCy@-{D=WbNzyN@@)K7= zmkj3ki=Cc?A6C(CChlrNWvG!vH;M{uLizk6*3uAbYj1t&=kOBJtCWjzIyCh|rU;Q2 z*D_-fwf{u|#Ov2vQ8#vr)&UQerRI|U!$;LCwrO9V_Adf<(RbQV<<1kJ&P5Wyr!}P< z(v^pHiKt-Tq@l;66%f0alwwF3Cej`Y;3zNUU^mnL&^vsk)T~(6 z+oZXKt?kvUhzh+M+#^*rVrNqSY+dKixArgY3-oGGk%$dVU?Y{nhesZbYf@zwN6YjV z?OlDRZjp|$bagrt_+1v0Bikd>h;ea64fDD7k!BE>beZ&Sf2I}BWx4pU%h$9u=4AHH zw*@8oB^$4MEKftN@FlS`j1x>_x#p#hs`PN*`CA1P&);m^J{~|Wimn<3&j|sxk#aEW z1F0q3aARa7>n&^Lr#g2eW=+^cVVJDbDNBqCq=8h-1z7pVa!0O!q7_-Xzr1kO?rZbi zp7$%veA-+&#j-&!Ko)s-Cz=9!@w&t2Sv^%dJlQWjfBXB7h90@t)AiOaae3lvN-wTZ z(*)j9=;5*qQYiXh8N&Y|sNok%GnLS=x76b+3gX732iS1Nq2Zqf_rbU+u0*hx4^ESI zSQ;XL+^3HBuGNlxRx0bBrBV4RbYH_$=e(Ay!2~q^Re>o~3289l7gA{Xjvkj`Y`J9j z``fj`>KZit<^;!qT3!bw3EMf+@l6uf z{jce^!RUa;H|KHt`%>>5v;o^DL>yPM$S z8WU?k8G@nOqF;3eT+)0z?Bb6JxpGCNw(}4`v$$xDfS^98&xO{nX*=zUTm8WYADNML z8pZ@rGlSx*)#9anbRo5 zKbU4xtHHtHX#~KnPpqgE|G*@}~Kckkvd7kHD4HP0u|~@@*!| z;)FjO4j(}6g~XqsgbGj`VP5m$0z9mBiP&X$6NW$OWqm`ei|Fxzzl`|w^?5gDdNLwR z{mqxFyu%SE9z6sR_9yq27V~wGi{6*R^e#(vnMY?LA}U^hwU?Anumoc7)s<7LsMs?K zCAaTtvpd7X*Z-R_;m$p&)I-gWe~K5r21XJ)rP4e5vhVDlpvo3~l}gPwM#HokY#FOc z!tKA2P9KPw1#Ton%j$P%*POpId9z}Y2M7G@mHCe(TIG%Qk&p|bAAhMxKVSb_NP)9S zU(+@T+-2S2FOF`Rf?yh`D!S_5c0wZY{R;kj4_KyxtsC5wCt5@(wtc^dGP6I#LJ7S% z%d&r+AidEM?_*6eYH&RuP0fC#+g~C-IMZ^fHOjb7#3Cd8B!Gq)Bg4Vt5Tp zF<%1_L6X<04EgzD*4qT>RkGkT;>^+qR!NSb!^;WH&(1T%FJ1bH#OYYsMxWpk=aU21 z0lk)OU`}1VxA{|jPg9Qc@r)R8B%6uuzPqsYEfP^#+Nd60KKXxB$@y{QDkNO)jW8&% zO8qe>Eu%UL`@YZN6CgR>T#TE(GW5NDCVqIdpmW~aUNI3Gj!{%``EhmNw3aqA6E)i* z`b0+M(`=aE$=}vl0_-{gcheBV3#aEalDyX;00|A(sfY7Ts3~ISY#28@FwK0Oy&-!Y zey!kZQXQ|8@*B>wh_5>qTB)^aW)*rt+#{W(!)r{(5r83Luv1ZpQOnId$8pb_%)!z)TE%4mubrJNC;>MapsDH{ znh9;->}>y#=&I0*eMGU@*$$^#of3)c0@&9i;NOkcrD_Is#+;8>_hux zr3O{!*kuqWq~hS>+`3HYm7_#!Uok8#m(1By$SXE~xv|F;?vm5_MZ{+N@2!z(9g50F zi|4ln6F#ZS+aDSNbv8%xBEOYn3th0PClf{1CA7U>%A3Ak{2|P{FsVTN4ybO?3=a%X z^i^$}=qv9Gl#J_tH_vZ7<}jht4YS33)gZeg-8@Uy>yCIprgV07?XNj~yAjhg=Ra#C zK>fRylXJu2puqFunVtSh#4AOcu=Nwaar^op32;O?6iLi>3D{T=<{FQ6y#asqcRgj4 zugaUwW6&bGZBDWJDi=qx8 z;szqHaLZaSPM_YITOr(0HWIEPWU3{G>4&M0b2cR} zk_Ej{mxvw4Xvb-WVT*Zo=f;j=z%be(HUF#SI$YbZz9hsX#KK4tOA}HL5S&EelUznL zOs~NJa+nOey|B^%VW{Jqjo>vN+CkhVdFHT##m~5C2NlA9yO6skp8I+_o(eQqVqk?Kb5C)j#I zs_Xlo(D~eIcXG-WK29p_^2rXB`7045kMT5Y*oM?QSqTxB5itZQh$w_4YM0;wj!N2I zE0p(&!65{wO=^<9-A=CAD>>s227tZiZuLMdTjLo~inQDD>08QABRUQbX3|@o6+VX2 z+&-X^TrG(pZ)KiOa@X$RQTZkS2M<0eIPj7#<(pDETqm-zf?w)` z4vjE7q1WXw*!@owi>ripn?D7!w#Q!dA-+!RhbQ{SW)CN4#PLleTPByv?X4GVbaAo09|#g!%9rNC8PY;drP2vFtoTmxdc zM>1z?bpNL1g>RG&6-#|u!4uv|_FJa(vHyI8u4k5eM2dQ@-;$*{Ef*MQ=3Z0_Y&M)g@@5INs+%wcwYw)Se70Y$BVXN@)C0TK)9UKaCJ09L6b>7~ha(ip zEtRvNrwm}Yq3Oa=F|CVNe_)XT(JJG z+g$>J!Zd!OT4lAOTo4y4a5}{8SOcLluJM!3Qq)4W>h|G38@*7y$g~a zu?Rjs;%3~W_h1`I_q|h0=suNhdQ%}Un%T5yL3F>_?%*v_TT}h92i(*Lohq=OO(eyN zzDh^gjQyq+@3vgWGy){GbpA@gnw4|R_O%|~wz_CV`S0&}M7pqAdI=LM-Z2mA`ULTD-6iYm1xCX(tkSV4+QPaXK6>V#mO1p zhxev`IM;FvmH2hw#}OU%-~=Nv=cV*}kM_M;4UJJmwk}>DHiEV6(jYT<(8Z^rrWTfb zvDUz;_g5~<8wwZ{+Pr4jaY7~UFGwvQC`O#qV;aRQZeH*2CXIWp(tKLM5TU@TB<%4i z(4(!_?Y(Xv6YX5U)cP5C>GZWSa?THG1<_Wdxj=$Mc5V0|O9o$MYyaKen#O5K*Sa%yP7Ktv;R2)*LETLG^a9}=KmWw7zJHJ>nepgG(xmYkIpSiE08&){I(%3Pdooiei@=!icW!eZ&(RjOa|51R8oo3>#j0lSN)->VV)qP)5@ZKb(j z^EWdB)*^(GdkgN%HQP@{bI_Y^7@jIzqMjd517&Ii3|yJ&q_!EQT6no<{V;7F9++-q zcUS7Uo*v-LUz=GIj)AEo9Xmg_4A)S6c32mBGop}{;AORhpwUz!L+VDtPBktd`%l@E zBByChN#DV=TI#J{aC9e_2x=-aPh6Dgp8R1YQK>%*3-z0rXpL4hV;4ca1AH<@RF}MOKjQrE`7;(H-rntmX6@lq_BCJK#NxBbv+N zqdnj)dMj}kw$5?=gQ%iMU9dp%e%sfL@JW}?x(M!GWGV|jGTj)g%dmNpUz{`Evpe*2 zcIV*09)_Z(iQ+$!J-;6pOl%unuz{ye8eBnzFZx`uk2*E6pGm1TA#8TAb^dOXUX|-% zfCx%aQ=|Y!82Z`rtUgpkwcH8$e{KUEeWMY_XAEkTx|!65n*DKGS4L6E+8|{0 z5ufV9n|3~suuRL$86R@v04eLUOmsnSt~5-68qAW7iRyDR`=4MM8-@Y3+1-}ZYpU)* zkk#fKUEj(BwzP;liwbewV4?HL8<~9mk}KkV(5z;vPVC(D z4)x)Va4#lxB~@VKvhJP~|P zKI414uDQ-%M;PFUQ&oGJ-A$XPX{BnRa@AYwr;e$OEa)CJSd7sO%UuJFE@ywVB6c=4 zfNhnyuhSoyy^u~*g#zmC*;!glOAPDqnJ*48is#S&mr1Fi1w+Rq1un^3RxQW|90LIj z#Z``91Xlo7U$V6=w;a$vJ`U@9MiWL91qSqhS&(Dk_TrfiBbYE;^!{bh6-Z@&kUV?0 zevk?eWk{tx$~ae-bquIo%q$6L@L;C7oM7AC1XaNZ`8_^~aP@PUGsO7^!@aB<$d0yN z@{Vb8Ym)B17;$m0(KLV~RUDZ;SPO`7V9M%f>m_KppB&LhDo0ofmaA23J2w)8{!Vyq zkzUBxt-zVR{PL`OG4V5u{=oOR&lj5-5YT8-4^Pa#GeMlOJ=wqGePOx@uirk4z^;Yx zR#cM^AsgGuNm^{YLUH``ZuNR~ z#q_)c_?#fVMBmAEXRFEp7=G>EJw87ErDtC~B-FeBH|(v4* z&@3yMg#i|~yTWJ;6UCADMi>Yqse^b5;RsIs|MGjtXEC3^d4e#;rVY^oCP(kNmg!9t d1R+B}9m~WIsVjL`fWH$#WCssYqg_bm{{W`I#)$v` diff --git a/htroot/env/grafics/YaCyLogo_60ppi.png b/htroot/env/grafics/YaCyLogo_60ppi.png index 9afddaa95cfe6b13fd099d0827205807fcc30c14..c4f6b5461e90cbde38112c306fe0c6747835caf1 100644 GIT binary patch literal 3786 zcmV;*4mI(KP)KHC;&V000g}NklIHby{?D_6tDp{Y@kwBnjpO?MHG?VR8*RQKmy-)awoaW%uQ|*;Cc4@&U5&IDQ9Nh znK|b@liTM%|Ee@JG+*$!KilpPLyHA{2hG)7^~~l?oV|YEv1_GeH9en3pPrlOwnRjw zn2lbd)#ACTZ*FtSGe9cs=*dj}H548{i|c1<-9rdi-RyRzv$M0?llga0sHpm4G(@{A zpjLx|BfC8x^(d5<*W7#b^8A&1XS@P6)Uy|Ze6Bxvm6+Yu)~?j%_C21x3jye`NzU!< zlHNxlzo6LOd3R4{2G{=LwP1-viklUS+l_}V#GC1kbR4+plF6>yn(LsgK6(?^P{T#V zrS~!lk3wS&1oL}kmu#2qYHAX(mb2b|z-x5`w&o|pgqISuRl)ff-BHVMQzy>f2K#O8 zlAs5Xi&pM29q9nBM=;iLxyymzr_nHOeM6HLD8$5P=nkIOi^p}z>bT-r9ja{aWK!i9 zQ_Bz#`qMV@qo6l_6>9l=*gMTp*ctgAx|r?`;<0SH`wqJwW`~~&{*E#`7TVw1jcS&d zlA{EL)Qo(*QNQ61$dF_38}4i-`Xxiv$%Ic-c|zHD`H8h$%8!Df%}&(vVoIszC@i>> z30;11QBJ+|r?@~BGHb!Pr+O1si3w4$MfEk~Kf%$r2}e~P|dPAj{fRXO8k+O6zL)X~`KI29uT z?u8R5Y}t912oO%k(RdH@>C5?oaZwf|_DFU_$kzskkRJu>zzo!~x~Y|_TJk{VUt#vA z5{hwUqYtbZ_orp`A}HMaJRdi5DYeYuLAE7xW?40|--oYp?2NVIwsr!A8S}T`Ee37# zMLplvHd3>M*rRl>Kx~3CQLn}Kvz;Ej#TEEJ+YC$Wd9I0|)768+|!9KFfnDR}y;=u|ztQ zp)^W0tGdh8K?aEmogclWjC}eozi!q+8B2rT;5zPFvQnpWnc?hVgn7~*3VzRvup1{VU6oi zBF|YZ&vRZB(i?s^M;6S1FHnaZ#8p9QayiKbCDWNZ1^T0fjxs2AxJS}die*`oNfw7F z7I0a-@XYIC59pM@P7 z&z5sJ0g^G-lDOD|09{4BsE4C@ff>U{G!a1tJ^i|vGQ!4=^7 z0$*lJ*t6E(Y6re9zuO^vC_H#Vq*?mYHXGfqv)Y+NT+Z1R9P#92GJQf^u~$9_OfXq7Z#AB$o!gt)|LZDg{Bh z8B4KHMze~{_wF)hO>iNuFLuo#gKlj4$%n$yHG6Rs2D*Byx|}@X5|WF06=5zQjA0O0 zGYB*%_$DJKs`=}C#78-mnstj{akb6j(D$FI1pIKE?5qtw_y}Lr$X{0}g`f+Cp%b0( z4ucOp7IN}7wSpWH6+tgDk36v@U1-)e$?ud^V1_a?%H@*cU7JhWxdJ&0v>dp_8()<3 z`h$EZ{HSlDxij+{!5A`c`|mkMv2bZ0tgleX#sPo7Z zlx|Qz3mrdqlMjWY)Leo)PmptGwU0%(00l(7DE7h?f98c66jC6~?ax!B;b9yWM@hB9 ze;dpsQsgJkU-6-EJ&<^TVt*sxzP|GaqAo&Z4@Z>jiITq&a9`h52?QE8RW0PCSd_$< z%FmiE`O8?rhf+Qib{!yI-v_KZjjWtYDk=UUQmRtURXGwT4}NhaY8$6 ztR{N$0T((O|T1nE%vYlxoNJXIgo$WfqP-%Z?ZutO@@ zk$yPB1>hU0?gd3#y+qzD^d4pEaTj0Il(`#KK_ExL#BveAolm^rA9YlA$r4n%TI6iQL7rv+s+fMSKP*wev2oS=BXuPS{Q-Lff)%RcN zo5==04OMVHMt|~JaKkd4v_@~zS{{}C?zrwe>gcb|=|~ih<3{!b1M>8~$A>~xTpGch zr>;liZG1-OIXuD!KaC(b_p<+7g5IlD*jUt;`&vy&k>YV&f9eL@LS%F*9|~Sq9}wKx z&I4HiKCd|nkXb<&^D~kCMl4aQeYO8$%5=|NG^=V}rvnnMpqM%qcDRM|%Flc#xVoRg zTeH7;Kh^jvBM^@$S#Hb!K|xQH{f6S3sEuFGCdzUd^HbqO)EjEo$@^t&AL{kp^sgYH ziadu$(9Z|IkPfLA)$G_at^AIO`%S$29Q$oN5XexVzQjeS%N~n3ukWe^l7>-C@H1Mp zPph?!p0+wAJrD9ct}b3le4Ai6e;58$f*QZ$84KPG?+%1W4GKCYU9EKl3y>iQd))7L zRH;vf{3kj6X`Aq|`wQk9SuEhHaNIJ7RUQHNBgA5rH0J(e`U?;U7RJkt8ao|Vqe~C# zj$TgnjZb=`1_c56X`|V((C$TY2!1+3;#hqut>Pb>MvBlODI3q1;v^S)CYFV+Ce)Gqsb0@|IyfU6hP{} z)w}QYGPJPOY68Cuv)h0CnyU9TB`CmOSw-!T@w7+jwi=k*#KdRx3Iwc(T3X>;goY(N zdH#xDL&Fo&v);9e+qAMTe{n0XsGV)Ut>+L7o9K+bqx}DpuHU9QIywMsr}YPgV;yx& zth8GEz~T1u9e4WP3i&B&+J!ZA?-oCXKyCnQ+T)#+I#6mc(Te-IS|A z7}qv-DdAQUVif9kWtS!4H~u*1`JD4P=Q-y*=lgw@WQwD`2v!aY0D#C*2ODRBbQLs1 zG)mA{(UJEA0>f}P7c3A*_8M%_=k77VB*NeFq$L0h4Zm}u>I=rQA-2?z(}8{=H$8)V z0ZX4iFJIEptDgS8&c2>LVb@>!ngW1m)lnNum(UR}9~l+0!^=;K4pP8%AUo|B&>_#{ zPJU|o6L)FSj|q4YK~^uWgh0`wRI)%i;8ark5P)@<4#RU#Q>GCaLF~%IisT5H=9ZR- zj0cGz0{@oeRP%$}%XAz%OiopN3KQs1)r&xp&90EyT5uT5^s*wbgrwx%de#^>vd((5 z6b5`#w7)Ei!jscfOV}U?v(uph+AHS8A7ydb*03O*v(VikJLol$2<`6d`U5H`68P`# zkeD*b=;2MW2CGXQm6f89 zp&L!+=9Nd_yIQ;hPC{A}qeCAe!Tm|;&%;;!3lm;QDIVogl&z)8wCG+-=78e@3+-B}x3IPY^l$bCYS^_|mB3Y@KLD05>4D^dQ z`B7MHcDwuZ{KKr6jB?Eqjc&&E{}BSp*sy%>(2@>*fGy0U3chZBh_So9k^FaKDc1mm z=oDe}m@!Uu^3nCjU61f5t#{Si4D|IiMPXF>5_{&^f zQ3n+x6^1IYf_Wgy>%)pBUw(ZK)slb6wZ-y#r*Thi>3S|I0SevFsaP&6)2u)5Sitx~p7c zo2ltQ?ZlM5X4FY~fcfY6|G?}km#`W*ofYpmzSNQhF}uo0wtI{Ry?Y zVWF_mI|h-c$CuB_st2&10eG_3I}LF%|IO__V?QBoU4lRiboU6*?6%s;==LuZsZQ9P|qECQzyTxRj@<;-Jg$ zoYq#I4vNe7_{K*%hiP{8*#_}bz*Gb!r`uJFj~x?IYUa)4I^FtyS6l z;Qh=aVgf&4b)u3<bx}t$`c?-@`Uw>=jcrZ|VJ8vncj_a(W#jsqygS|UY zVF>3oh*W$1cpnj(iloR>ooM*LQ)@!h=IF7TS*(+U;UnDOqRIZ*y5WLqSl8Zp&M62> zQ|*KUMap+5or93|yy-B|kc|>A<`cm1D=-C9t7G+8qa%2Z3e08P?xLorLcf|MH<#tN zQTEEa)@oe2YI`(WzExeyeik@8)Moz!0F#pG1}CgDpX7k$qry*Lb6gK5&F{A;bZEQS z{1Bn@+juYJ?u)oK(HxzAb3o|qcuKJMOj-k_K8Ba`CZ^3g3)v%40l;i@8JQ7&TyTAD zD}6DbFZogN&qes8uCe$7PY&-4sd)5R{4@VVm;K(UY}Wxr>Dpe%CD}3s(P*8;mdppB z;e?*pyETg?l*SM(yM*zN(guCI=fHnIuU-PXS((v;qW?L z0k5ULt9FPVW7g#+Y0Av0l;CKK39J>Vl^a5eN9J16Y-@ksK2ru=VtP4-#IY%kcMTZw~N)*1v(O`PRb8YL5ux90in6r~&G z(~NR0VFY!6CO$W*!#u z&Wj~BSxeJW5iJX+>-_}?^ zaPem+073XzVx^kB5{8?|-Yd`^+=3g)e-`IuZAq6ktUx21Jj1Jql1>f)k`0 z)*rO_@|gCacDTsQH_!yY^ISm&8bJP|o1YF?X(MCzEU3k`YonOp#-^k( z7SSp-XY^7D-bYWBdt(DLXIh46+D^}O9;>s6;Tec=39DjmzaaS9@?HK7o#o5Nsa9g$Z=rpE z`J>=_Y136E(M8qal18jH>SVVABe+-#X&Z`e>ponwqA+5~T)ok(r_~$^H+5XX)tjY^ zAKJ!YDK;tazty~@r~KKH@s~BlnwYqH|22t-O0o-oFH3>5Fh{+wi=mW!jjK#oiDjc( zUlx=Y)#xiJ?Ny&qG}-Z}*No_{pendai&*82{H*LujvkGXcihhaLI_xcj4(KnsnN&r zFA_7W{!ph>oR!_9u2iU*F*Wfis$+3uK(=bm&${=dn`%W$yh!sGIZ}5W4wT&k_>kZI zgMYYTIeG~n>J;^fdE9tbAz;jq+BoX26_wsrC<_Tz4S882%1TKut)uyEBixDc!3Yys zpF9gMtxbba(K!n^dVIUVgLS|JLah*hGRRxzteF~It!r@ zYL`9G*Iv%Sv7okeb^C zj5=A$A3p&SThG(8$O$<=pEQV{%0B}+d|oJO^x@Y2AhjN4PEbHNvHeFJWT zGT#3c>_Ok|TgMa(^%E2luQD-)H8NS@_88ufYJlpQePlq-N+G^7IWLh86G=e?u)gE8 z_e&l`zY`gt*_{m*64YmzMCzDMxIe5S4-5qDi<1@5aW?z25J=#?H09#;w-pt*yPK&gY!Y=bWXroSdzc&fbj1-i(x`guUK` zjHQH(oO`|AdxV^Gdz5Rn)@yT&W3AR>YlK>@)>^I3T5E(?T4Q@sT6;>R&Pq~qLY&S* zQfop=YdM_8IZA6WIa(={#wnD&DU7`-Ia(ovts#4@A#4jX;^kWYZfku)0puyM>EKtT$GUC75_M7|3$a_ks76i1Am_(*Wb z_kjkFSv-cRu<)V94Mu~43F)ydpMO%OFj|NmORNDt0WBuvLt%^1n z;Q$F>goY5h!2$^_YG4&4P)H3D`5?3~i5donz-Lw9#&G)>5l7|sKP&|X|`z6LO; zg%1kk1uz<60{pd?3h^;g9RL+9!h?EiAr(MXFX1x*g?qKo*MchCBbSIR_(NSKi{Q}0 zX@(&d01Cqa#@vdWc~)2p!+$l9V?{zlP)G^`k+uMhiM2o;2Lh51;{Qpz;R7If7LYfA z3_%DngCXX~FkAo-pdeCe&XHgiT3cGM<%R-sAOUl-3Bf`k2P}eJfqUgfVS?frDG^T` zLU0-hR|z2sArCC#Lx6kYlL3tdkZ=!xYbJW2L}*QiK_R>dF`IiYeHs6Sp;Fe%b(7>&@Hn5?Q$3@xzkA|=?P%#J*Y1jwdAm9yKF~|@@ zv&{|z0|^#fU;$zm#3aNESPJnG2@5QcfVls)bqo(bq^oYa+I*2oAnwz(EEQbI?L`Wfc^{h&BzlxP$_NiQL@l+{(+R;5B@dX>iwg%F(keBK+_cF79;Rca{GNyls_uo8535w#mHO9#Gmf~+GOt1tnHaVZ+5Tb8B6KVwfmdGh4>=}De# zB}D~z)kYu$%Rm~5L_(2BFdPns!hul89}0P&guKBg?$-qEfuP+Vr2PSt&#(6d^egW5 zYrTGr+oN&1HFme!<5PRQYPV1A^r-FbQrfO`dzDtFl3u~C)LWH$n$$T-t)0}GaHkiy zdvL28r=7UoifiroN~zIHb+lByf=*hh#q1u;>c(g{X0l^?E2g6{EsbgPm|Bl1^(d)D zaSe*A(=?_uwLVR$O&wfRjF`Nmr=y^=S=QS@UGmeVCO2R?*ITb# z?LF6TNxXSo7T7e>L{D z#X8(07`sv44&{uaV;^v?nHfEpHNuQGMb zaR~8V3S}^()8WB>IWfRXdu#Hy`l`baH&8M+eF`gWw#e4lqAZou12C7LTU!X;J<$OF zu}uZ0?Kqw~9A~4J zsD`h-DFlp&uT(Jl5}Mb_M56H7(+S6mcxV%E&rW92{S#BwFzo@X1)@J#@$yjP5YSaI zw@0L*%G{=wCyEDCb=^_@)O(1nXeTp9QpDHrC3_6^TGvRwGs4NvqHxH@b08&uF@m-ZZVuzaR3>Qz5Aa{J}P zttJnhtRg$%V zX@U`>82lV@9c~_rUK^K9z-t7s!V5S@$-Ktd^5J1~y_k9Tsq_JLeIotS9dX5(zZ*6? zuFe%_X8&8lrifb0(9ioc85;&Jn6Aem+rxq9=ld=r&B+!S7z=AzOdvfr_7>j SBM;9!vlxFp63!q6VDo acceptDepth) orderDepth = acceptDepth; - - // check if requester is authorized - if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { - // this request has a wrong target - response = "denied"; - reason = "authentify-problem"; - delay = "3600"; // may request one hour later again - } else if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(iam))) { - // check network environment, if we are a robinson peer or in a robinson cluster - // then the request must come from a peer that is in the same cluster as we are - reason = "not in my cluster"; - response = "denied"; - delay = "9999"; - } else if (orderDepth > 0) { - response = "denied"; - reason = "order depth must be 0"; - delay = "3600"; // may request one hour later again - } else if (!(granted)) { - response = "denied"; - reason = "not granted to remote crawl"; - delay = "3600"; // may request one hour later again - } else try { - yacySeed requester = yacyCore.seedDB.getConnected(iam); - int queuesize = switchboard.crawlQueues.coreCrawlJobSize() + switchboard.crawlQueues.limitCrawlJobSize() + switchboard.crawlQueues.remoteTriggeredCrawlJobSize() + switchboard.queueSize(); - if (requester == null) { - response = "denied"; - reason = "unknown-client"; - delay = "240"; - } else if (!((requester.isSenior()) || (requester.isPrincipal()))) { - response = "denied"; - reason = "not-qualified"; - delay = "240"; - } else if (queuesize > 100) { - response = "rejected"; - reason = "busy"; - delay = Integer.toString(30 + queuesize * acceptDelay); - } else if (!(process.equals("crawl"))) { - response = "denied"; - reason = "unknown-order"; - delay = "9999"; - } else { - // read the urls/referrer-vector - ArrayList urlv = new ArrayList(); - ArrayList refv = new ArrayList(); - String refencoded = post.get("referrer", null); - String urlencoded = post.get("url", null); - if (urlencoded != null) { - // old method: only one url - urlv.add(crypt.simpleDecode(urlencoded, key)); // the url string to crawl - } else { - // new method: read a vector of urls - while ((urlencoded = post.get("url" + urlv.size(), null)) != null) { - urlv.add(crypt.simpleDecode(urlencoded, key)); - } - } - if (refencoded != null) { - // old method: only one url - env.getLog().logFinest("crawlOrder: refencoded=" + refencoded + " key=" + key); - refv.add(crypt.simpleDecode(refencoded, key)); // the referrer url - } else { - // new method: read a vector of urls - while ((refencoded = post.get("ref" + refv.size(), null)) != null) { - env.getLog().logFinest("crawlOrder: refencoded=" + refencoded + " key=" + key); - refv.add(crypt.simpleDecode(refencoded, key)); - } - } - - // stack the urls - Object[] stackresult; - int count = Math.min(urlv.size(), refv.size()); - if (count == 1) { - // old method: only one url - - // normalizing URL - yacyURL url = new yacyURL((String) urlv.get(0), null); - String newURL = url.toNormalform(true, true); - if (!newURL.equals(urlv.get(0))) { - env.getLog().logWarning("crawlOrder: Received not normalized URL " + urlv.get(0)); - } - yacyURL refURL = (refv.get(0) == null) ? null : new yacyURL((String) refv.get(0), null); - if ((refURL != null) && (!refURL.equals(refv.get(0)))) { - env.getLog().logWarning("crawlOrder: Received not normalized Referer URL " + refv.get(0) + " of URL " + urlv.get(0)); - } - - if (!switchboard.acceptURL(url)) { - env.getLog().logWarning("crawlOrder: Received URL outside of our domain: " + newURL); - return null; - } - - // adding URL to noticeURL Queue - env.getLog().logFinest("crawlOrder: a: url='" + newURL + "'"); - - stackresult = stack(switchboard, url, refURL, iam, youare); - response = (String) stackresult[0]; - reason = (String) stackresult[1]; - lurl = (String) stackresult[2]; - delay = (response.equals("stacked")) ? Integer.toString(5 + acceptDelay) : "1"; // this value needs to be calculated individually - } else { - // new method: several urls - int stackCount = 0; - //int doubleCount = 0; - //int rejectedCount = 0; - for (int i = 0; i < count; i++) { - env.getLog().logFinest("crawlOrder: b: url='" + (String) urlv.get(i) + "'"); - try { - stackresult = stack(switchboard, new yacyURL((String) urlv.get(i), null), ((refv.get(i) == null) || (((String) refv.get(i)).length() == 0)) ? null : new yacyURL((String) refv.get(i), null), iam, youare); - response = (String) stackresult[0]; - prop.put("list_" + i + "_job", (String) stackresult[0] + "," + (String) stackresult[1]); - prop.put("list_" + i + "_lurl", (String) stackresult[2]); - prop.put("list_" + i + "_count", i); - } catch (MalformedURLException e) {} - } - prop.put("list", count); - response = "enqueued"; - reason = "ok"; - lurl = ""; - delay = Integer.toString(stackCount * acceptDelay + 1); - } - } - } catch (Exception e) { - // mist - reason = "ERROR: " + e.getMessage(); - env.getLog().logSevere("crawlOrder: " + reason, e); - delay = "600"; - } - - prop.put("response", response); - prop.put("reason", reason); - prop.put("delay", delay); - prop.put("depth", acceptDepth); - prop.put("lurl", lurl); - prop.put("forward", ""); - prop.put("key", key); - - // return rewrite properties - return prop; - } - - private static Object[] stack(plasmaSwitchboard switchboard, yacyURL url, yacyURL referrer, String iam, String youare) { - String response, reason, lurl; - // stack url - switchboard.getLog().logFinest("crawlOrder: stack: url='" + url + "'"); - String reasonString = null; - reasonString = switchboard.crawlStacker.stackCrawl(url, referrer, iam, "REMOTE-CRAWLING", new Date(), 0, switchboard.defaultRemoteProfile); - - if (reasonString == null) { - // liftoff! - response = "stacked"; - reason = "ok"; - lurl = ""; - } else if (reasonString.startsWith("double")) { - // case where we have already the url loaded; - reason = reasonString; - // send lurl-Entry as response - indexURLEntry entry; - entry = switchboard.wordIndex.loadedURL.load(url.hash(), null, 0); - if (entry == null) { - response = "rejected"; - lurl = ""; - } else { - response = "double"; - switchboard.wordIndex.loadedURL.notifyGCrawl(entry.hash(), iam, youare); - lurl = crypt.simpleEncode(entry.toString()); - } - } else { - response = "rejected"; - reason = reasonString; - lurl = ""; - } - return new Object[]{response, reason, lurl}; - } - -} diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index c8d6adf64..5070a263b 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -71,48 +71,27 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println("*** Last Startup time: " + stt + " milliseconds"); long start = System.currentTimeMillis(); - - if (serverMemory.request(neededRAM, true)) { - // we can use a RAM index + if (!serverMemory.request(neededRAM, true)) { + System.out.println("WARNING: NOT ENOUGH MEMORY FOR RAM INDEX " + new File(path, tablename).toString()); + } - if (indexfile.exists()) { - // delete existing index file - System.out.println("*** Delete File index " + indexfile); - indexfile.delete(); - } + if (indexfile.exists()) { + // delete existing index file + System.out.println("*** Delete File index " + indexfile); + indexfile.delete(); + } - // fill the index - System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath); - index = initializeRamIndex(minimumSpace); - - System.out.println(" -done-"); - System.out.println(index.size() - + " index entries initialized and sorted from " - + super.col[0].size() + " keys."); - RAMIndex = true; - tableTracker.put(this.filename(), this); - } else { - // too less ram for a ram index - kelondroIndex ki; - if (indexfile.exists()) { - // use existing index file - System.out.println("*** Using File index " + indexfile); - ki = new kelondroCache(kelondroTree.open(indexfile, true, preloadTime, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80), true, false); - RAMIndex = false; - } else { - // generate new index file - System.out.println("*** Generating File index for " + size() + " entries from " + indexfile); - System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " Bytes) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB more RAM to enable a RAM index."); - ki = initializeTreeIndex(indexfile, preloadTime, rowdef.objectOrder); - - System.out.println(" -done-"); - System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys."); - RAMIndex = false; - } - index = new kelondroBytesIntMap(ki); - assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + // fill the index + System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath); + index = initializeRamIndex(minimumSpace); - } + System.out.println(" -done-"); + System.out.println(index.size() + + " index entries initialized and sorted from " + + super.col[0].size() + " keys."); + RAMIndex = true; + tableTracker.put(this.filename(), this); + // assign index to wrapper description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";"; super.col[0].setDescription(description.getBytes()); diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java index b8ed2c86b..f5f374574 100644 --- a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java +++ b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java @@ -55,9 +55,9 @@ public class plasmaCrawlQueues { private plasmaSwitchboard sb; private serverLog log; - private HashMap workers; // mapping from url hash to Worker thread object + private HashMap workers; // mapping from url hash to Worker thread object private plasmaProtocolLoader loader; - private ArrayList remoteCrawlProviderHashes; + private ArrayList remoteCrawlProviderHashes; public plasmaCrawlNURL noticeURL; public plasmaCrawlZURL errorURL, delegatedURL; @@ -65,9 +65,9 @@ public class plasmaCrawlQueues { public plasmaCrawlQueues(plasmaSwitchboard sb, File plasmaPath) { this.sb = sb; this.log = new serverLog("CRAWLER"); - this.workers = new HashMap(); + this.workers = new HashMap(); this.loader = new plasmaProtocolLoader(sb, log); - this.remoteCrawlProviderHashes = new ArrayList(); + this.remoteCrawlProviderHashes = new ArrayList(); // start crawling management log.logConfig("Starting Crawling Management"); @@ -85,7 +85,7 @@ public class plasmaCrawlQueues { if (noticeURL.existsInStack(hash)) return "crawler"; if (delegatedURL.exists(hash)) return "delegated"; if (errorURL.exists(hash)) return "errors"; - if (workers.containsKey(new Integer(hash.hashCode()))) return "workers"; + if (workers.containsKey(hash)) return "workers"; return null; } @@ -97,9 +97,9 @@ public class plasmaCrawlQueues { public yacyURL getURL(String urlhash) { if (urlhash.equals(yacyURL.dummyHash)) return null; - plasmaCrawlEntry ne = (plasmaCrawlEntry) workers.get(new Integer(urlhash.hashCode())); - if (ne != null) return ne.url(); - ne = noticeURL.get(urlhash); + crawlWorker w = workers.get(urlhash); + if (w != null) return w.entry.url(); + plasmaCrawlEntry ne = noticeURL.get(urlhash); if (ne != null) return ne.url(); plasmaCrawlZURL.Entry ee = delegatedURL.getEntry(urlhash); if (ee != null) return ee.url(); @@ -110,7 +110,7 @@ public class plasmaCrawlQueues { public void close() { // wait for all workers to finish - Iterator i = workers.values().iterator(); + Iterator i = workers.values().iterator(); while (i.hasNext()) ((Thread) i.next()).interrupt(); // TODO: wait some more time until all threads are finished noticeURL.close(); @@ -122,9 +122,9 @@ public class plasmaCrawlQueues { synchronized (workers) { plasmaCrawlEntry[] w = new plasmaCrawlEntry[workers.size()]; int i = 0; - Iterator j = workers.values().iterator(); + Iterator j = workers.values().iterator(); while (j.hasNext()) { - w[i++] = ((crawlWorker) j.next()).entry; + w[i++] = j.next().entry; } return w; } @@ -260,9 +260,9 @@ public class plasmaCrawlQueues { (remoteTriggeredCrawlJobSize() == 0) && (sb.queueSize() < 10)) { if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { - Iterator e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs(); + Iterator e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs(); while (e.hasNext()) { - seed = (yacySeed) e.next(); + seed = e.next(); if (seed != null) { remoteCrawlProviderHashes.add(seed.hash); @@ -277,7 +277,14 @@ public class plasmaCrawlQueues { String hash = null; while ((seed == null) && (remoteCrawlProviderHashes.size() > 0)) { hash = (String) remoteCrawlProviderHashes.remove(remoteCrawlProviderHashes.size() - 1); + if (hash == null) continue; seed = yacyCore.seedDB.get(hash); + if (seed == null) continue; + // check if the peer is inside our cluster + if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(seed))) { + seed = null; + continue; + } } if (seed == null) return false; @@ -416,7 +423,7 @@ public class plasmaCrawlQueues { synchronized (this.workers) { crawlWorker w = new crawlWorker(entry); synchronized (workers) { - workers.put(new Integer(entry.hashCode()), w); + workers.put(entry.url().hash(), w); } } diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index d5f1274f6..6703b1984 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -124,11 +124,11 @@ public class yacyDHTAction implements yacyPeerAction { } } - public Iterator getProvidesRemoteCrawlURLs() { + public Iterator getProvidesRemoteCrawlURLs() { return new providesRemoteCrawlURLsEnum(); } - class providesRemoteCrawlURLsEnum implements Iterator { + class providesRemoteCrawlURLsEnum implements Iterator { Iterator se; yacySeed nextSeed; @@ -159,7 +159,7 @@ public class yacyDHTAction implements yacyPeerAction { return null; } - public Object next() { + public yacySeed next() { yacySeed next = nextSeed; nextSeed = nextInternal(); return next;