From f42c8cf69c65d8081ba922531a01631d78daf9e4 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 9 May 2008 00:01:47 +0000 Subject: [PATCH] updated terminal and dynamic webstructure applet: can now change when crawl is running git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4780 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../domaingraph/applet/domaingraph.jar | Bin 187496 -> 188727 bytes .../domaingraph/applet/domaingraph.java | 285 ++++++++++++------ .../domaingraph/applet/domaingraph.pde | 285 ++++++++++++------ htroot/processing/domaingraph/domaingraph.pde | 275 +++++++++++------ htroot/xml/webstructure.java | 4 +- .../de/anomic/crawler/NoticeURLImporter.java | 2 +- .../anomic/plasma/plasmaSwitchboardQueue.java | 2 +- .../de/anomic/plasma/plasmaWebStructure.java | 145 ++++++--- 8 files changed, 659 insertions(+), 339 deletions(-) diff --git a/htroot/processing/domaingraph/applet/domaingraph.jar b/htroot/processing/domaingraph/applet/domaingraph.jar index 9cb521ca7f24e6f654e887248d7691550726a22b..db614ad8f2d160f1f264111023e29d26907657bf 100755 GIT binary patch delta 6381 zcmYkAbyQUC7RF}=$q^(4h8_?kl|eeBTR@lr0g(`p&XJa(grOw|P>_%m6r{TmP=@Z1 zE-3+#=IVTY_ug~NO(lRQtnEULV#1RQ&14$fR(=T8V)9yzTlLEJkB*-eyad?c`aAsKgBw{ zqc~BAmko1g7(`V5kk1i>t7{by`8S0P^I{l8R?ZE(z=e{0fQ;oz!k5?ovNYMX_Vq}6MlCvr6VB1@=2|l|0?B;52Jzq&Gpb5`)5jRea!z<%17@1 z>y#P((<$dN?!c|8eCLVWKSeNe+kZBiVU>t0z)Y)vpb6xr1mz~rh~sg4u{|Lp`~#If zz$og!^FlEhqd=l+AsS$=<{zng+8kJQ>knam6Hj$h7_j~UF5ou`RKT5ib=wrsniZ$< zo&g^V<0#@TNQ)LnzvqQe;^^2A#6%3f%SlJ`HHKi#sS+0~e%g2Hi%* zX=t~}Qorm6=-y7k1RX0|0gqcsf*e#eIgV9Xx58k7Y`L#&1s9&xvO;Jt0jR_uWDJ5` zoark4p4i6plXIUdfJ-tpHPNk(02)`(--szCHaf+6?R zRw+=cG-36@y-%Gv9AFF_b$)zwLnSDd%5C0H29+Wk3CZ>+5nx0r*;+s)VwRoVJ>SAGF?Cm#-LD_U3~flKO8> z2sAIa1m<9Fy&aPnv+-6EMMmF#55?MbmogOP&2l~jLfhbHDM!FJet0?>ghW#Fde1C? z000RH0Qmp!3Rs8Wtda-E;h4funsF#om`&_i%GI!Q%j85Us;bz=N*~SsAy1mFMO*Vcw zFhV_$EuwC%&iz&>$HvvFrwz&}pr5zRWwEJGN1~iPXNGJN9Via89E?g;iqwegeWu^1 zOVT7*MW>g9aPR9m=S^qJV@{&Mf2rh7HT;sc=ZUf2MG^WNtvN&oLh4)T;^Y9WZexf! zeE}?SLvR!cYgv9momH6KW85Ey{fyoINpKN8S8$*DYl*}4m)Sp!yk@R%!8hewZ-?-1 zvziFrEDbcsE|IS%vScuzAviY*!;|Arvlo~Tc(60pmXmslY_+x+rX!kOCI!bCW~o-p zOcM!qY!BDGrWUD0O| zbPchFW4uRHTBtJ^V=Rm!3=gj`*Q3+?ZZi?dZ$qqSue^6P^dzJ@V!x*CLMZ z1U*hJj;AzCQO5stH@ycUn3+9veZLzXr+fFoj%p#eUDusdn#Yi!(hv@n%!CVrmib4E zgs}`?;)5>p(E8Tvh#r{0YXzE5eq;fsD3tk>a#C-iuv_)>csj?3Qe%f1 zskcQ;aHX}Pk>DXks#42u9|WCq4iXD^$WSgS6Eqx^w4LGz}xTIEER9P z<(Bd*{7lI&!M<$obj>T@m;Du#-~OGgh-;v?yM}qHgb{T5`$8UZaIz>Nh=#4%tjQ)< z+nzGdVFU$05Xj3o%g)>72dFF91h`PbH;@n>(|{b(YAkGThSz_(2MwlTlymG4jZR~# zc$2-lK++Bl*rZI-AL20#6v~e%aIduLOIzSE1S_Rt26EfZt#s}ncS+w@D8Kud{Ur&n zI`VW>c~_U;i2+T--g4bUyiVPIgGancE4OJx8+EOY00xe>xKc0C&;8!rK2m?MBx1Ci zNs4ULMZZUEI#bpV(}m@9Srj4Y$g$2@#+_*lZ0e2^^&rKk6S3i929fvJ80~px&E%{2k1F*{rLPfl}fU$rsHshmI?Wa ztZ*a@ri8|%N(n_45v+W0xW4>hz>eHF?Zx0~sR#*`p3)~(VQuQ6eOccwwBnaauk243XV$_={%1R+ra@x@}|eO5!c-23Rp8>pe_ zi7k7wuq-?^J}du@26WcJBu}HlF4VH3p7-YT<#Oh)4e6xNVIqoC1u#8|eTeV9b_xnD z-Az-;>3$Fdh$%H0vK2e#eF1&r*Q@Aps;R?#BwBP>s~BFP_iK6`Ovseq%&w~m<7Jk7 zfj^SvM6AL-dTH~5_SCRoNc6De4eEky0<I3-lQkaWjT$Ge1$a{5Fd({@$(h@>7=6 zCskJg;9Jf1Gz&UZHH?FkFVi`ElKivZ;)gY>2Aat`hO#MgX>k&E6b;;pxyw_Rs3wCN z>AvBim8{+Zyv)6BoU+uZSat0x9%qahbwti!$^o6--LTTro3*qAP*Oz z2uGl8qSJaWfE+tt^0|e4G?p2a_1R!V#b@#qLeFU5OND6D{s^^zFcgy*RI?B1j!k&Jd8YLZ69EEH#%G-W7-O#|Ac(V$yAErPmj;oXO8J~H1l_GfB zD z)4V9J)A}Hl@GR^0dQ?XAGs$BIZom5AM~Frdl_C}$cx}|P9$j$&i?;DV+Qt| zu!>A#BPqJhTa6r7P;Vq?Xs$RW;q2zCR3||aB~3ogHLP~YZ(s6^|DZH1+?~%AMTk(qj!44t_F_U7`rF}pOwHRHScQEX#EDiD z_$8b3rjB^muL;z#C|OfjMl#EYK-I(-JX7n&h|JQYYGU4B5gCn?Le=c6(7G)h5`lXi zXb4Zw`j(+JxmN&+_ZoBJGhNMwu;~3L7Qs2RoTilZlYOys+rp|MXw+7AD`yWKptR*% zLW2g>{r$*e<*2z5O&PyqqN;m723>Pv=n?H7GG1;$aWIr~#avgv*s-C%(UJP$T>qKn zc>}d0K_Q;*GQH^?9{ga%Jh=DuiqV}534$DtvK1?TrG`UX}Lgvu`Y9Zc!%$jE#wg}>*y&uE4EtEl zR}mj6Y*Z7fSVhg^5bvpUFL@3NCBxElyTf)1q)shFdXwWkFOW;rd+&Ah;GI|M%hRP{ zT}gGb8Ovo@A!M(8u3xn;?TRGV=DpEbA={_gO=|h_)moiSYntg;kM|Nu^(7L0&v$c* z@KJ?W?|NG~UjDZAb9^k=WB6qWJa`*|Sr|7RlUIL!q_ONK$zaUAyd%Dsr*AYC0Q z9yNT&X1Z@^>ZAZjyqDX1DSPvk(^PqFV5pKjZlrKu=1=&w%_b4VJ^EE>8bK@V2J+=? zv$)L$m@p#$z43<8^^c<)p%aJEWgD*r8w~S2A3m_-sc@UtTDpo2`3?*q9^^YuZYd}i zlwoK{4yuNwsjo~t_nqFpAz#ID_*&!>myLljYE|2 z;Andr_Gxm91X4S5xNy`_V0X~gbn75>#rkT-&5+kdv_a$7IJ7J9 zNz&AwE%E@Z<=h>0!0*Q)<^lm6wxQ_l%*c5DZ>&b*qLlue{1Ib% z6JNipf64Iuo$MwjPpvdky(@jY=cg1J5*J18x-pSS9ih~dX12|(s|2()Gbi}T1TOu=m%>fsl#2;lM=Co9?y0cg^*Ycn*utS2Q6^V3S@=9?6_*hpv}?- z?0Cb@YXD-i?#B94y)I>}EqHv0-{$oOj}LlmFd@aADtT5nv>86C&Ql(_YclmLYP{R(D9XxD5KeJ^!0c?F zWtd%oe2wiqInu1h=ALj5_4RmK=2;rCUR;du569Vdj+6DaV8q9n-s>-EOxBKdZHTn* zdhb+GX?J7FzD&%+f1qy2_kB!O6$h1BN@&M!d;)F+t^0KQOH*Qh?tS>FO7}euI>yfL z(6Yl|%V@+rg{%uF$?7Djqv@U|iv4!&aF$eaPT`@Ahp=Oyb@k$ih@H}~U^DHuQeCXf zyX_hZ?ej;BUSJuX!5zEX`;XK6d8a>1-`rXSXByjs`TMl0W1gdayp%ZW;@#-AE=mqZ zDz!!t7*JMne&5n&_>`n8#7|;v#WQSino~R9k#T(eBE`9f0@AbL($6ADYFSe0RwsNm z>BuFrDm6#&!H@k{oAiRLZGgJ74_)d$#qGt19O%!PA)n{{W7!*|$yTYA5)!7yj(&Y{ zA1ghys;gTUx9I{aBDRrOwXF)BCz2wXoIXFEqvP!u*vr~c%<1Ffd59T}d6Q_NU)fw) zEfP7`C&BnK_`7}(8y|wQEknz&wfXe3z@}%fmY!tVNd%m`kU!i}Qv9H>w}dA<05sd9 zQ9LjR#EhGTw(I<$cq1i$XZZUe^AusqRpMNW;~B-8rj=+8pIY30$MzzrKohW8;@j4k z_fucNgN1ZqRI2*xOsHW~C%akAl0aO3H~%-!?+xN(&aP8$_P(5z9$dmaJ^W}oKQ|Ab zj^&GkzHcr{n#}0t94N@o>GW`VYWo$MDIzRSd3dK)6 zV!ay0h1>YQ5O7&HD~&~!O*#*HuNi^!ji-w1t=70XQC@^JEA>Q4yE*%30J=+ZdzFQ+ zENsz!*M!dLIgeAYInAt^{(wppa9l>qa|q)y{!z!wFPim4_{}i`-Ig)kx-s3VWHIyS zudZVI&E&Fg(XwoDDVFUbnUD1Bp(Pk4DNo#_H>H<%;Y@2%xp)LR+(MC zMsKAJ&X(1+mSemTU(>ISnBPZ7oP9q}^9o%4{^&*Jc88!L3$y8+n0iwS2MP0Y^g{67 z+xER`WAXzy!1F_1`Gb$8--s1Us9e&ienVSu0vT-BTq|2tEHx(kif&uXc-^=PRvzsO zr9YBq=;oG6WReQf^JY9%n~COfzu{8Z*f)!nUr+YzT@$!`5f}OGw(=LfyWQn<3-zeM z07=;kqbB}YyGi9>pM@;&a$HVywJ4Xqhf9GhZ*Qld=!JrBL`K!PqX8xO43GM529=~SJ$Q?g!^(fjr7`uz&{0hT2jGs``+EI8RmY}uPbPvyw^H)2#^1k=U zJ-bC<1?0Oy=4?ywhCZ|W>tm!um!V6lkhdQn;T*NEW;!nv?f(k=&Hcw)q>K^M>$=+6 z!~jPd#>kb7H%tH#W#+^;EVak|pH?`KO$qZpHHs=n1BqAfxiXYz2j1+c(eo4}OPf4Z{gY%cu!~{)9$@=R3@txUo(kahy`%}nAQ*WumyHA$1 z6tOao7($XeF0KdChibJUJTAi;hECpI;s5dWL~KxV2~+@p74(m{2Lw?A{_C1!+WSHG zab_$207wZ34+cOgIKcc13j-iMoFY01Qo}*;APDFEVulAniZ}%_1p3FcwHf-GQ}Gv) zhyO;jhW|#=hC#}>qNRU={Q@sC$u781Q^l!-QUnuz(PX7g!F_00i{M{JnAr6km zK-xG^9|!5;;O#g_6$hK+f7c^6@pl=46Mv_`PJnP;H->5QFE~tsjB!=HlOP=&FirhE zo@Y~k%S)&Jj(j%tcY`X^e{sL2E;)S KJo4L&1^fqvpUBq$ delta 5150 zcmY+IbyQUC7RKSwB_W7}G$JiEgmi~=N{7UNppt^XASxXa!we|`gOt?JA>G|El!OS< zAtB(^`S82<&RMh0?|t^O_Zw%OKhCahCHhuFM53dCg-wZp|NE-M%8Mh3!TNnG%c#cr zb4Qxso?!if1QT3F@{{V?f~N+MUpe|&!iu(@e~ z4CyX!LV&KNZN!QEp;3zVCj1`2ATt0Saz?A>Uj^By8;$v|*IR$`pDDGCVEF*A}t&V;?es7pWVRw z1C@T5QIywnv%rGpfg9E0RG9A{{*kItHkeiHe+c%QxT-&gV%BS-1O5}33h2YHZk@$+ zU_xu$KQYZk&=heW(4t1u?|Eq{&~*9;xO4Ll!G8N#swG?ja`bcheIhK?&pZHx20D1V z3AhJ&*QbspKpFC@q7t?-KIvCjY+mEk)rZeVC5j_=kISG=`{M`AWCwKzu2U1fh^1D% zJ@y94uv2Yvo-l$l5Ak0FmPpB>S2$);n+VgOS08&d_RoiSoHZ zrG=Z5jjzQ)jcwvpjMSa9l`-_k(dUDTjMy>|vd|b}qj!!UI;UCu9+SN6wC!JnLi za|Cb4B9_d567QNelNn~OM301tE8KI)b+*8L1C(cgc^y+kBf&HusGi-oG0Dmi$$02R zn~fhmDQ&wY;+%~ue3HYnci(1ExUA~#T`L|2(3ik6Zb?W|grbXC^y;(FxS$GtaBe5X z!&)jRYRQ=OahcvrHV5yR=o@gO5&<$CrEul=Jw#5{!DO;aK}FcD*U2KcF->W6T)^Sw z;cRzW-r3V~B0IvZygDA*^Foo>hV1DD!HEhJTvTIgXQ!sQ!cQm>zMLMAUc=f<+oTO% zD9XT8jvJ!X#wC^*MIU7KIid<=W3J-z=DAdY%m$k&f-cWo^;rEiR^Npjk-%=qHr)J+ zQ+?cIWIOI%5rVlb(9izv;b-ayTv!>wl1TQ5Dht?K?71OVT1QIWqiNot89Zh|*-yRa z3guN03$|NopQt^|(x6Xse=t?*hN{DKKPcJ~ja@)y$mudkx#?yhd$|wa*w2p_>XCy5 z$<_EN*2q~Sf;?+O@;oEI`^!j@xmDLX_CIVznpvE??hbk5PGFwn%Tq;%Mt!@#2UDqs zJ0o-2T>%%z+@=Aqr%b93ICJy}ESQa0ydy!PCrMK~jM0-jLU?35WjfCNg0NRQ3$rMc zVLillCc>J5gjxA#Q5ai38&Qp-?;B;sGdD3lc0DU;dsi|-<>cbMqT{q3Py&G`n^S*i zo$5F*{9V!9Zw>$^&M~H8Cr$#o^w~|j5E8EouDKC&9#xhAb_~Y&sWK41IzP) z{ue|PMu4y?-o}lJP%pQxfoz3)d`Ftx&@D)0PKe_^q+Z=;OiJ+NzBQGIodR&oy4)#O zN6L4%i>>CJ5`7M+eCFu+`J1rCSRBL@+`>rm6|RSC6%JDP4pdt0Jq2ZeX$q9?``s>r zM?!trJ+qGAkb-N>@ESf!HF5l*?Ceo03zBFCUBpmrR;;Zy=TLSRqKiE%Zkz|#VH@J8 zJxtx+X4l+J>;ZV1lSN2~$!Xg}(;ec|Sus%%rT(lY9;EQGUZY7NOCM6d=v@8I-7V`h zT}rFx=ICUw;d12l#UJZbJ?fRYsKr#$D{`Tu&nf%&ph9X~0SQ_WZ<|Qx-PkjNN11P` z!yI%|4Bu*gRO+CCejNii+t!oaS-wCd&#TmlEP5j`)+=}+Wjs`wun}^Ut-zRb6m3sV zvO&rqLp;cVMgae%YmpLScl3l!zl5u#K{@A%pQc~Bg!2|?ESfA8q{u9Zy2=}+I|=~5 zm9Z}9p*WrST5Iw36H|INW5E$~onBgPkV%h2{^$c?e#RmhY}x2%HOx4?%$N`uvdKTF zmvO9qd02bN_MY@Dg>gv4d=F=Q)$Q5!_rr>`dNEL@6op)>Oc|2V_aj5_;@-Wv%W8JF z59y-2q{##VOtLa68X=nV4&%>wO+^Zo7$0v~4O>MZc3aj;?YHzfcf5Zz#f4B3Y#VLt zJ}z|1ZVGJ?=t|^$4JMgpy3&Q1D$`xw9Gfdsyj^db?c=St0V~dhkW;Odqb}%ZJA%k( zr5-ciB|9bc`SnwKV$Bq z{>|WaUas|Ld`7lP5qIY23}$w>TIhqu1EGf&(lOzZ#pRnZU%P#&27OW>`2o*es(WGL zNgf|YUwo9+rHN7;>AnQy+M^=R+-+b?h_V;4$dy9#&B0{6+D3P4CDt@!fyQl-&TQQ= zxgk^N`XP`=$^VA%iC$7%gxTm(ZeO*pr>a$tv5j4KE?2WL^`3}1Us%V?`sg;RMQZ~b zwgeZ`_YErHT_UvP`W6M@VC9SdYHV)mh{|N=Q?Z146$c|`$f=f6nD|#VX@HG0OiPAU zD*8l5MX9Cqh`qHW(ct=&k_540^4Mp?dbL^-{oaf>^o6E2wR zGka5MXzgGc_O>^`?ao}V)P0N0;*^|TRQkcHS)7&7tsf1jFY;6z6-#a&4SlV};MQq# zVjb#eE;`8|Kf5$Y{X(-loPUbnBp%#F?MM$@Wh}lw zNLMYl9^Dswh^wCFlYG*-(c#N~v@pH1Z-$?4tScT^()kT+0)2fFBV;gA{N1pYyHY*X z`EI_^l(z1ts=3?=Ki$DLrHf2WuR2W^FV}QUx?_kqNMtiG74+RP|5KA}{f4lYmL9e8 z{H5s-cveC!fNU|wu}ckaYih}dB#j0Bp*e`2t^;+Aa>5BQ$g^GVAj?3Uf9^vgO2N{izqFroG+6A#09h--udqk#WoqscVkqUHi_ zayZ1$#79JIpYv1K2Sb27sTRZ@JK}m^c)J2@^>V_ucd8S`aj&E|_W`P~qW($Ui9kwh z5EZWMIh>+u1GDsrJe;9;l%Z$u@a=BuAcV&b8qA=KgV^1&s9&g34!q4}KSC}MjBtstT z%_*c02i3={FBt34)xPLD;&L6$UAN4Ao#1)vOe|9I+iVSSK?U__8m!rubHI0lC;dCM zuB1hQWoAHE?qS`JlphicIG{$~l&egvhH87g%9~Rac2M_q^dWz`nm~BLWx4%3dD7Z}t_^m*vxBsuK@T|%XBq2-34Rq) z@*Jjrkg>{s_->Tr$AnnFN2{cEZ%;)J!$b+}%-TD7vFx%$t2LM%qZjx5?YaU{rq;a- zri|Jb5l(o$u6WK6yi>+kM|`F+>k^W5-Lh`-<8n4>W^H@Z?N0B043kw{9veX7{4q-t zMp3T{uFN<-gDzh;xSl(%X;5OESqW?*Hx|8Qx0-T7FM6q^f~QLM%agrA2c%OE8-{`# zcV6yT+Mvu6)!GCr6tl>AXq6P`m0r`mB@)=9Q0-#yB#cbC7tR*d($d)O9`B{OrX>sl zr#B5(PDMUa_;E()#Gi0~_TmF%{RO#x7q;uCz!>givXMbr#d6D$KFU-VSIdn?V=yMK z_2IF|$C}R;CF&lCJfd|{YTRrU+_CtM2JhytB*W9)N2eW*$1Dh{T?-EE@3aD3@(K6JCcQ}h?9)o=2S42Kjd`8r3}fe=eWJclo#)@2b6cTvMYu%eWme=-nUx5Abjh4%W&V-pqZ$< zE3*5}{it6&khY@($`kKi5;yryuaA$32y=+E4z|0ewn2=Z`LjNf2h8tC322n5Df2*! z2?9In9CM2uyC)cx#*pV(VzDIWB1i30Hm7v&&IKC3^IkC#>Qv2{(Gy$e7uZUh_n*vO z>`TXbI?-~TbP{ZR%j8%5EK%s3k!wq1I6-;H5pfnsE}qs#^f7*a2Qln>bNxg0Ucjq@ z;ZWnT2Tr8BOpmEC&WOHE$OH?!{mSqX!H?d$uQbRqkrMGsE>#9H57VXdynl6*9v-NE z=I(uzyJ>kdt*}8B@25xP8B)se^oz!NBOYawN=h2#a~=P7@t~e92_A0bx-SZjwBJ~` zl2Uu>c+~GjoQ9{+r+P)%aWEx16mp}8*|d1FvYAb$z@)G1AzWe4X{}t|^ujlRDeT82)FfdkW z|9m}T0+bm4c9~%DS#rE<{F@XVfioc&t8Z8|3c4S zxP<~t(diy2Ko<>NC_ozxG;@F+8r 0) { - String line = myClient.readStringUntil((byte) 10); - //println("Line: " + line); - if (line == null) line = ""; else line = line.trim(); - if (line.startsWith(" 1) updateCentroid(invisible); + centroid.tick(); + + background( 0 ); + translate( width/2 , height/2 ); + scale( centroid.z() ); + translate( -centroid.x(), -centroid.y() ); + + drawNetwork(invisible); } -public HashMap parseProps(String s) { - String[] l = s.split(" "); - HashMap map = new HashMap(); - int p; - String z; - for (int i = 0; i < l.length; i++) { - p = l[i].indexOf("="); - if (p > 0) { - z = l[i].substring(p + 1).trim(); - if (z.charAt(0) == '"') z = z.substring(1); - if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); - map.put(l[i].substring(0, p), z); +public void initRequest(boolean update) { + myClient = new Client(this, host, port); + myClient.write((update) ? "GET /xml/webstructure.xml?latest= HTTP/1.1\n" : "GET /xml/webstructure.xml HTTP/1.1\n"); + myClient.write("Host: localhost\n\n"); +} + +public void processRequestResponse(int steps) { + if (((myClient == null) || (myClient.available() <= 0)) && (System.currentTimeMillis() - lastUpdate > 10000)) { + initRequest(true); + lastUpdate = Long.MAX_VALUE; + return; + } + for (int i = 0; i < steps; i++) { + if (myClient.available() > 0) { + String line = myClient.readStringUntil((byte) 10); + //println("Line: " + line); + if (line == null) line = ""; else line = line.trim(); + if (line.startsWith(" TWO_PI) a -= TWO_PI; nodes.put(parsingHostID, h); addAttraction(h.node); } @@ -97,113 +144,160 @@ public void processCitation(HashMap props) { } catch (NumberFormatException e) {} // find the two nodes that have a relation host h = (host) nodes.get(id); - if (h == null) return; // host is not known TODO: store these and create relation later + if (h == null) { + return; /* + h = new host(host, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0)); + a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI; + nodes.put(id, h); + addAttraction(h.node);*/ + } + h.time = System.currentTimeMillis(); host p = (host) nodes.get(parsingHostID); // this should be successful - addRelation(h.node, p.node); + // prevent that a spring is made twice + for ( int i = 0; i < physics.numberOfSprings(); ++i ) { + Spring e = physics.getSpring(i); + Particle a = e.getOneEnd(); + Particle b = e.getTheOtherEnd(); + if (((a == h.node) && (b == p.node)) || ((b == h.node) && (a == p.node))) return; + } + physics.makeSpring(h.node, p.node, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH ); } -public void draw() { - processRequestResponse(); - - physics.tick( 1.0f ); - if (physics.numberOfParticles() > 1) updateCentroid(); - centroid.tick(); +public HashMap parseProps(String s) { + String[] l = s.split(" "); + HashMap map = new HashMap(); + int p; + String z; + for (int i = 0; i < l.length; i++) { + p = l[i].indexOf("="); + if (p > 0) { + z = l[i].substring(p + 1).trim(); + if (z.charAt(0) == '"') z = z.substring(1); + if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); + map.put(l[i].substring(0, p), z); + } + } + return map; +} - background( 0 ); - translate( width/2 , height/2 ); - scale( centroid.z() ); - translate( -centroid.x(), -centroid.y() ); - - drawNetwork(); +public HashSet invisibleParticles() { + // get nodes that have no edges + HashSet particles = new HashSet(); + Iterator j = nodes.values().iterator(); + host h; + long t = 0, n = System.currentTimeMillis(); + while (j.hasNext()) { + h = (host) j.next(); + t += n - h.time; + particles.add(h.node); + } + t = t / (nodes.size() + 1); + for ( int i = 0; i < physics.numberOfSprings(); ++i ) { + Spring e = physics.getSpring(i); + particles.remove(e.getOneEnd()); + particles.remove(e.getTheOtherEnd()); + } + // add more nodes if the number is too large + if (nodes.size() > 80) { + j = nodes.values().iterator(); + while (j.hasNext()) { + h = (host) j.next(); + if (n - h.time > 15000) particles.add(h.node); + if (nodes.size() - particles.size() < 80) break; + } + } + return particles; } -public void drawNetwork() { - fill( 100, 255, 100 ); +public void drawNetwork(HashSet invisible) { // draw vertices + fill( 120, 255, 120 ); noStroke(); String name; - Iterator j = nodes.values().iterator(); host h; + Iterator j = nodes.values().iterator(); while (j.hasNext()) { h = (host) j.next(); Particle v = h.node; + if (invisible.contains(v)) continue; ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE); name = h.name; text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14); } - + // draw center - /* - ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); - name = "Center0"; - text(name, center0.position().x() - (name.length() * 26 / 10), center0.position().y() + 14); - ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); - name = "Center1"; - text(name, center1.position().x() - (name.length() * 26 / 10), center1.position().y() + 14); - */ - + //fill( 255, 0, 0 ); + //ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); + //ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); + // draw edges - stroke( 160 ); - beginShape( LINES ); + stroke( 200 ); for ( int i = 0; i < physics.numberOfSprings(); ++i ) { Spring e = physics.getSpring( i ); Particle a = e.getOneEnd(); + if (invisible.contains(a)) continue; Particle b = e.getTheOtherEnd(); - vertex( a.position().x(), a.position().y() ); - vertex( b.position().x(), b.position().y() ); + if (invisible.contains(b)) continue; + line(a.position().x(), a.position().y(), b.position().x(), b.position().y()); } - endShape(); + } public void keyPressed() { - if ( key == 'c' ) { - initializePhysics(); - return; - } - - if ( key == ' ' ) { - Particle p = physics.makeParticle(); - addRelation(p, physics.getParticle( (int) random( 0, physics.numberOfParticles()-1) )); - addAttraction(p); + if ( key == 'c' ) initializePhysics(); + if ( key == 'a' ) x = Math.max(-1.0f, x - 0.1f); + if ( key == 'd' ) x = Math.min( 1.0f, x + 0.1f); + if ( key == 'w' ) y = Math.max(-1.0f, y - 0.1f); + if ( key == 's' ) y = Math.min( 1.0f, y + 0.1f); + if ( key == '-' ) z = Math.max( 1.0f, z - 1.0f); + if ( key == '+' ) z = Math.min(10.0f, z + 1.0f); + if ( key == '0' ) { x = 0.0f; y = 0.0f; z = 1.0f; } + if ( key == 't' ) { + HashSet hs = new HashSet(); + for (int i = 0; i < physics.numberOfParticles(); ++i ) { + hs.add(physics.getParticle(i)); + } + for (int i = 0; i < physics.numberOfSprings(); ++i ) { + hs.remove(physics.getSpring(i).getOneEnd()); + hs.remove(physics.getSpring(i).getTheOtherEnd()); + } + Iterator i = hs.iterator(); + while (i.hasNext()) { + ((Particle) i.next()).kill(); + } return; } } -public void updateCentroid() { +public void updateCentroid(HashSet invisible) { float xMax = Float.NEGATIVE_INFINITY, xMin = Float.POSITIVE_INFINITY, yMin = Float.POSITIVE_INFINITY, yMax = Float.NEGATIVE_INFINITY; - for ( int i = 0; i < physics.numberOfParticles(); ++i ) { - Particle p = physics.getParticle( i ); + for (int i = 0; i < physics.numberOfParticles(); ++i) { + Particle p = physics.getParticle(i); + if ((i >= 2) && ((p == center0) || (p == center1) || (invisible.contains(p)))) continue; xMax = max( xMax, p.position().x() ); xMin = min( xMin, p.position().x() ); yMin = min( yMin, p.position().y() ); yMax = max( yMax, p.position().y() ); } + float deltaX = xMax-xMin; float deltaY = yMax-yMin; - if ( deltaY > deltaX ) - centroid.setTarget( xMin + 0.5f*deltaX, yMin +0.5f*deltaY, height/(deltaY+50) ); - else - centroid.setTarget( xMin + 0.5f*deltaX, yMin +0.5f*deltaY, width/(deltaX+50) ); -} - -public void initializePhysics() { - physics.clear(); - center0 = physics.makeParticle(1.0f, 0, 0, 0); - center0.makeFixed(); - center1 = physics.makeParticle(1.0f, EDGE_LENGTH * 30, 0, 0); - center1.makeFixed(); - centroid.setValue( 0, 0, 1.0f ); + centroid.setTarget( + xMin + (x + 1) * 0.5f * deltaX, + yMin + (y + 1) * 0.5f * deltaY, + z * ((deltaY > deltaX) ? height / (deltaY + 50) : width / (deltaX + 50)) + ); } public void addAttraction(Particle p) { - physics.makeAttraction(center0, p, 5000.0f, 3 * EDGE_LENGTH); - physics.makeAttraction(center1, p, 5000.0f, 3 * EDGE_LENGTH); + physics.makeAttraction(center0, p, SPACER_STRENGTH * 10000.0f, 100 * EDGE_LENGTH); + physics.makeAttraction(center1, p, SPACER_STRENGTH * 10000.0f, 100 * EDGE_LENGTH); // spacers for ( int i = 0; i < physics.numberOfParticles(); ++i ) { @@ -212,17 +306,14 @@ public void addAttraction(Particle p) { } } -public void addRelation(Particle p, Particle other) { - physics.makeSpring( p, other, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH ); - //p.moveTo( q.position().x() + random( -1, 1 ), q.position().y() + random( -1, 1 ), 0 ); -} - static class host { String name; Particle node; + long time; public host(String name, Particle node) { this.name = name; this.node = node; + this.time = System.currentTimeMillis(); } } diff --git a/htroot/processing/domaingraph/applet/domaingraph.pde b/htroot/processing/domaingraph/applet/domaingraph.pde index e79f083e7..4805b93be 100755 --- a/htroot/processing/domaingraph/applet/domaingraph.pde +++ b/htroot/processing/domaingraph/applet/domaingraph.pde @@ -12,77 +12,124 @@ import traer.animation.*; import processing.net.*; final float NODE_SIZE = 6; -final float EDGE_LENGTH = 50; -final float EDGE_STRENGTH = 0.01; -final float SPACER_STRENGTH = 10; +final float EDGE_LENGTH = 30; +final float EDGE_STRENGTH = 0.001; +final float SPACER_STRENGTH = 250; ParticleSystem physics; Smoother3D centroid; PFont font; +float x = 0.0; +float y = 0.0; +float z = 1.0; Client myClient; Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields String parsingHostName = ""; String parsingHostID = ""; HashMap nodes = new HashMap(); // map that holds host objects +String host; +int port; +float a = 0.0; +long lastUpdate = Long.MAX_VALUE; +boolean initTime = true; void setup() { String[] fontList = PFont.list(); //println(fontList); font = createFont(fontList[0], 32); //just take any, should be mostly Arial - textFont(font, 12); + textFont(font, 9); size(660, 400); smooth(); - frameRate( 24 ); - strokeWeight( 2 ); + frameRate( 12 ); + strokeWeight( 1 ); ellipseMode( CENTER ); physics = new ParticleSystem( 0, 0.25 ); centroid = new Smoother3D( 0.8 ); initializePhysics(); - initRequest(); + URL url = null; + try { + url = getDocumentBase(); + } catch (NullPointerException e) {} + if (url == null) { + host="localhost"; + port=8080; + } else { + host=url.getHost(); + port=url.getPort(); + } + //println("CodeBase: " + url); + //println("host: " + host); + //println("port: " + port); + + initRequest(false); } -void initRequest() { - myClient = new Client(this, "localhost", 8080); - myClient.write("GET /xml/webstructure.xml HTTP/1.1\n"); - myClient.write("Host: localhost\n\n"); +void initializePhysics() { + physics.clear(); + center0 = physics.makeParticle(1.0, -EDGE_LENGTH * 10, 0, 0); + center0.makeFixed(); + center1 = physics.makeParticle(1.0, EDGE_LENGTH * 10, 0, 0); + center1.makeFixed(); + centroid.setValue( 0, 0, 1.0 ); } -void processRequestResponse() { - if (myClient.available() > 0) { - String line = myClient.readStringUntil((byte) 10); - //println("Line: " + line); - if (line == null) line = ""; else line = line.trim(); - if (line.startsWith(" 1) updateCentroid(invisible); + centroid.tick(); + + background( 0 ); + translate( width/2 , height/2 ); + scale( centroid.z() ); + translate( -centroid.x(), -centroid.y() ); + + drawNetwork(invisible); } -HashMap parseProps(String s) { - String[] l = s.split(" "); - HashMap map = new HashMap(); - int p; - String z; - for (int i = 0; i < l.length; i++) { - p = l[i].indexOf("="); - if (p > 0) { - z = l[i].substring(p + 1).trim(); - if (z.charAt(0) == '"') z = z.substring(1); - if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); - map.put(l[i].substring(0, p), z); +void initRequest(boolean update) { + myClient = new Client(this, host, port); + myClient.write((update) ? "GET /xml/webstructure.xml?latest= HTTP/1.1\n" : "GET /xml/webstructure.xml HTTP/1.1\n"); + myClient.write("Host: localhost\n\n"); +} + +void processRequestResponse(int steps) { + if (((myClient == null) || (myClient.available() <= 0)) && (System.currentTimeMillis() - lastUpdate > 10000)) { + initRequest(true); + lastUpdate = Long.MAX_VALUE; + return; + } + for (int i = 0; i < steps; i++) { + if (myClient.available() > 0) { + String line = myClient.readStringUntil((byte) 10); + //println("Line: " + line); + if (line == null) line = ""; else line = line.trim(); + if (line.startsWith(" TWO_PI) a -= TWO_PI; nodes.put(parsingHostID, h); addAttraction(h.node); } @@ -97,113 +144,160 @@ void processCitation(HashMap props) { } catch (NumberFormatException e) {} // find the two nodes that have a relation host h = (host) nodes.get(id); - if (h == null) return; // host is not known TODO: store these and create relation later + if (h == null) { + return; /* + h = new host(host, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0)); + a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI; + nodes.put(id, h); + addAttraction(h.node);*/ + } + h.time = System.currentTimeMillis(); host p = (host) nodes.get(parsingHostID); // this should be successful - addRelation(h.node, p.node); + // prevent that a spring is made twice + for ( int i = 0; i < physics.numberOfSprings(); ++i ) { + Spring e = physics.getSpring(i); + Particle a = e.getOneEnd(); + Particle b = e.getTheOtherEnd(); + if (((a == h.node) && (b == p.node)) || ((b == h.node) && (a == p.node))) return; + } + physics.makeSpring(h.node, p.node, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH ); } -void draw() { - processRequestResponse(); - - physics.tick( 1.0 ); - if (physics.numberOfParticles() > 1) updateCentroid(); - centroid.tick(); +HashMap parseProps(String s) { + String[] l = s.split(" "); + HashMap map = new HashMap(); + int p; + String z; + for (int i = 0; i < l.length; i++) { + p = l[i].indexOf("="); + if (p > 0) { + z = l[i].substring(p + 1).trim(); + if (z.charAt(0) == '"') z = z.substring(1); + if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); + map.put(l[i].substring(0, p), z); + } + } + return map; +} - background( 0 ); - translate( width/2 , height/2 ); - scale( centroid.z() ); - translate( -centroid.x(), -centroid.y() ); - - drawNetwork(); +HashSet invisibleParticles() { + // get nodes that have no edges + HashSet particles = new HashSet(); + Iterator j = nodes.values().iterator(); + host h; + long t = 0, n = System.currentTimeMillis(); + while (j.hasNext()) { + h = (host) j.next(); + t += n - h.time; + particles.add(h.node); + } + t = t / (nodes.size() + 1); + for ( int i = 0; i < physics.numberOfSprings(); ++i ) { + Spring e = physics.getSpring(i); + particles.remove(e.getOneEnd()); + particles.remove(e.getTheOtherEnd()); + } + // add more nodes if the number is too large + if (nodes.size() > 80) { + j = nodes.values().iterator(); + while (j.hasNext()) { + h = (host) j.next(); + if (n - h.time > 15000) particles.add(h.node); + if (nodes.size() - particles.size() < 80) break; + } + } + return particles; } -void drawNetwork() { - fill( 100, 255, 100 ); +void drawNetwork(HashSet invisible) { // draw vertices + fill( 120, 255, 120 ); noStroke(); String name; - Iterator j = nodes.values().iterator(); host h; + Iterator j = nodes.values().iterator(); while (j.hasNext()) { h = (host) j.next(); Particle v = h.node; + if (invisible.contains(v)) continue; ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE); name = h.name; text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14); } - + // draw center - /* - ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); - name = "Center0"; - text(name, center0.position().x() - (name.length() * 26 / 10), center0.position().y() + 14); - ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); - name = "Center1"; - text(name, center1.position().x() - (name.length() * 26 / 10), center1.position().y() + 14); - */ - + //fill( 255, 0, 0 ); + //ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); + //ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); + // draw edges - stroke( 160 ); - beginShape( LINES ); + stroke( 200 ); for ( int i = 0; i < physics.numberOfSprings(); ++i ) { Spring e = physics.getSpring( i ); Particle a = e.getOneEnd(); + if (invisible.contains(a)) continue; Particle b = e.getTheOtherEnd(); - vertex( a.position().x(), a.position().y() ); - vertex( b.position().x(), b.position().y() ); + if (invisible.contains(b)) continue; + line(a.position().x(), a.position().y(), b.position().x(), b.position().y()); } - endShape(); + } void keyPressed() { - if ( key == 'c' ) { - initializePhysics(); - return; - } - - if ( key == ' ' ) { - Particle p = physics.makeParticle(); - addRelation(p, physics.getParticle( (int) random( 0, physics.numberOfParticles()-1) )); - addAttraction(p); + if ( key == 'c' ) initializePhysics(); + if ( key == 'a' ) x = Math.max(-1.0, x - 0.1); + if ( key == 'd' ) x = Math.min( 1.0, x + 0.1); + if ( key == 'w' ) y = Math.max(-1.0, y - 0.1); + if ( key == 's' ) y = Math.min( 1.0, y + 0.1); + if ( key == '-' ) z = Math.max( 1.0, z - 1.0); + if ( key == '+' ) z = Math.min(10.0, z + 1.0); + if ( key == '0' ) { x = 0.0; y = 0.0; z = 1.0; } + if ( key == 't' ) { + HashSet hs = new HashSet(); + for (int i = 0; i < physics.numberOfParticles(); ++i ) { + hs.add(physics.getParticle(i)); + } + for (int i = 0; i < physics.numberOfSprings(); ++i ) { + hs.remove(physics.getSpring(i).getOneEnd()); + hs.remove(physics.getSpring(i).getTheOtherEnd()); + } + Iterator i = hs.iterator(); + while (i.hasNext()) { + ((Particle) i.next()).kill(); + } return; } } -void updateCentroid() { +void updateCentroid(HashSet invisible) { float xMax = Float.NEGATIVE_INFINITY, xMin = Float.POSITIVE_INFINITY, yMin = Float.POSITIVE_INFINITY, yMax = Float.NEGATIVE_INFINITY; - for ( int i = 0; i < physics.numberOfParticles(); ++i ) { - Particle p = physics.getParticle( i ); + for (int i = 0; i < physics.numberOfParticles(); ++i) { + Particle p = physics.getParticle(i); + if ((i >= 2) && ((p == center0) || (p == center1) || (invisible.contains(p)))) continue; xMax = max( xMax, p.position().x() ); xMin = min( xMin, p.position().x() ); yMin = min( yMin, p.position().y() ); yMax = max( yMax, p.position().y() ); } + float deltaX = xMax-xMin; float deltaY = yMax-yMin; - if ( deltaY > deltaX ) - centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, height/(deltaY+50) ); - else - centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, width/(deltaX+50) ); -} - -void initializePhysics() { - physics.clear(); - center0 = physics.makeParticle(1.0, 0, 0, 0); - center0.makeFixed(); - center1 = physics.makeParticle(1.0, EDGE_LENGTH * 30, 0, 0); - center1.makeFixed(); - centroid.setValue( 0, 0, 1.0 ); + centroid.setTarget( + xMin + (x + 1) * 0.5 * deltaX, + yMin + (y + 1) * 0.5 * deltaY, + z * ((deltaY > deltaX) ? height / (deltaY + 50) : width / (deltaX + 50)) + ); } void addAttraction(Particle p) { - physics.makeAttraction(center0, p, 5000.0, 3 * EDGE_LENGTH); - physics.makeAttraction(center1, p, 5000.0, 3 * EDGE_LENGTH); + physics.makeAttraction(center0, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH); + physics.makeAttraction(center1, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH); // spacers for ( int i = 0; i < physics.numberOfParticles(); ++i ) { @@ -212,16 +306,13 @@ void addAttraction(Particle p) { } } -void addRelation(Particle p, Particle other) { - physics.makeSpring( p, other, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH ); - //p.moveTo( q.position().x() + random( -1, 1 ), q.position().y() + random( -1, 1 ), 0 ); -} - static class host { String name; Particle node; + long time; public host(String name, Particle node) { this.name = name; this.node = node; + this.time = System.currentTimeMillis(); } } diff --git a/htroot/processing/domaingraph/domaingraph.pde b/htroot/processing/domaingraph/domaingraph.pde index 5c38a76d9..4805b93be 100755 --- a/htroot/processing/domaingraph/domaingraph.pde +++ b/htroot/processing/domaingraph/domaingraph.pde @@ -12,77 +12,124 @@ import traer.animation.*; import processing.net.*; final float NODE_SIZE = 6; -final float EDGE_LENGTH = 50; +final float EDGE_LENGTH = 30; final float EDGE_STRENGTH = 0.001; -final float SPACER_STRENGTH = 500; +final float SPACER_STRENGTH = 250; ParticleSystem physics; Smoother3D centroid; PFont font; +float x = 0.0; +float y = 0.0; +float z = 1.0; Client myClient; Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields String parsingHostName = ""; String parsingHostID = ""; HashMap nodes = new HashMap(); // map that holds host objects +String host; +int port; +float a = 0.0; +long lastUpdate = Long.MAX_VALUE; +boolean initTime = true; void setup() { String[] fontList = PFont.list(); //println(fontList); font = createFont(fontList[0], 32); //just take any, should be mostly Arial - textFont(font, 12); + textFont(font, 9); size(660, 400); smooth(); - frameRate( 24 ); - strokeWeight( 2 ); + frameRate( 12 ); + strokeWeight( 1 ); ellipseMode( CENTER ); physics = new ParticleSystem( 0, 0.25 ); centroid = new Smoother3D( 0.8 ); initializePhysics(); - initRequest(); + URL url = null; + try { + url = getDocumentBase(); + } catch (NullPointerException e) {} + if (url == null) { + host="localhost"; + port=8080; + } else { + host=url.getHost(); + port=url.getPort(); + } + //println("CodeBase: " + url); + //println("host: " + host); + //println("port: " + port); + + initRequest(false); } -void initRequest() { - myClient = new Client(this, "localhost", 8080); - myClient.write("GET /xml/webstructure.xml HTTP/1.1\n"); - myClient.write("Host: localhost\n\n"); +void initializePhysics() { + physics.clear(); + center0 = physics.makeParticle(1.0, -EDGE_LENGTH * 10, 0, 0); + center0.makeFixed(); + center1 = physics.makeParticle(1.0, EDGE_LENGTH * 10, 0, 0); + center1.makeFixed(); + centroid.setValue( 0, 0, 1.0 ); } -void processRequestResponse() { - if (myClient.available() > 0) { - String line = myClient.readStringUntil((byte) 10); - //println("Line: " + line); - if (line == null) line = ""; else line = line.trim(); - if (line.startsWith(" 1) updateCentroid(invisible); + centroid.tick(); + + background( 0 ); + translate( width/2 , height/2 ); + scale( centroid.z() ); + translate( -centroid.x(), -centroid.y() ); + + drawNetwork(invisible); } -HashMap parseProps(String s) { - String[] l = s.split(" "); - HashMap map = new HashMap(); - int p; - String z; - for (int i = 0; i < l.length; i++) { - p = l[i].indexOf("="); - if (p > 0) { - z = l[i].substring(p + 1).trim(); - if (z.charAt(0) == '"') z = z.substring(1); - if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); - map.put(l[i].substring(0, p), z); +void initRequest(boolean update) { + myClient = new Client(this, host, port); + myClient.write((update) ? "GET /xml/webstructure.xml?latest= HTTP/1.1\n" : "GET /xml/webstructure.xml HTTP/1.1\n"); + myClient.write("Host: localhost\n\n"); +} + +void processRequestResponse(int steps) { + if (((myClient == null) || (myClient.available() <= 0)) && (System.currentTimeMillis() - lastUpdate > 10000)) { + initRequest(true); + lastUpdate = Long.MAX_VALUE; + return; + } + for (int i = 0; i < steps; i++) { + if (myClient.available() > 0) { + String line = myClient.readStringUntil((byte) 10); + //println("Line: " + line); + if (line == null) line = ""; else line = line.trim(); + if (line.startsWith(" TWO_PI) a -= TWO_PI; nodes.put(parsingHostID, h); addAttraction(h.node); } @@ -97,72 +144,116 @@ void processCitation(HashMap props) { } catch (NumberFormatException e) {} // find the two nodes that have a relation host h = (host) nodes.get(id); - if (h == null) return; // host is not known TODO: store these and create relation later + if (h == null) { + return; /* + h = new host(host, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0)); + a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI; + nodes.put(id, h); + addAttraction(h.node);*/ + } + h.time = System.currentTimeMillis(); host p = (host) nodes.get(parsingHostID); // this should be successful - addRelation(h.node, p.node); + // prevent that a spring is made twice + for ( int i = 0; i < physics.numberOfSprings(); ++i ) { + Spring e = physics.getSpring(i); + Particle a = e.getOneEnd(); + Particle b = e.getTheOtherEnd(); + if (((a == h.node) && (b == p.node)) || ((b == h.node) && (a == p.node))) return; + } + physics.makeSpring(h.node, p.node, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH ); } -void draw() { - processRequestResponse(); - - physics.tick( 1.0 ); - if (physics.numberOfParticles() > 1) updateCentroid(); - centroid.tick(); +HashMap parseProps(String s) { + String[] l = s.split(" "); + HashMap map = new HashMap(); + int p; + String z; + for (int i = 0; i < l.length; i++) { + p = l[i].indexOf("="); + if (p > 0) { + z = l[i].substring(p + 1).trim(); + if (z.charAt(0) == '"') z = z.substring(1); + if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); + map.put(l[i].substring(0, p), z); + } + } + return map; +} - background( 0 ); - translate( width/2 , height/2 ); - scale( centroid.z() ); - translate( -centroid.x(), -centroid.y() ); - - drawNetwork(); +HashSet invisibleParticles() { + // get nodes that have no edges + HashSet particles = new HashSet(); + Iterator j = nodes.values().iterator(); + host h; + long t = 0, n = System.currentTimeMillis(); + while (j.hasNext()) { + h = (host) j.next(); + t += n - h.time; + particles.add(h.node); + } + t = t / (nodes.size() + 1); + for ( int i = 0; i < physics.numberOfSprings(); ++i ) { + Spring e = physics.getSpring(i); + particles.remove(e.getOneEnd()); + particles.remove(e.getTheOtherEnd()); + } + // add more nodes if the number is too large + if (nodes.size() > 80) { + j = nodes.values().iterator(); + while (j.hasNext()) { + h = (host) j.next(); + if (n - h.time > 15000) particles.add(h.node); + if (nodes.size() - particles.size() < 80) break; + } + } + return particles; } -void drawNetwork() { - fill( 100, 255, 100 ); +void drawNetwork(HashSet invisible) { // draw vertices + fill( 120, 255, 120 ); noStroke(); String name; - Iterator j = nodes.values().iterator(); host h; + Iterator j = nodes.values().iterator(); while (j.hasNext()) { h = (host) j.next(); Particle v = h.node; + if (invisible.contains(v)) continue; ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE); name = h.name; text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14); } - + // draw center - /* - ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); - name = "Center0"; - text(name, center0.position().x() - (name.length() * 26 / 10), center0.position().y() + 14); - ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); - name = "Center1"; - text(name, center1.position().x() - (name.length() * 26 / 10), center1.position().y() + 14); - */ - + //fill( 255, 0, 0 ); + //ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); + //ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); + // draw edges - stroke( 160 ); - beginShape( LINES ); + stroke( 200 ); for ( int i = 0; i < physics.numberOfSprings(); ++i ) { Spring e = physics.getSpring( i ); Particle a = e.getOneEnd(); + if (invisible.contains(a)) continue; Particle b = e.getTheOtherEnd(); - vertex( a.position().x(), a.position().y() ); - vertex( b.position().x(), b.position().y() ); + if (invisible.contains(b)) continue; + line(a.position().x(), a.position().y(), b.position().x(), b.position().y()); } - endShape(); + } void keyPressed() { - if ( key == 'c' ) { - initializePhysics(); - return; - } - - if ( key == 'd' ) { + if ( key == 'c' ) initializePhysics(); + if ( key == 'a' ) x = Math.max(-1.0, x - 0.1); + if ( key == 'd' ) x = Math.min( 1.0, x + 0.1); + if ( key == 'w' ) y = Math.max(-1.0, y - 0.1); + if ( key == 's' ) y = Math.min( 1.0, y + 0.1); + if ( key == '-' ) z = Math.max( 1.0, z - 1.0); + if ( key == '+' ) z = Math.min(10.0, z + 1.0); + if ( key == '0' ) { x = 0.0; y = 0.0; z = 1.0; } + if ( key == 't' ) { HashSet hs = new HashSet(); for (int i = 0; i < physics.numberOfParticles(); ++i ) { hs.add(physics.getParticle(i)); @@ -177,49 +268,36 @@ void keyPressed() { } return; } - - if ( key == ' ' ) { - Particle p = physics.makeParticle(); - addRelation(p, physics.getParticle( (int) random( 0, physics.numberOfParticles()-1) )); - addAttraction(p); - return; - } } -void updateCentroid() { +void updateCentroid(HashSet invisible) { float xMax = Float.NEGATIVE_INFINITY, xMin = Float.POSITIVE_INFINITY, yMin = Float.POSITIVE_INFINITY, yMax = Float.NEGATIVE_INFINITY; - for ( int i = 0; i < physics.numberOfParticles(); ++i ) { - Particle p = physics.getParticle( i ); + for (int i = 0; i < physics.numberOfParticles(); ++i) { + Particle p = physics.getParticle(i); + if ((i >= 2) && ((p == center0) || (p == center1) || (invisible.contains(p)))) continue; xMax = max( xMax, p.position().x() ); xMin = min( xMin, p.position().x() ); yMin = min( yMin, p.position().y() ); yMax = max( yMax, p.position().y() ); } + float deltaX = xMax-xMin; float deltaY = yMax-yMin; - if ( deltaY > deltaX ) - centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, height/(deltaY+50) ); - else - centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, width/(deltaX+50) ); -} - -void initializePhysics() { - physics.clear(); - center0 = physics.makeParticle(1.0, 0, 0, 0); - center0.makeFixed(); - center1 = physics.makeParticle(1.0, EDGE_LENGTH * 30, 0, 0); - center1.makeFixed(); - centroid.setValue( 0, 0, 1.0 ); + centroid.setTarget( + xMin + (x + 1) * 0.5 * deltaX, + yMin + (y + 1) * 0.5 * deltaY, + z * ((deltaY > deltaX) ? height / (deltaY + 50) : width / (deltaX + 50)) + ); } void addAttraction(Particle p) { - physics.makeAttraction(center0, p, 5000.0, 3 * EDGE_LENGTH); - physics.makeAttraction(center1, p, 5000.0, 3 * EDGE_LENGTH); + physics.makeAttraction(center0, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH); + physics.makeAttraction(center1, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH); // spacers for ( int i = 0; i < physics.numberOfParticles(); ++i ) { @@ -228,16 +306,13 @@ void addAttraction(Particle p) { } } -void addRelation(Particle p, Particle other) { - physics.makeSpring( p, other, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH ); - //p.moveTo( q.position().x() + random( -1, 1 ), q.position().y() + random( -1, 1 ), 0 ); -} - static class host { String name; Particle node; + long time; public host(String name, Particle node) { this.name = name; this.node = node; + this.time = System.currentTimeMillis(); } } diff --git a/htroot/xml/webstructure.java b/htroot/xml/webstructure.java index 6d1f08994..4b5f339d3 100644 --- a/htroot/xml/webstructure.java +++ b/htroot/xml/webstructure.java @@ -41,7 +41,8 @@ public class webstructure { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { serverObjects prop = new serverObjects(); plasmaSwitchboard sb = (plasmaSwitchboard) env; - Iterator i = sb.webStructure.structureEntryIterator(); + boolean latest = ((post == null) ? false : post.containsKey("latest")); + Iterator i = sb.webStructure.structureEntryIterator(latest); int c = 0, d; plasmaWebStructure.structureEntry sentry; Map.Entry refentry; @@ -71,6 +72,7 @@ public class webstructure { } prop.put("domains", c); prop.put("maxref", plasmaWebStructure.maxref); + if (latest) sb.webStructure.joinOldNew(); // return rewrite properties return prop; diff --git a/source/de/anomic/crawler/NoticeURLImporter.java b/source/de/anomic/crawler/NoticeURLImporter.java index 36356dc1b..de1c46881 100644 --- a/source/de/anomic/crawler/NoticeURLImporter.java +++ b/source/de/anomic/crawler/NoticeURLImporter.java @@ -21,7 +21,7 @@ public class NoticeURLImporter extends AbstractImporter implements Importer { private CrawlProfile activeCrawls; private ImporterManager dbImportManager; - public NoticeURLImporter(File crawlerPath, CrawlQueues crawlQueues, CrawlProfile activeCrawls, ImporterManager dbImportManager) throws ImporterException { + public NoticeURLImporter(File crawlerPath, CrawlQueues crawlQueues, CrawlProfile activeCrawls, ImporterManager dbImportManager) { super("NURL"); this.crawlQueues = crawlQueues; this.activeCrawls = activeCrawls; diff --git a/source/de/anomic/plasma/plasmaSwitchboardQueue.java b/source/de/anomic/plasma/plasmaSwitchboardQueue.java index a8e86bdbf..24168e631 100644 --- a/source/de/anomic/plasma/plasmaSwitchboardQueue.java +++ b/source/de/anomic/plasma/plasmaSwitchboardQueue.java @@ -392,7 +392,7 @@ public class plasmaSwitchboardQueue { } public String referrerHash() { - return referrerHash; + return (referrerHash == null) ? "" : referrerHash; } public String anchorName() { diff --git a/source/de/anomic/plasma/plasmaWebStructure.java b/source/de/anomic/plasma/plasmaWebStructure.java index 9916696ac..536facae6 100644 --- a/source/de/anomic/plasma/plasmaWebStructure.java +++ b/source/de/anomic/plasma/plasmaWebStructure.java @@ -54,7 +54,7 @@ public class plasmaWebStructure { private serverLog log; private File rankingPath, structureFile; private String crlFile, crgFile; - TreeMap structure; // ',' to {}* + TreeMap structure_old, structure_new; // ',' to {}* public plasmaWebStructure(serverLog log, File rankingPath, String crlFile, String crgFile, File structureFile) { this.log = log; @@ -62,27 +62,28 @@ public class plasmaWebStructure { this.crlFile = crlFile; this.crgFile = crgFile; this.crg = new StringBuffer(maxCRGDump); - this.structure = new TreeMap(); + this.structure_old = new TreeMap(); + this.structure_new = new TreeMap(); this.structureFile = structureFile; // load web structure Map loadedStructure = (this.structureFile.exists()) ? serverFileUtils.loadHashMap(this.structureFile) : new TreeMap(); - if (loadedStructure != null) this.structure.putAll(loadedStructure); + if (loadedStructure != null) this.structure_old.putAll(loadedStructure); - // delete outdated entries in case the structure is too big - if (this.structure.size() > maxhosts) { + // delete out-dated entries in case the structure is too big + if (this.structure_old.size() > maxhosts) { // fill a set with last-modified - dates of the structure TreeSet delset = new TreeSet(); String key, value; - for (Map.Entry entry : this.structure.entrySet()) { + for (Map.Entry entry : this.structure_old.entrySet()) { key = entry.getKey(); value = entry.getValue(); delset.add(value.substring(0, 8) + key); } - int delcount = this.structure.size() - (maxhosts * 9 / 10); + int delcount = this.structure_old.size() - (maxhosts * 9 / 10); Iterator j = delset.iterator(); while ((delcount > 0) && (j.hasNext())) { - this.structure.remove(j.next().substring(8)); + this.structure_old.remove(j.next().substring(8)); delcount--; } } @@ -216,46 +217,78 @@ public class plasmaWebStructure { public Map references(String domhash) { // returns a map with a domhash(String):refcount(Integer) relation assert domhash.length() == 6; - synchronized(structure) { - SortedMap tailMap = structure.tailMap(domhash); - if ((tailMap == null) || (tailMap.isEmpty())) return new HashMap(); - String key = tailMap.firstKey(); - if (key.startsWith(domhash)) { - return refstr2map(tailMap.get(key)); - } else { - return new HashMap(); + SortedMap tailMap; + Map h = new HashMap(); + synchronized (structure_old) { + tailMap = structure_old.tailMap(domhash); + if (!tailMap.isEmpty()) { + String key = tailMap.firstKey(); + if (key.startsWith(domhash)) { + h = refstr2map(tailMap.get(key)); + } + } + } + synchronized (structure_new) { + tailMap = structure_new.tailMap(domhash); + if (!tailMap.isEmpty()) { + String key = tailMap.firstKey(); + if (key.startsWith(domhash)) { + h.putAll(refstr2map(tailMap.get(key))); + } } } + return h; } public int referencesCount(String domhash) { // returns the number of domains that are referenced by this domhash assert domhash.length() == 6 : "domhash = " + domhash; - synchronized(structure) { - SortedMap tailMap = structure.tailMap(domhash); - if ((tailMap == null) || (tailMap.isEmpty())) return 0; - String key = tailMap.firstKey(); - if (key.startsWith(domhash)) { - return refstr2count(tailMap.get(key)); - } else { - return 0; + SortedMap tailMap; + int c = 0; + synchronized (structure_old) { + tailMap = structure_old.tailMap(domhash); + if (!tailMap.isEmpty()) { + String key = tailMap.firstKey(); + if (key.startsWith(domhash)) { + c = refstr2count(tailMap.get(key)); + } + } + } + synchronized (structure_new) { + tailMap = structure_new.tailMap(domhash); + if (!tailMap.isEmpty()) { + String key = tailMap.firstKey(); + if (key.startsWith(domhash)) { + c += refstr2count(tailMap.get(key)); + } } } + return c; } public String resolveDomHash2DomString(String domhash) { // returns the domain as string, null if unknown assert domhash.length() == 6; - synchronized(structure) { - SortedMap tailMap = structure.tailMap(domhash); - if ((tailMap == null) || (tailMap.isEmpty())) return null; - String key = tailMap.firstKey(); - if (key.startsWith(domhash)) { - return key.substring(7); - } else { - return null; + SortedMap tailMap; + synchronized(structure_old) { + tailMap = structure_old.tailMap(domhash); + if (!tailMap.isEmpty()) { + String key = tailMap.firstKey(); + if (key.startsWith(domhash)) { + return key.substring(7); + } } } + synchronized(structure_new) { + tailMap = structure_new.tailMap(domhash); + if (!tailMap.isEmpty()) { + String key = tailMap.firstKey(); + if (key.startsWith(domhash)) { + return key.substring(7); + } + } + } + return null; } private void learn(yacyURL url, StringBuffer reference /*string of b64(12digits)-hashes*/) { @@ -297,15 +330,42 @@ public class plasmaWebStructure { } // store the map back to the structure - synchronized(structure) { - structure.put(domhash + "," + url.getHost(), map2refstr(refs)); + synchronized(structure_new) { + structure_new.put(domhash + "," + url.getHost(), map2refstr(refs)); + } + } + + private static final void joinStructure(TreeMap into, TreeMap from) { + for (Map.Entry e: from.entrySet()) { + if (into.containsKey(e.getKey())) { + Map s0 = refstr2map(into.get(e.getKey())); + Map s1 = refstr2map(e.getValue()); + for (Map.Entry r: s1.entrySet()) { + if (s0.containsKey(r.getKey())) { + s0.put(r.getKey(), s0.get(r.getKey()).intValue() + r.getValue().intValue()); + } else { + s0.put(r.getKey(), r.getValue().intValue()); + } + } + into.put(e.getKey(), map2refstr(s0)); + } else { + into.put(e.getKey(), e.getValue()); + } + } + } + + public void joinOldNew() { + synchronized(structure_new) { + joinStructure(this.structure_old, this.structure_new); + this.structure_new.clear(); } } public void saveWebStructure() { + joinOldNew(); try { - synchronized(structure) { - serverFileUtils.saveMap(this.structureFile, this.structure, "Web Structure Syntax: ',' to {}*"); + synchronized(structure_old) { + serverFileUtils.saveMap(this.structureFile, this.structure_old, "Web Structure Syntax: ',' to {}*"); } } catch (IOException e) { e.printStackTrace(); @@ -316,8 +376,9 @@ public class plasmaWebStructure { // find domain with most references String maxhost = null; int refsize, maxref = 0; - synchronized(structure) { - for (Map.Entry entry : structure.entrySet()) { + joinOldNew(); + synchronized(structure_new) { + for (Map.Entry entry : structure_old.entrySet()) { refsize = entry.getValue().length(); if (refsize > maxref) { maxref = refsize; @@ -328,9 +389,9 @@ public class plasmaWebStructure { return maxhost; } - public Iterator structureEntryIterator() { + public Iterator structureEntryIterator(boolean latest) { // iterates objects of type structureEntry - return new structureIterator(); + return new structureIterator(latest); } public class structureIterator implements Iterator { @@ -338,8 +399,8 @@ public class plasmaWebStructure { private Iterator> i; private structureEntry nextentry; - public structureIterator() { - i = structure.entrySet().iterator(); + public structureIterator(boolean latest) { + i = ((latest) ? structure_new : structure_old).entrySet().iterator(); next0(); }