diff --git a/build.nsi b/build.nsi old mode 100755 new mode 100644 index e56e0a736..92d567fc4 --- a/build.nsi +++ b/build.nsi @@ -1,426 +1,426 @@ -;yacy.nsi -; ---------------------------------------- -;(C) 2004-2006 by Alexander Schier -;(C) 2008-2010 by David Wieditz -;(C) 2011- by Rene Kluge -/*---------------------------------------- -MANUALS -http://nsis.sourceforge.net/Docs/ -http://nsis.sourceforge.net/Docs/Modern%20UI%202/Readme.html ----------------------------------------- -COMMAND LINE OPTIONS (case sensitive): -/S - silent install/uninstall -/D="C:\yacy" - installation folder -----------------------------------------*/ - -; ---------------------------------------- -; MODERN UI - -!include MUI2.nsh -!include x64.nsh -!include FileFunc.nsh -!include WinVer.nsh - -; ---------------------------------------- -; GENERAL - -VIProductVersion "@REPL_VERSION@.0.0" -VIAddVersionKey "ProductName" "YaCy" -VIAddVersionKey "LegalCopyright" "YaCy" -VIAddVersionKey "FileVersion" "@REPL_VERSION@" -VIAddVersionKey "FileDescription" "YaCy" -VIAddVersionKey "OriginalFilename" "yacy_v@REPL_VERSION@_@REPL_DATE@_@REPL_REVISION_NR@.exe" - -Name "YaCy @REPL_VERSION@" -OutFile "RELEASE\WINDOWS\yacy_v@REPL_VERSION@_@REPL_DATE@_@REPL_REVISION_NR@.exe" - -;default installation folder -InstallDir "$PROFILE\YaCy" -InstallDirRegKey HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "UninstallString" - -;recommend free space in GB for YaCy -!define RecommendSpace "4" - -; commands for firewall config, see http://support.microsoft.com/kb/947709/en-us -!define WinXPAddFwRulePort 'netsh firewall add allowedprogram name="YaCy" program="%SystemRoot%\System32\javaw.exe"' -!define WinXPDelFwRulePort 'netsh firewall del allowedprogram program="%SystemRoot%\System32\javaw.exe"' -!define WinVistaAddFwRulePort 'netsh advfirewall firewall add rule name="YaCy" program="%SystemRoot%\System32\javaw.exe" dir=in action=allow' -!define WinVistaDelFwRulePort 'netsh advfirewall firewall del rule name="YaCy"' -var WinAddFwRulePort -var WinDelFwRulePort - -;requested execution level on Vista / 7 -RequestExecutionLevel admin - -SetCompressor /SOLID LZMA -!insertmacro MUI_RESERVEFILE_LANGDLL ;loads faster - -; ---------------------------------------- -; JAVA VERSION -; http://www.java.com/de/download/manual.jsp BundleId +1 / +2 -; User-Agent to see the 64bit link: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; Trident/4.0) - -; at least we need Java 6 -!define JRE_VERSION6 "1.6" - -; download link Oracle Java 7 Update 4 -; 32 bit -!define JRE_32 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63691" -; 64 bit -!define JRE_64 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63692" - -;!define JRE_32 "http://yacy.berlios.de/download.php?what=jre&version=32&yacyrevnr=@REPL_REVISION_NR@" -;!define JRE_64 "http://yacy.berlios.de/download.php?what=jre&version=64&yacyrevnr=@REPL_REVISION_NR@" - -; ---------------------------------------- -; GENERAL APPEARANCE - -;BrandingText "yacy.net" - -!define MUI_ICON "RELEASE\MAIN\addon\YaCy.ico" -!define MUI_UNICON "${NSISDIR}\Contrib\Graphics\Icons\orange-uninstall.ico" - -!define MUI_WELCOMEFINISHPAGE_BITMAP "RELEASE\MAIN\addon\installer\network.bmp" - -!define MUI_HEADERIMAGE -!define MUI_HEADERIMAGE_BITMAP "RELEASE\MAIN\addon\installer\logo.bmp" -!define MUI_HEADERIMAGE_BITMAP_NOSTRETCH - -!define MUI_ABORTWARNING ;display warning before aborting installation - -; ---------------------------------------- -; INSTALLER PAGES - -!insertmacro MUI_PAGE_WELCOME - -!insertmacro MUI_PAGE_LICENSE gpl.txt - -!define MUI_COMPONENTSPAGE_NODESC -!insertmacro MUI_PAGE_COMPONENTS -ComponentText "YaCy v@REPL_VERSION@ (Build @REPL_DATE@)" - -!define MUI_PAGE_CUSTOMFUNCTION_LEAVE CheckDriveSpace -!insertmacro MUI_PAGE_DIRECTORY - -!insertmacro MUI_PAGE_INSTFILES - -!define MUI_PAGE_CUSTOMFUNCTION_SHOW SHOW_PageFinish_custom -!define MUI_FINISHPAGE_SHOWREADME http://www.yacy-websuche.de/wiki/index.php/InstallerFinished -!define MUI_FINISHPAGE_SHOWREADME_TEXT $(finishPage) -!insertmacro MUI_PAGE_FINISH - -; ---------------------------------------- -; UNINSTALLER PAGES - -!insertmacro MUI_UNPAGE_CONFIRM - -!insertmacro MUI_UNPAGE_INSTFILES - -; ---------------------------------------- -; LANGUAGES - -!insertmacro MUI_LANGUAGE "English" -!insertmacro MUI_LANGUAGE "French" -!insertmacro MUI_LANGUAGE "German" - -LangString stillRunning ${LANG_ENGLISH} "YaCy is still active. Please stop YaCy first." -LangString keepData 0 "Do you want to keep the data?" -LangString noAdminForJava 0 "You need Administrator privileges to install Java. Java will now be downloaded to the shared documents folder. YaCy won't run without Java." -LangString finishPage 0 "Show how to configure the Windows Firewall for YaCy." -LangString yacyNoHd 0 "YaCy should be installed on a hard disk. Continue with selected folder?" -LangString yacyNeedSpace 0 "We recommend ${RecommendSpace} GB free space for YaCy. There are only $TempDriveFree GB left. Continue anyway?" -LangString yacyNeedOs 0 "Please run Windows 2000 or better (e.g. Windows XP, Vista or Windows 7) for YaCy." - -LangString stillRunning ${LANG_FRENCH} "YaCy is still active. Please stop YaCy first." -LangString keepData 0 "Do you want to keep the data?" -LangString noAdminForJava 0 "You need Administrator privileges to install Java. Java will now be downloaded to the shared documents folder. YaCy won't run without Java." -LangString finishPage 0 "Show how to configure the Windows Firewall for YaCy." -LangString yacyNoHd 0 "YaCy should be installed on a hard disk. Continue with selected folder?" -LangString yacyNeedSpace 0 "We recommend ${RecommendSpace} GB free space for YaCy. There are only $TempDriveFree GB left. Continue anyway?" -LangString yacyNeedOs 0 "Please run Windows 2000 or better (e.g. Windows XP, Vista or Windows 7) for YaCy." - -LangString stillRunning ${LANG_GERMAN} "YaCy ist noch aktiv. Bitte beenden Sie YaCy." -LangString keepData 0 "Moechten Sie die Daten behalten?" -LangString noAdminForJava 0 "Sie benoetigen Administrator-Rechte um Java zu installieren. Java wird nun in 'Gemeinsame Dokumente' gespeichert. YaCy benoetigt Java zur Ausfuehrung." -LangString finishPage 0 "Zeige die Windows Firewall Konfiguration fuer YaCy." -LangString yacyNoHd 0 "YaCy sollte auf einer Festplatte installiert werden. Soll der gewaehlte Ordner trotzdem verwendet werden?" -LangString yacyNeedSpace 0 "Wir empfehlen ${RecommendSpace} GB fuer YaCy. Es sind noch $TempDriveFree GB frei. Trotzdem fortfahren?" -LangString yacyNeedOs 0 "YaCy benoetigt Windows 2000 oder besser (z.B. Windows XP, Vista oder Windows 7)." - -; ---------------------------------------- -; INSTALLABLE MODULES - -;InstType "Normal" - -Section "YaCy" - SectionIn 1 RO - SetShellVarContext current ; use system variables (folders) for current user - RMDir /r "$INSTDIR\lib" ;remove old libraries in case of update - RMDir /r "$SMPROGRAMS\YaCy" ;clear old shortcuts - Delete "$QUICKLAUNCH\YaCy-Search.lnk" ;old - Delete "$DESKTOP\YaCy-Search.lnk" ;old - Delete "$SMSTARTUP\start YaCy (no console).lnk" ;old - - SetOutPath $INSTDIR - ;set noindex attribute for windows indexing service - nsExec::Exec 'attrib +I "$INSTDIR"' - nsExec::Exec 'attrib +I "$INSTDIR\*" /S /D' - - File /r "RELEASE\MAIN\*" - - WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "DisplayName" "YaCy" - WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "UninstallString" '"$INSTDIR\uninstall.exe"' - WriteUninstaller "uninstall.exe" -SectionEnd - -Section "Java (Oracle JRE)" Sec_Java_id - SectionIn 1 RO - SetShellVarContext current - Call GetJRE -SectionEnd - -Section "Start Menu Group" - SectionIn 1 - SetShellVarContext current - CreateDirectory "$SMPROGRAMS\YaCy" - CreateShortCut "$SMPROGRAMS\YaCy\YaCy.lnk" "$INSTDIR\startYACY.bat" "" "$INSTDIR\addon\YaCy.ico" "" SW_SHOWMINIMIZED - CreateShortCut "$SMPROGRAMS\YaCy\Readme.lnk" "$INSTDIR\readme.txt" - CreateShortCut "$SMPROGRAMS\YaCy\Uninstall.lnk" "$INSTDIR\Uninstall.exe" -SectionEnd - -Section "Desktop Icon" - SectionIn 1 - SetShellVarContext current - CreateShortCut "$DESKTOP\YaCy.lnk" "$INSTDIR\startYACY.bat" "" "$INSTDIR\addon\YaCy.ico" "" SW_SHOWMINIMIZED -SectionEnd - -Section "Configure Firewall" Sec_Firewall_id - SectionIn 1 - SetShellVarContext current - call OpenFirewall -SectionEnd - -/* -Section "YaCy in Startup" - SetShellVarContext current - CreateShortCut "$SMSTARTUP\YaCy.lnk" "$INSTDIR\startYACY.bat" "" "$INSTDIR\addon\YaCy.ico" "" SW_SHOWMINIMIZED -SectionEnd -*/ - -; ---------------------------------------- -; UNINSTALLER - -Section "Uninstall" - ClearErrors - Delete "$INSTDIR\DATA\yacy.running" - IfErrors 0 uninstall - MessageBox MB_ICONSTOP "$(stillRunning)" /SD IDOK - Goto nouninstall - - uninstall: - Call un.CloseFirewall - SetShellVarContext current - - RMDir /r "$INSTDIR\addon" - RMDir /r "$INSTDIR\bin" - RMDir /r "$INSTDIR\classes" - RMDir /r "$INSTDIR\defaults" - RMDir /r "$INSTDIR\htroot" - RMDir /r "$INSTDIR\lib" - RMDir /r "$INSTDIR\libx" - RMDir /r "$INSTDIR\locales" - RMDir /r "$INSTDIR\ranking" - RMDir /r "$INSTDIR\skins" - RMDir /r "$INSTDIR\source" - Delete "$INSTDIR\*.*" - - MessageBox MB_YESNO|MB_ICONQUESTION "$(keepData)" /SD IDYES IDYES keepdata - - ;delete all - RMDir /r "$INSTDIR" - - ;or jump to this - keepdata: - RMDir /r "$SMPROGRAMS\YaCy" - Delete "$DESKTOP\YaCy.lnk" - Delete "$SMSTARTUP\YaCy.lnk" - - DeleteRegKey HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" - nouninstall: -SectionEnd - -; ---------------------------------------- -; FUNCTIONS - -Function .onInit - ; check Windows-Version, need Win 2000 or higher - ${If} ${AtMostWinME} - MessageBox MB_ICONSTOP "$(yacyNeedOs)" - Abort - ${EndIf} - - ; init of JRE section - ; detect JRE first - var /global InstalledJREVersion - ${If} ${RunningX64} - SetRegView 64 - ${EndIf} - ReadRegStr $InstalledJREVersion HKLM "SOFTWARE\JavaSoft\Java Runtime Environment" "CurrentVersion" - ; if right JRE already installed hide and deselect JRE section - ${If} $InstalledJREVersion = ${JRE_VERSION6} - SectionSetText ${Sec_Java_id} "" - SectionSetFlags ${Sec_Java_id} 0 - ${EndIf} - - ; init of Firewall section, only valid for WindowsXP SP2/SP3 and Vista/Win 7 with Admin - var /global FirewallServiceStart - IntOp $FirewallServiceStart 3 + 0 - - ${If} ${IsWinVista} - ${OrIf} ${IsWin7} - StrCpy $WinAddFwRulePort '${WinVistaAddFwRulePort}' - StrCpy $WinDelFwRulePort '${WinVistaDelFwRulePort}' - ReadRegDWORD $FirewallServiceStart HKLM "SYSTEM\CurrentControlSet\services\MpsSvc" "Start" - ${EndIf} - - ${If} ${IsWinXP} - ${AndIf} ${AtLeastServicePack} 2 - StrCpy $WinAddFwRulePort '${WinXPAddFwRulePort}' - StrCpy $WinDelFwRulePort '${WinXPDelFwRulePort}' - ReadRegDWORD $FirewallServiceStart HKLM "SYSTEM\CurrentControlSet\services\SharedAccess" "Start" - ${EndIf} - - ;need Admin for firewall-config - ${IfNot} $0 = "Admin" - IntOp $FirewallServiceStart 3 + 0 - ${EndIf} - - ; hide and deselect Firewall if no proper configuration - ${If} $FirewallServiceStart > 2 - SectionSetText ${Sec_Firewall_id} "" - SectionSetFlags ${Sec_Firewall_id} 0 - ${EndIf} -FunctionEnd - -Function GetJRE -; based on http://nsis.sourceforge.net/Simple_Java_Runtime_Download_Script - ${If} ${RunningX64} - StrCpy $3 ${JRE_64} - ${Else} - StrCpy $3 ${JRE_32} - ${EndIf} - - ;check if admin before download, advise if non - userInfo::getAccountType - Pop $0 - - ${IfNot} $0 = "Admin" - MessageBox MB_ICONEXCLAMATION "$(noAdminForJava)" /SD IDOK - ${EndIf} - - SetShellVarContext all - StrCpy $2 "$DOCUMENTS\Java Runtime (install for YaCy).exe" - SetShellVarContext current - nsisdl::download /TIMEOUT=30000 $3 $2 - Pop $R0 ;Get the return value - - ${IfNot} $R0 = "success" - MessageBox MB_OK "Download failed: $R0" /SD IDOK - Return - ${EndIf} - - ${If} $0 == "Admin" - ExecWait "$2 /s" - Delete $2 - ${Else} - CreateShortCut "$DESKTOP\Install Java for YaCy.lnk" "$2" - ${EndIf} -FunctionEnd - -Function CheckDriveSpace - var /global RootFolder - var /global TempDriveFree - var /global RootFolderType - - ; if "\\Folder" it's a Network-Folder - StrCpy $RootFolder $InstDir 2 - StrCmp $RootFolder "\\" NetworkFolder Driveletter - - Networkfolder: - ; prepare String for DriveSpace - ${GetRoot} $RootFolder $InstDir - goto NoHDD - - ; now check drive-letters - Driveletter: - StrCpy $RootFolder $InstDir 3 - - ; prepare for {GetDrives-Loop} - StrCpy $RootFolderType "invalid" - ${GetDrives} "ALL" "CheckDriveType" - - ; jump if error - StrCmp $RootFolderType "invalid" CheckSpace - - ; jump if HDD - StrCmp $RootFolderType "HDD" CheckSpace - - NoHDD: - ; stay on folder-selection if user wants to give another folder, else check free space - MessageBox MB_ICONEXCLAMATION|MB_YESNO "$(yacyNoHd)" IDYES NextPage - Abort - - CheckSpace: - - ClearErrors - ${DriveSpace} $RootFolder "/D=F /S=G" $TempDriveFree - ; if DriveSpace fails for any reason -> jump ahead - IfErrors NextPage - - ${If} $TempDriveFree < ${RecommendSpace} - MessageBox MB_ICONEXCLAMATION|MB_YESNO "$(yacyNeedSpace)" IDYES NextPage - Abort - ${EndIf} - - NextPage: -FunctionEnd - -Function CheckDriveType -; based on http://nsis.sourceforge.net/GetDrives - ${If} $9 == $RootFolder - StrCpy $RootFolderType $8 - StrCpy $0 StopGetDrives - ${EndIf} - Push $0 -FunctionEnd - -Function OpenFirewall - var /global ExecErrorCode - ; run netsh - nsExec::ExecToStack '$WinAddFwRulePort' - pop $ExecErrorCode - ; if run without error register for uninstall and clear finish page - ${If} $ExecErrorCode = "0" - WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "DelFwRulePort" '$WinDelFwRulePort' - IntOp $FirewallServiceStart 0 + 0 - ${Else} - IntOp $FirewallServiceStart 3 + 0 - ${EndIf} -FunctionEnd - -Function SHOW_PageFinish_custom - ; hide and disable firewall info from wiki if firewall is open - ${If} $FirewallServiceStart = 0 - SendMessage $mui.FinishPage.ShowReadme ${BM_SETCHECK} 0 0 - ShowWindow $mui.FinishPage.ShowReadme ${SW_HIDE} - ${EndIf} -FunctionEnd - -Function un.CloseFirewall - ; get string for closing port from registy - ReadRegStr '$WinDelFwRulePort' HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "DelFwRulePort" - ; if found > run netsh to close port - ${IfNot} '$WinDelFwRulePort' == '' - nsExec::ExecToStack '$WinDelFwRulePort' - ${EndIf} -FunctionEnd +;yacy.nsi +; ---------------------------------------- +;(C) 2004-2006 by Alexander Schier +;(C) 2008-2010 by David Wieditz +;(C) 2011- by Rene Kluge +/*---------------------------------------- +MANUALS +http://nsis.sourceforge.net/Docs/ +http://nsis.sourceforge.net/Docs/Modern%20UI%202/Readme.html +---------------------------------------- +COMMAND LINE OPTIONS (case sensitive): +/S - silent install/uninstall +/D="C:\yacy" - installation folder +----------------------------------------*/ + +; ---------------------------------------- +; MODERN UI + +!include MUI2.nsh +!include x64.nsh +!include FileFunc.nsh +!include WinVer.nsh + +; ---------------------------------------- +; GENERAL + +VIProductVersion "@REPL_VERSION@.0.0" +VIAddVersionKey "ProductName" "YaCy" +VIAddVersionKey "LegalCopyright" "YaCy" +VIAddVersionKey "FileVersion" "@REPL_VERSION@" +VIAddVersionKey "FileDescription" "YaCy" +VIAddVersionKey "OriginalFilename" "yacy_v@REPL_VERSION@_@REPL_DATE@_@REPL_REVISION_NR@.exe" + +Name "YaCy @REPL_VERSION@" +OutFile "RELEASE\WINDOWS\yacy_v@REPL_VERSION@_@REPL_DATE@_@REPL_REVISION_NR@.exe" + +;default installation folder +InstallDir "$PROFILE\YaCy" +InstallDirRegKey HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "UninstallString" + +;recommend free space in GB for YaCy +!define RecommendSpace "4" + +; commands for firewall config, see http://support.microsoft.com/kb/947709/en-us +!define WinXPAddFwRulePort 'netsh firewall add allowedprogram name="YaCy" program="%SystemRoot%\System32\javaw.exe"' +!define WinXPDelFwRulePort 'netsh firewall del allowedprogram program="%SystemRoot%\System32\javaw.exe"' +!define WinVistaAddFwRulePort 'netsh advfirewall firewall add rule name="YaCy" program="%SystemRoot%\System32\javaw.exe" dir=in action=allow' +!define WinVistaDelFwRulePort 'netsh advfirewall firewall del rule name="YaCy"' +var WinAddFwRulePort +var WinDelFwRulePort + +;requested execution level on Vista / 7 +RequestExecutionLevel admin + +SetCompressor /SOLID LZMA +!insertmacro MUI_RESERVEFILE_LANGDLL ;loads faster + +; ---------------------------------------- +; JAVA VERSION +; http://www.java.com/de/download/manual.jsp BundleId +1 / +2 +; User-Agent to see the 64bit link: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; Trident/4.0) + +; at least we need Java 6 +!define JRE_VERSION6 "1.6" + +; download link Oracle Java 7 Update 4 +; 32 bit +!define JRE_32 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63691" +; 64 bit +!define JRE_64 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63692" + +;!define JRE_32 "http://yacy.berlios.de/download.php?what=jre&version=32&yacyrevnr=@REPL_REVISION_NR@" +;!define JRE_64 "http://yacy.berlios.de/download.php?what=jre&version=64&yacyrevnr=@REPL_REVISION_NR@" + +; ---------------------------------------- +; GENERAL APPEARANCE + +;BrandingText "yacy.net" + +!define MUI_ICON "RELEASE\MAIN\addon\YaCy.ico" +!define MUI_UNICON "${NSISDIR}\Contrib\Graphics\Icons\orange-uninstall.ico" + +!define MUI_WELCOMEFINISHPAGE_BITMAP "RELEASE\MAIN\addon\installer\network.bmp" + +!define MUI_HEADERIMAGE +!define MUI_HEADERIMAGE_BITMAP "RELEASE\MAIN\addon\installer\logo.bmp" +!define MUI_HEADERIMAGE_BITMAP_NOSTRETCH + +!define MUI_ABORTWARNING ;display warning before aborting installation + +; ---------------------------------------- +; INSTALLER PAGES + +!insertmacro MUI_PAGE_WELCOME + +!insertmacro MUI_PAGE_LICENSE gpl.txt + +!define MUI_COMPONENTSPAGE_NODESC +!insertmacro MUI_PAGE_COMPONENTS +ComponentText "YaCy v@REPL_VERSION@ (Build @REPL_DATE@)" + +!define MUI_PAGE_CUSTOMFUNCTION_LEAVE CheckDriveSpace +!insertmacro MUI_PAGE_DIRECTORY + +!insertmacro MUI_PAGE_INSTFILES + +!define MUI_PAGE_CUSTOMFUNCTION_SHOW SHOW_PageFinish_custom +!define MUI_FINISHPAGE_SHOWREADME http://www.yacy-websuche.de/wiki/index.php/InstallerFinished +!define MUI_FINISHPAGE_SHOWREADME_TEXT $(finishPage) +!insertmacro MUI_PAGE_FINISH + +; ---------------------------------------- +; UNINSTALLER PAGES + +!insertmacro MUI_UNPAGE_CONFIRM + +!insertmacro MUI_UNPAGE_INSTFILES + +; ---------------------------------------- +; LANGUAGES + +!insertmacro MUI_LANGUAGE "English" +!insertmacro MUI_LANGUAGE "French" +!insertmacro MUI_LANGUAGE "German" + +LangString stillRunning ${LANG_ENGLISH} "YaCy is still active. Please stop YaCy first." +LangString keepData 0 "Do you want to keep the data?" +LangString noAdminForJava 0 "You need Administrator privileges to install Java. Java will now be downloaded to the shared documents folder. YaCy won't run without Java." +LangString finishPage 0 "Show how to configure the Windows Firewall for YaCy." +LangString yacyNoHd 0 "YaCy should be installed on a hard disk. Continue with selected folder?" +LangString yacyNeedSpace 0 "We recommend ${RecommendSpace} GB free space for YaCy. There are only $TempDriveFree GB left. Continue anyway?" +LangString yacyNeedOs 0 "Please run Windows 2000 or better (e.g. Windows XP, Vista or Windows 7) for YaCy." + +LangString stillRunning ${LANG_FRENCH} "YaCy is still active. Please stop YaCy first." +LangString keepData 0 "Do you want to keep the data?" +LangString noAdminForJava 0 "You need Administrator privileges to install Java. Java will now be downloaded to the shared documents folder. YaCy won't run without Java." +LangString finishPage 0 "Show how to configure the Windows Firewall for YaCy." +LangString yacyNoHd 0 "YaCy should be installed on a hard disk. Continue with selected folder?" +LangString yacyNeedSpace 0 "We recommend ${RecommendSpace} GB free space for YaCy. There are only $TempDriveFree GB left. Continue anyway?" +LangString yacyNeedOs 0 "Please run Windows 2000 or better (e.g. Windows XP, Vista or Windows 7) for YaCy." + +LangString stillRunning ${LANG_GERMAN} "YaCy ist noch aktiv. Bitte beenden Sie YaCy." +LangString keepData 0 "Moechten Sie die Daten behalten?" +LangString noAdminForJava 0 "Sie benoetigen Administrator-Rechte um Java zu installieren. Java wird nun in 'Gemeinsame Dokumente' gespeichert. YaCy benoetigt Java zur Ausfuehrung." +LangString finishPage 0 "Zeige die Windows Firewall Konfiguration fuer YaCy." +LangString yacyNoHd 0 "YaCy sollte auf einer Festplatte installiert werden. Soll der gewaehlte Ordner trotzdem verwendet werden?" +LangString yacyNeedSpace 0 "Wir empfehlen ${RecommendSpace} GB fuer YaCy. Es sind noch $TempDriveFree GB frei. Trotzdem fortfahren?" +LangString yacyNeedOs 0 "YaCy benoetigt Windows 2000 oder besser (z.B. Windows XP, Vista oder Windows 7)." + +; ---------------------------------------- +; INSTALLABLE MODULES + +;InstType "Normal" + +Section "YaCy" + SectionIn 1 RO + SetShellVarContext current ; use system variables (folders) for current user + RMDir /r "$INSTDIR\lib" ;remove old libraries in case of update + RMDir /r "$SMPROGRAMS\YaCy" ;clear old shortcuts + Delete "$QUICKLAUNCH\YaCy-Search.lnk" ;old + Delete "$DESKTOP\YaCy-Search.lnk" ;old + Delete "$SMSTARTUP\start YaCy (no console).lnk" ;old + + SetOutPath $INSTDIR + ;set noindex attribute for windows indexing service + nsExec::Exec 'attrib +I "$INSTDIR"' + nsExec::Exec 'attrib +I "$INSTDIR\*" /S /D' + + File /r /x *.sh "RELEASE\MAIN\*" + + WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "DisplayName" "YaCy" + WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "UninstallString" '"$INSTDIR\uninstall.exe"' + WriteUninstaller "uninstall.exe" +SectionEnd + +Section "Java (Oracle JRE)" Sec_Java_id + SectionIn 1 RO + SetShellVarContext current + Call GetJRE +SectionEnd + +Section "Start Menu Group" + SectionIn 1 + SetShellVarContext current + CreateDirectory "$SMPROGRAMS\YaCy" + CreateShortCut "$SMPROGRAMS\YaCy\YaCy.lnk" "$INSTDIR\startYACY.bat" "" "$INSTDIR\addon\YaCy.ico" "" SW_SHOWMINIMIZED + CreateShortCut "$SMPROGRAMS\YaCy\Readme.lnk" "$INSTDIR\readme.txt" + CreateShortCut "$SMPROGRAMS\YaCy\Uninstall.lnk" "$INSTDIR\Uninstall.exe" +SectionEnd + +Section "Desktop Icon" + SectionIn 1 + SetShellVarContext current + CreateShortCut "$DESKTOP\YaCy.lnk" "$INSTDIR\startYACY.bat" "" "$INSTDIR\addon\YaCy.ico" "" SW_SHOWMINIMIZED +SectionEnd + +Section "Configure Firewall" Sec_Firewall_id + SectionIn 1 + SetShellVarContext current + call OpenFirewall +SectionEnd + +/* +Section "YaCy in Startup" + SetShellVarContext current + CreateShortCut "$SMSTARTUP\YaCy.lnk" "$INSTDIR\startYACY.bat" "" "$INSTDIR\addon\YaCy.ico" "" SW_SHOWMINIMIZED +SectionEnd +*/ + +; ---------------------------------------- +; UNINSTALLER + +Section "Uninstall" + ClearErrors + Delete "$INSTDIR\DATA\yacy.running" + IfErrors 0 uninstall + MessageBox MB_ICONSTOP "$(stillRunning)" /SD IDOK + Goto nouninstall + + uninstall: + Call un.CloseFirewall + SetShellVarContext current + + RMDir /r "$INSTDIR\addon" + RMDir /r "$INSTDIR\bin" + RMDir /r "$INSTDIR\classes" + RMDir /r "$INSTDIR\defaults" + RMDir /r "$INSTDIR\htroot" + RMDir /r "$INSTDIR\lib" + RMDir /r "$INSTDIR\libx" + RMDir /r "$INSTDIR\locales" + RMDir /r "$INSTDIR\ranking" + RMDir /r "$INSTDIR\skins" + RMDir /r "$INSTDIR\source" + Delete "$INSTDIR\*.*" + + MessageBox MB_YESNO|MB_ICONQUESTION "$(keepData)" /SD IDYES IDYES keepdata + + ;delete all + RMDir /r "$INSTDIR" + + ;or jump to this + keepdata: + RMDir /r "$SMPROGRAMS\YaCy" + Delete "$DESKTOP\YaCy.lnk" + Delete "$SMSTARTUP\YaCy.lnk" + + DeleteRegKey HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" + nouninstall: +SectionEnd + +; ---------------------------------------- +; FUNCTIONS + +Function .onInit + ; check Windows-Version, need Win 2000 or higher + ${If} ${AtMostWinME} + MessageBox MB_ICONSTOP "$(yacyNeedOs)" + Abort + ${EndIf} + + ; init of JRE section + ; detect JRE first + var /global InstalledJREVersion + ${If} ${RunningX64} + SetRegView 64 + ${EndIf} + ReadRegStr $InstalledJREVersion HKLM "SOFTWARE\JavaSoft\Java Runtime Environment" "CurrentVersion" + ; if right JRE already installed hide and deselect JRE section + ${If} $InstalledJREVersion = ${JRE_VERSION6} + SectionSetText ${Sec_Java_id} "" + SectionSetFlags ${Sec_Java_id} 0 + ${EndIf} + + ; init of Firewall section, only valid for WindowsXP SP2/SP3 and Vista/Win 7 with Admin + var /global FirewallServiceStart + IntOp $FirewallServiceStart 3 + 0 + + ${If} ${IsWinVista} + ${OrIf} ${IsWin7} + StrCpy $WinAddFwRulePort '${WinVistaAddFwRulePort}' + StrCpy $WinDelFwRulePort '${WinVistaDelFwRulePort}' + ReadRegDWORD $FirewallServiceStart HKLM "SYSTEM\CurrentControlSet\services\MpsSvc" "Start" + ${EndIf} + + ${If} ${IsWinXP} + ${AndIf} ${AtLeastServicePack} 2 + StrCpy $WinAddFwRulePort '${WinXPAddFwRulePort}' + StrCpy $WinDelFwRulePort '${WinXPDelFwRulePort}' + ReadRegDWORD $FirewallServiceStart HKLM "SYSTEM\CurrentControlSet\services\SharedAccess" "Start" + ${EndIf} + + ;need Admin for firewall-config + ${IfNot} $0 = "Admin" + IntOp $FirewallServiceStart 3 + 0 + ${EndIf} + + ; hide and deselect Firewall if no proper configuration + ${If} $FirewallServiceStart > 2 + SectionSetText ${Sec_Firewall_id} "" + SectionSetFlags ${Sec_Firewall_id} 0 + ${EndIf} +FunctionEnd + +Function GetJRE +; based on http://nsis.sourceforge.net/Simple_Java_Runtime_Download_Script + ${If} ${RunningX64} + StrCpy $3 ${JRE_64} + ${Else} + StrCpy $3 ${JRE_32} + ${EndIf} + + ;check if admin before download, advise if non + userInfo::getAccountType + Pop $0 + + ${IfNot} $0 = "Admin" + MessageBox MB_ICONEXCLAMATION "$(noAdminForJava)" /SD IDOK + ${EndIf} + + SetShellVarContext all + StrCpy $2 "$DOCUMENTS\Java Runtime (install for YaCy).exe" + SetShellVarContext current + nsisdl::download /TIMEOUT=30000 $3 $2 + Pop $R0 ;Get the return value + + ${IfNot} $R0 = "success" + MessageBox MB_OK "Download failed: $R0" /SD IDOK + Return + ${EndIf} + + ${If} $0 == "Admin" + ExecWait "$2 /s" + Delete $2 + ${Else} + CreateShortCut "$DESKTOP\Install Java for YaCy.lnk" "$2" + ${EndIf} +FunctionEnd + +Function CheckDriveSpace + var /global RootFolder + var /global TempDriveFree + var /global RootFolderType + + ; if "\\Folder" it's a Network-Folder + StrCpy $RootFolder $InstDir 2 + StrCmp $RootFolder "\\" NetworkFolder Driveletter + + Networkfolder: + ; prepare String for DriveSpace + ${GetRoot} $RootFolder $InstDir + goto NoHDD + + ; now check drive-letters + Driveletter: + StrCpy $RootFolder $InstDir 3 + + ; prepare for {GetDrives-Loop} + StrCpy $RootFolderType "invalid" + ${GetDrives} "ALL" "CheckDriveType" + + ; jump if error + StrCmp $RootFolderType "invalid" CheckSpace + + ; jump if HDD + StrCmp $RootFolderType "HDD" CheckSpace + + NoHDD: + ; stay on folder-selection if user wants to give another folder, else check free space + MessageBox MB_ICONEXCLAMATION|MB_YESNO "$(yacyNoHd)" IDYES NextPage + Abort + + CheckSpace: + + ClearErrors + ${DriveSpace} $RootFolder "/D=F /S=G" $TempDriveFree + ; if DriveSpace fails for any reason -> jump ahead + IfErrors NextPage + + ${If} $TempDriveFree < ${RecommendSpace} + MessageBox MB_ICONEXCLAMATION|MB_YESNO "$(yacyNeedSpace)" IDYES NextPage + Abort + ${EndIf} + + NextPage: +FunctionEnd + +Function CheckDriveType +; based on http://nsis.sourceforge.net/GetDrives + ${If} $9 == $RootFolder + StrCpy $RootFolderType $8 + StrCpy $0 StopGetDrives + ${EndIf} + Push $0 +FunctionEnd + +Function OpenFirewall + var /global ExecErrorCode + ; run netsh + nsExec::ExecToStack '$WinAddFwRulePort' + pop $ExecErrorCode + ; if run without error register for uninstall and clear finish page + ${If} $ExecErrorCode = "0" + WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "DelFwRulePort" '$WinDelFwRulePort' + IntOp $FirewallServiceStart 0 + 0 + ${Else} + IntOp $FirewallServiceStart 3 + 0 + ${EndIf} +FunctionEnd + +Function SHOW_PageFinish_custom + ; hide and disable firewall info from wiki if firewall is open + ${If} $FirewallServiceStart = 0 + SendMessage $mui.FinishPage.ShowReadme ${BM_SETCHECK} 0 0 + ShowWindow $mui.FinishPage.ShowReadme ${SW_HIDE} + ${EndIf} +FunctionEnd + +Function un.CloseFirewall + ; get string for closing port from registy + ReadRegStr '$WinDelFwRulePort' HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\YaCy" "DelFwRulePort" + ; if found > run netsh to close port + ${IfNot} '$WinDelFwRulePort' == '' + nsExec::ExecToStack '$WinDelFwRulePort' + ${EndIf} +FunctionEnd diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index d4215a3c3..327a4d6d5 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -1,699 +1,698 @@ -// Blacklist_p.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://www.anomic.de -// Frankfurt, Germany, 2004 -// -// This File is contributed by Alexander Schier -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -// You must compile this file with -// javac -classpath .:../classes Blacklist_p.java -// if the shell's current path is HTROOT - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.MalformedURLException; -import java.util.Arrays; -import java.util.List; - -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.data.ListManager; -import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; -import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.FileUtils; -import net.yacy.repository.Blacklist; -import net.yacy.repository.Blacklist.BlacklistType; -import net.yacy.search.Switchboard; -import net.yacy.search.query.SearchEventCache; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class Blacklist_p { - private final static String EDIT = "edit_"; - private final static String DISABLED = "disabled_"; - private final static String BLACKLIST = "blackLists_"; - private final static String BLACKLIST_MOVE = "blackListsMove_"; - private final static String BLACKLIST_SHARED = "BlackLists.Shared"; - - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - - // initialize the list manager - ListManager.switchboard = (Switchboard) env; - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS")); - - // load all blacklist files located in the directory - List dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); - - String blacklistToUse = null; - final serverObjects prop = new serverObjects(); - prop.putHTML("blacklistEngine", Blacklist.getEngineInfo()); - - // do all post operations - if (post != null) { - - final String action = post.get("action", ""); - - if(post.containsKey("testList")) { - prop.put("testlist", "1"); - String urlstring = post.get("testurl", ""); - if (!urlstring.startsWith("http://") && - !urlstring.startsWith("https://") && - !urlstring.startsWith("ftp://") && - !urlstring.startsWith("smb://") && - !urlstring.startsWith("file://")) { - urlstring = "http://"+urlstring; - } - DigestURI testurl; - try { - testurl = new DigestURI(urlstring); - } catch (final MalformedURLException e) { - testurl = null; - } - if(testurl != null) { - prop.putHTML("testlist_url",testurl.toString()); - if (Switchboard.urlBlacklist.isListed(BlacklistType.CRAWLER, testurl)) { - prop.put("testlist_listedincrawler", "1"); - } - if (Switchboard.urlBlacklist.isListed(BlacklistType.DHT, testurl)) { - prop.put("testlist_listedindht", "1"); - } - if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, testurl)) { - prop.put("testlist_listedinnews", "1"); - } - if (Switchboard.urlBlacklist.isListed(BlacklistType.PROXY, testurl)) { - prop.put("testlist_listedinproxy", "1"); - } - if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, testurl)) { - prop.put("testlist_listedinsearch", "1"); - } - if (Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS, testurl)) { - prop.put("testlist_listedinsurftips", "1"); - } - } else { - prop.put("testlist_url","not valid"); - } - } - if (post.containsKey("selectList")) { - blacklistToUse = post.get("selectedListName"); - if (blacklistToUse != null && blacklistToUse.isEmpty()) { - blacklistToUse = null; - } - } - if (post.containsKey("createNewList")) { - /* =========================================================== - * Creation of a new blacklist - * =========================================================== */ - - blacklistToUse = post.get("newListName", "").trim(); - if (blacklistToUse.isEmpty()) { - prop.put("LOCATION",""); - return prop; - } - - // Check if blacklist name only consists of "legal" characters. - // This is mainly done to prevent files from being written to other directories - // than the LISTS directory. - if (!blacklistToUse.matches("^[\\p{L}\\d\\+\\-_]+[\\p{L}\\d\\+\\-_.]*(\\.black){0,1}$")) { - prop.put("error", 1); - prop.putHTML("error_name", blacklistToUse); - blacklistToUse = null; - } else { - - if (!blacklistToUse.endsWith(".black")) { - blacklistToUse += ".black"; - } - - if (!dirlist.contains(blacklistToUse)) { - try { - final File newFile = new File(ListManager.listsPath, blacklistToUse); - newFile.createNewFile(); - - // share the newly created blacklist - ListManager.updateListSet(BLACKLIST_SHARED, blacklistToUse); - - // activate it for all known blacklist types - for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - ListManager.updateListSet(supportedBlacklistType + ".BlackLists", blacklistToUse); - } - } catch (final IOException e) {/* */} - } else { - prop.put("error", 2); - prop.putHTML("error_name", blacklistToUse); - blacklistToUse = null; - } - - // reload Blacklists - dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); - } - - } else if (post.containsKey("deleteList")) { - /* =========================================================== - * Delete a blacklist - * =========================================================== */ - - blacklistToUse = post.get("selectedListName"); - if (blacklistToUse == null || blacklistToUse.isEmpty()) { - prop.put("LOCATION",""); - return prop; - } - - final File blackListFile = new File(ListManager.listsPath, blacklistToUse); - if(!blackListFile.delete()) { - Log.logWarning("Blacklist", "file "+ blackListFile +" could not be deleted!"); - } - - for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); - } - - // remove it from the shared list - ListManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse); - blacklistToUse = null; - - // reload Blacklists - dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); - - } else if (post.containsKey("activateList")) { - - /* =========================================================== - * Activate/Deactivate a blacklist - * =========================================================== */ - - blacklistToUse = post.get("selectedListName", "").trim(); - if (blacklistToUse == null || blacklistToUse.isEmpty()) { - prop.put("LOCATION", ""); - return prop; - } - - for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - if (post.containsKey("activateList4" + supportedBlacklistType)) { - ListManager.updateListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); - } else { - ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); - } - } - - Switchboard.urlBlacklist.clear(); - ListManager.reloadBlacklists(); - - } else if (post.containsKey("shareList")) { - - /* =========================================================== - * Share a blacklist - * =========================================================== */ - - blacklistToUse = post.get("selectedListName", "").trim(); - if (blacklistToUse == null || blacklistToUse.isEmpty()) { - prop.put("LOCATION", ""); - return prop; - } - - if (ListManager.listSetContains(BLACKLIST_SHARED, blacklistToUse)) { - // Remove from shared BlackLists - ListManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse); - } else { // inactive list -> enable - ListManager.updateListSet(BLACKLIST_SHARED, blacklistToUse); - } - } else if ("deleteBlacklistEntry".equals(action)) { - - /* =========================================================== - * Delete an entry from a blacklist - * =========================================================== */ - - blacklistToUse = post.get("currentBlacklist", "").trim(); - - final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*"); - - if (selectedBlacklistEntries.length > 0) { - String temp = null; - for (final String selectedBlacklistEntry : selectedBlacklistEntries) { - if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) { - prop.put("LOCATION", temp); - return prop; - } - } - } - - Switchboard.urlBlacklist.clear(); - ListManager.reloadBlacklists(); - - } else if (post.containsKey("addBlacklistEntry")) { - - /* =========================================================== - * Add new entry to blacklist - * =========================================================== */ - - blacklistToUse = post.get("currentBlacklist", "").trim(); - final String blentry = post.get("newEntry", "").trim(); - - // store this call as api call - ListManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry); - - final String temp = addBlacklistEntry(blacklistToUse, blentry, header, BlacklistType.values()); - if (temp != null) { - prop.put("LOCATION", temp); - return prop; - } - - Switchboard.urlBlacklist.clear(); - ListManager.reloadBlacklists(); - - } else if ("moveBlacklistEntry".equals(action)) { - - /* =========================================================== - * Move an entry from one blacklist to another - * =========================================================== */ - - blacklistToUse = post.get("currentBlacklist", "").trim(); - final String targetBlacklist = post.get("targetBlacklist"); - - final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*"); - - if (selectedBlacklistEntries != null && - selectedBlacklistEntries.length > 0 && - targetBlacklist != null && - blacklistToUse != null && - !targetBlacklist.equals(blacklistToUse)) { - String temp; - for (final String selectedBlacklistEntry : selectedBlacklistEntries) { - if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header, BlacklistType.values())) != null) { - prop.put("LOCATION", temp); - return prop; - } - - if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) { - prop.put("LOCATION", temp); - return prop; - - } - } - } - - Switchboard.urlBlacklist.clear(); - ListManager.reloadBlacklists(); - - } else if ("editBlacklistEntry".equals(action)) { - - /* =========================================================== - * Edit entry of a blacklist - * =========================================================== */ - - blacklistToUse = post.get("currentBlacklist", "").trim(); - - final String[] editedBlacklistEntries = post.getAll("editedBlacklistEntry.*"); - - // if edited entry has been posted, save changes - if (editedBlacklistEntries.length > 0) { - - final String[] selectedBlacklistEntries = post.getAll("selectedBlacklistEntry.*"); - - if (selectedBlacklistEntries.length != editedBlacklistEntries.length) { - prop.put("LOCATION", ""); - return prop; - } - - String temp = null; - - for (int i = 0; i < selectedBlacklistEntries.length; i++) { - - if (!selectedBlacklistEntries[i].equals(editedBlacklistEntries[i])) { - - if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, BlacklistType.values())) != null) { - prop.put("LOCATION", temp); - return prop; - } - - if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header, BlacklistType.values())) != null) { - prop.put("LOCATION", temp); - return prop; - } - } - } - - Switchboard.urlBlacklist.clear(); - ListManager.reloadBlacklists(); - prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse); - - // else return entry to be edited - } else { - final String[] selectedEntries = post.getAll("selectedEntry.*"); - if (selectedEntries != null && selectedEntries.length > 0 && blacklistToUse != null) { - for (int i = 0; i < selectedEntries.length; i++) { - prop.putHTML(DISABLED + EDIT + "editList_" + i + "_item", selectedEntries[i]); - prop.put(DISABLED + EDIT + "editList_" + i + "_count", i); - } - prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse); - prop.put(DISABLED + "edit", "1"); - prop.put(DISABLED + EDIT + "editList", selectedEntries.length); - } - } - } else if ("selectRange".equals(action)) { - blacklistToUse = post.get("currentBlacklist"); - } - - } - - // if we have not chosen a blacklist until yet we use the first file - if (blacklistToUse == null && dirlist != null && !dirlist.isEmpty()) { - blacklistToUse = dirlist.get(0); - } - - // Read the blacklist items from file - if (blacklistToUse != null) { - int entryCount = 0; - final List list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse)); - - // sort them - final String[] sortedlist = new String[list.size()]; - Arrays.sort(list.toArray(sortedlist)); - - // display them - boolean dark = true; - int offset = 0; - int size = 50; - int to = 50; - if (post != null) { - offset = post.getInt("offset", 0); - size = post.getInt("size", 50); - to = offset + size; - } - if (offset > sortedlist.length || offset < 0) { - offset = 0; - } - if (to > sortedlist.length || size < 1) { - to = sortedlist.length; - } - - for (int j = offset; j < to; ++j){ - final String nextEntry = sortedlist[j]; - - if (nextEntry.isEmpty()) continue; - if (nextEntry.charAt(0) == '#') continue; - prop.put(DISABLED + EDIT + "Itemlist_" + entryCount + "_dark", dark ? "1" : "0"); - dark = !dark; - prop.putHTML(DISABLED + EDIT + "Itemlist_" + entryCount + "_item", nextEntry); - prop.put(DISABLED + EDIT + "Itemlist_" + entryCount + "_count", entryCount); - entryCount++; - } - prop.put(DISABLED + EDIT + "Itemlist", entryCount); - - // create selection of sublist - entryCount = 0; - int end = -1; - int start = -1; - if (sortedlist.length > 0) { - while (end < sortedlist.length) { - if (size > 0) { - start = entryCount * size; - end = (entryCount + 1) * size; - } else { - start = 0; - end = sortedlist.length; - } - prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_value", start); - prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_fvalue", start + 1); - if (end > sortedlist.length) { - end = sortedlist.length; - } - prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_tvalue", end); - if (start == offset) { - prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_selected", 1); - } - entryCount++; - } - } else { - prop.put(DISABLED + EDIT + "subListOffset_0_value", 0); - prop.put(DISABLED + EDIT + "subListOffset_0_fvalue", 0); - prop.put(DISABLED + EDIT + "subListOffset_0_tvalue", 0); - entryCount++; - } - prop.put(DISABLED + EDIT + "subListOffset", entryCount); - - // create selection of list size - final int[] sizes = {10,25,50,100,250,-1}; - for (int i = 0; i < sizes.length; i++) { - prop.put(DISABLED + EDIT + "subListSize_" + i + "_value", sizes[i]); - if (sizes[i] == -1) { - prop.put(DISABLED + EDIT + "subListSize_" + i + "_text", "all"); - } else { - prop.put(DISABLED + EDIT + "subListSize_" + i + "_text", sizes[i]); - } - if (sizes[i] == size) { - prop.put(DISABLED + EDIT + "subListSize_" + i + "_selected", 1); - } - } - prop.put(DISABLED + EDIT + "subListSize", sizes.length); - } - - // List BlackLists - int blacklistCount = 0; - int blacklistMoveCount = 0; - if (dirlist != null) { - - for (final String element : dirlist) { - prop.putXML(DISABLED + BLACKLIST + blacklistCount + "_name", element); - prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", "0"); - - if (element.equals(blacklistToUse)) { //current List - prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", "1"); - - for (int blTypes=0; blTypes < BlacklistType.values().length; blTypes++) { - prop.putXML(DISABLED + "currentActiveFor_" + blTypes + "_blTypeName",BlacklistType.values()[blTypes].toString()); - prop.put(DISABLED + "currentActiveFor_" + blTypes + "_checked", - ListManager.listSetContains(BlacklistType.values()[blTypes] + ".BlackLists", element) ? "0" : "1"); - } - prop.put(DISABLED + "currentActiveFor", BlacklistType.values().length); - - } else { - prop.putXML(DISABLED + EDIT + BLACKLIST_MOVE + blacklistMoveCount + "_name", element); - blacklistMoveCount++; - } - - if (ListManager.listSetContains(BLACKLIST_SHARED, element)) { - prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", "1"); - } else { - prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", "0"); - } - - int activeCount = 0; - for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", element)) { - prop.putHTML(DISABLED + BLACKLIST + blacklistCount + "_active_" + activeCount + "_blTypeName", supportedBlacklistType.toString()); - activeCount++; - } - } - prop.put(DISABLED + BLACKLIST + blacklistCount + "_active", activeCount); - blacklistCount++; - } - } - prop.put(DISABLED + "blackLists", blacklistCount); - prop.put(DISABLED + EDIT + "blackListsMove", blacklistMoveCount); - - prop.putXML(DISABLED + "currentBlacklist", (blacklistToUse==null) ? "" : blacklistToUse); - prop.putXML(DISABLED + EDIT + "currentBlacklist", (blacklistToUse==null) ? "" : blacklistToUse); - prop.put("disabled", (blacklistToUse == null) ? "1" : "0"); - return prop; - } - - - - /** - * This method adds a new entry to the chosen blacklist. - * @param blacklistToUse the name of the blacklist the entry is to be added to - * @param newEntry the entry that is to be added - * @param header - * @param supportedBlacklistTypes - * @return null if no error occurred, else a String to put into LOCATION - */ - private static String addBlacklistEntry( - final String blacklistToUse, - final String newEntry, - final RequestHeader header, - final BlacklistType[] supportedBlacklistTypes) { - - if (blacklistToUse == null || blacklistToUse.isEmpty()) { - return ""; - } - - if (newEntry == null || newEntry.isEmpty()) { - return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; - } - - addBlacklistEntry(ListManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); - SearchEventCache.cleanupEvents(true); - return null; - } - - - /** - * This method deletes a blacklist entry. - * @param blacklistToUse the name of the blacklist the entry is to be deleted from - * @param oldEntry the entry that is to be deleted - * @param header - * @param supportedBlacklistTypes - * @return null if no error occured, else a String to put into LOCATION - */ - private static String deleteBlacklistEntry( - final String blacklistToUse, - final String oldEntry, - final RequestHeader header, - final BlacklistType[] supportedBlacklistTypes) { - - if (blacklistToUse == null || blacklistToUse.isEmpty()) { - return ""; - } - - if (oldEntry == null || oldEntry.isEmpty()) { - return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; - } - - deleteBlacklistEntry(ListManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes); - SearchEventCache.cleanupEvents(true); - return null; - } - - /** - * This method deletes a blacklist entry. - * @param blacklistToUse the name of the blacklist the entry is to be deleted from - * @param oldEntry the entry that is to be deleted - * @param supportedBlacklistTypes - */ - private static void deleteBlacklistEntry( - final File listsPath, - final String blacklistToUse, - String oldEntry, - final BlacklistType[] supportedBlacklistTypes) { - - // load blacklist data from file - final List list = FileUtils.getListArray(new File(listsPath, blacklistToUse)); - - // delete the old entry from file - if (list != null) { - for (final String entry : list) { - if (entry.equals(oldEntry)) { - list.remove(entry); - break; - } - } - FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()])); - } - - // remove the entry from the running blacklist engine - int pos = oldEntry.indexOf('/',0); - if (pos < 0) { - // add default empty path pattern - pos = oldEntry.length(); - oldEntry = oldEntry + "/.*"; - } - for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) { - if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) { - Switchboard.urlBlacklist.remove(supportedBlacklistType,oldEntry.substring(0, pos), oldEntry.substring(pos + 1)); - } - } - } - - - - /** - * This method adds a new entry to the chosen blacklist. - * @param blacklistToUse the name of the blacklist the entry is to be added to - * @param newEntry the entry that is to be added - * @param supportedBlacklistTypes - */ - private static void addBlacklistEntry( - final File listsPath, - final String blacklistToUse, - String newEntry, - final BlacklistType[] supportedBlacklistTypes) { - - // ignore empty entries - if(newEntry == null || newEntry.isEmpty()) { - Log.logWarning("Blacklist", "skipped adding an empty entry"); - return; - } - - if (newEntry.startsWith("http://") ){ - newEntry = newEntry.substring(7); - } else if (newEntry.startsWith("https://")) { - newEntry = newEntry.substring(8); - } - - if (newEntry.indexOf("*") < 0) { - // user did not use any wild cards and just submitted a word - - addBlacklistEntry0(listsPath, blacklistToUse, ".*" + newEntry + ".*/.*", supportedBlacklistTypes); - addBlacklistEntry0(listsPath, blacklistToUse, ".*.*/.*" + newEntry + ".*", supportedBlacklistTypes); - - } else { - - int pos = newEntry.indexOf('/',0); - if (pos < 0) { - // add default empty path pattern - pos = newEntry.length(); - newEntry = newEntry + "/.*"; - } - - addBlacklistEntry0(listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); - } - } - - private static void addBlacklistEntry0( - final File listsPath, - final String blacklistToUse, - String newEntry, - final BlacklistType[] supportedBlacklistTypes) { - - if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) { - // append the line to the file - PrintWriter pw = null; - try { - pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true)); - pw.println(newEntry); - pw.close(); - } catch (final IOException e) { - Log.logException(e); - } finally { - if (pw != null) { - try { - pw.close(); - } catch (final Exception e) { - Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage()); - } - - } - } - - // add to blacklist - int pos = newEntry.indexOf('/',0); - for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) { - if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) { - Switchboard.urlBlacklist.add(supportedBlacklistType, newEntry.substring(0, pos), newEntry.substring(pos + 1)); - } - } - } - } -} +// Blacklist_p.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// This File is contributed by Alexander Schier +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +// You must compile this file with +// javac -classpath .:../classes Blacklist_p.java +// if the shell's current path is HTROOT + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.MalformedURLException; +import java.util.Arrays; +import java.util.List; + +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.data.ListManager; +import net.yacy.data.WorkTables; +import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.FileUtils; +import net.yacy.repository.Blacklist; +import net.yacy.repository.Blacklist.BlacklistType; +import net.yacy.search.Switchboard; +import net.yacy.search.query.SearchEventCache; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class Blacklist_p { + private final static String EDIT = "edit_"; + private final static String DISABLED = "disabled_"; + private final static String BLACKLIST = "blackLists_"; + private final static String BLACKLIST_MOVE = "blackListsMove_"; + private final static String BLACKLIST_SHARED = "BlackLists.Shared"; + + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + + // initialize the list manager + ListManager.switchboard = (Switchboard) env; + ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS")); + + // load all blacklist files located in the directory + List dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); + + String blacklistToUse = null; + final serverObjects prop = new serverObjects(); + prop.putHTML("blacklistEngine", Blacklist.getEngineInfo()); + + // do all post operations + if (post != null) { + + final String action = post.get("action", ""); + + if(post.containsKey("testList")) { + prop.put("testlist", "1"); + String urlstring = post.get("testurl", ""); + if (!urlstring.startsWith("http://") && + !urlstring.startsWith("https://") && + !urlstring.startsWith("ftp://") && + !urlstring.startsWith("smb://") && + !urlstring.startsWith("file://")) { + urlstring = "http://"+urlstring; + } + DigestURI testurl; + try { + testurl = new DigestURI(urlstring); + } catch (final MalformedURLException e) { + testurl = null; + } + if(testurl != null) { + prop.putHTML("testlist_url",testurl.toString()); + if (Switchboard.urlBlacklist.isListed(BlacklistType.CRAWLER, testurl)) { + prop.put("testlist_listedincrawler", "1"); + } + if (Switchboard.urlBlacklist.isListed(BlacklistType.DHT, testurl)) { + prop.put("testlist_listedindht", "1"); + } + if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, testurl)) { + prop.put("testlist_listedinnews", "1"); + } + if (Switchboard.urlBlacklist.isListed(BlacklistType.PROXY, testurl)) { + prop.put("testlist_listedinproxy", "1"); + } + if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, testurl)) { + prop.put("testlist_listedinsearch", "1"); + } + if (Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS, testurl)) { + prop.put("testlist_listedinsurftips", "1"); + } + } else { + prop.put("testlist_url","not valid"); + } + } + if (post.containsKey("selectList")) { + blacklistToUse = post.get("selectedListName"); + if (blacklistToUse != null && blacklistToUse.isEmpty()) { + blacklistToUse = null; + } + } + if (post.containsKey("createNewList")) { + /* =========================================================== + * Creation of a new blacklist + * =========================================================== */ + + blacklistToUse = post.get("newListName", "").trim(); + if (blacklistToUse.isEmpty()) { + prop.put("LOCATION",""); + return prop; + } + + // Check if blacklist name only consists of "legal" characters. + // This is mainly done to prevent files from being written to other directories + // than the LISTS directory. + if (!blacklistToUse.matches("^[\\p{L}\\d\\+\\-_]+[\\p{L}\\d\\+\\-_.]*(\\.black){0,1}$")) { + prop.put("error", 1); + prop.putHTML("error_name", blacklistToUse); + blacklistToUse = null; + } else { + + if (!blacklistToUse.endsWith(".black")) { + blacklistToUse += ".black"; + } + + if (!dirlist.contains(blacklistToUse)) { + try { + final File newFile = new File(ListManager.listsPath, blacklistToUse); + newFile.createNewFile(); + + // share the newly created blacklist + ListManager.updateListSet(BLACKLIST_SHARED, blacklistToUse); + + // activate it for all known blacklist types + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + ListManager.updateListSet(supportedBlacklistType + ".BlackLists", blacklistToUse); + } + } catch (final IOException e) {/* */} + } else { + prop.put("error", 2); + prop.putHTML("error_name", blacklistToUse); + blacklistToUse = null; + } + + // reload Blacklists + dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); + } + + } else if (post.containsKey("deleteList")) { + /* =========================================================== + * Delete a blacklist + * =========================================================== */ + + blacklistToUse = post.get("selectedListName"); + if (blacklistToUse == null || blacklistToUse.isEmpty()) { + prop.put("LOCATION",""); + return prop; + } + + final File blackListFile = new File(ListManager.listsPath, blacklistToUse); + if(!blackListFile.delete()) { + Log.logWarning("Blacklist", "file "+ blackListFile +" could not be deleted!"); + } + + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); + } + + // remove it from the shared list + ListManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse); + blacklistToUse = null; + + // reload Blacklists + dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); + + } else if (post.containsKey("activateList")) { + + /* =========================================================== + * Activate/Deactivate a blacklist + * =========================================================== */ + + blacklistToUse = post.get("selectedListName", "").trim(); + if (blacklistToUse == null || blacklistToUse.isEmpty()) { + prop.put("LOCATION", ""); + return prop; + } + + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + if (post.containsKey("activateList4" + supportedBlacklistType)) { + ListManager.updateListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); + } else { + ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); + } + } + + Switchboard.urlBlacklist.clear(); + ListManager.reloadBlacklists(); + + } else if (post.containsKey("shareList")) { + + /* =========================================================== + * Share a blacklist + * =========================================================== */ + + blacklistToUse = post.get("selectedListName", "").trim(); + if (blacklistToUse == null || blacklistToUse.isEmpty()) { + prop.put("LOCATION", ""); + return prop; + } + + if (ListManager.listSetContains(BLACKLIST_SHARED, blacklistToUse)) { + // Remove from shared BlackLists + ListManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse); + } else { // inactive list -> enable + ListManager.updateListSet(BLACKLIST_SHARED, blacklistToUse); + } + } else if ("deleteBlacklistEntry".equals(action)) { + + /* =========================================================== + * Delete an entry from a blacklist + * =========================================================== */ + + blacklistToUse = post.get("currentBlacklist", "").trim(); + + final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*"); + + if (selectedBlacklistEntries.length > 0) { + String temp = null; + for (final String selectedBlacklistEntry : selectedBlacklistEntries) { + if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) { + prop.put("LOCATION", temp); + return prop; + } + } + } + + Switchboard.urlBlacklist.clear(); + ListManager.reloadBlacklists(); + + } else if (post.containsKey("addBlacklistEntry")) { + + /* =========================================================== + * Add new entry to blacklist + * =========================================================== */ + + blacklistToUse = post.get("currentBlacklist", "").trim(); + final String blentry = post.get("newEntry", "").trim(); + + // store this call as api call + ListManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry); + + final String temp = addBlacklistEntry(blacklistToUse, blentry, header, BlacklistType.values()); + if (temp != null) { + prop.put("LOCATION", temp); + return prop; + } + + Switchboard.urlBlacklist.clear(); + ListManager.reloadBlacklists(); + + } else if ("moveBlacklistEntry".equals(action)) { + + /* =========================================================== + * Move an entry from one blacklist to another + * =========================================================== */ + + blacklistToUse = post.get("currentBlacklist", "").trim(); + final String targetBlacklist = post.get("targetBlacklist"); + + final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*"); + + if (selectedBlacklistEntries != null && + selectedBlacklistEntries.length > 0 && + targetBlacklist != null && + blacklistToUse != null && + !targetBlacklist.equals(blacklistToUse)) { + String temp; + for (final String selectedBlacklistEntry : selectedBlacklistEntries) { + if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header, BlacklistType.values())) != null) { + prop.put("LOCATION", temp); + return prop; + } + + if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) { + prop.put("LOCATION", temp); + return prop; + + } + } + } + + Switchboard.urlBlacklist.clear(); + ListManager.reloadBlacklists(); + + } else if ("editBlacklistEntry".equals(action)) { + + /* =========================================================== + * Edit entry of a blacklist + * =========================================================== */ + + blacklistToUse = post.get("currentBlacklist", "").trim(); + + final String[] editedBlacklistEntries = post.getAll("editedBlacklistEntry.*"); + + // if edited entry has been posted, save changes + if (editedBlacklistEntries.length > 0) { + + final String[] selectedBlacklistEntries = post.getAll("selectedBlacklistEntry.*"); + + if (selectedBlacklistEntries.length != editedBlacklistEntries.length) { + prop.put("LOCATION", ""); + return prop; + } + + String temp = null; + + for (int i = 0; i < selectedBlacklistEntries.length; i++) { + + if (!selectedBlacklistEntries[i].equals(editedBlacklistEntries[i])) { + + if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, BlacklistType.values())) != null) { + prop.put("LOCATION", temp); + return prop; + } + + if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header, BlacklistType.values())) != null) { + prop.put("LOCATION", temp); + return prop; + } + } + } + + Switchboard.urlBlacklist.clear(); + ListManager.reloadBlacklists(); + prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse); + + // else return entry to be edited + } else { + final String[] selectedEntries = post.getAll("selectedEntry.*"); + if (selectedEntries != null && selectedEntries.length > 0 && blacklistToUse != null) { + for (int i = 0; i < selectedEntries.length; i++) { + prop.putHTML(DISABLED + EDIT + "editList_" + i + "_item", selectedEntries[i]); + prop.put(DISABLED + EDIT + "editList_" + i + "_count", i); + } + prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse); + prop.put(DISABLED + "edit", "1"); + prop.put(DISABLED + EDIT + "editList", selectedEntries.length); + } + } + } else if ("selectRange".equals(action)) { + blacklistToUse = post.get("currentBlacklist"); + } + + } + + // if we have not chosen a blacklist until yet we use the first file + if (blacklistToUse == null && dirlist != null && !dirlist.isEmpty()) { + blacklistToUse = dirlist.get(0); + } + + // Read the blacklist items from file + if (blacklistToUse != null) { + int entryCount = 0; + final List list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse)); + + // sort them + final String[] sortedlist = new String[list.size()]; + Arrays.sort(list.toArray(sortedlist)); + + // display them + boolean dark = true; + int offset = 0; + int size = 50; + int to = 50; + if (post != null) { + offset = post.getInt("offset", 0); + size = post.getInt("size", 50); + to = offset + size; + } + if (offset > sortedlist.length || offset < 0) { + offset = 0; + } + if (to > sortedlist.length || size < 1) { + to = sortedlist.length; + } + + for (int j = offset; j < to; ++j){ + final String nextEntry = sortedlist[j]; + + if (nextEntry.isEmpty()) continue; + if (nextEntry.charAt(0) == '#') continue; + prop.put(DISABLED + EDIT + "Itemlist_" + entryCount + "_dark", dark ? "1" : "0"); + dark = !dark; + prop.putHTML(DISABLED + EDIT + "Itemlist_" + entryCount + "_item", nextEntry); + prop.put(DISABLED + EDIT + "Itemlist_" + entryCount + "_count", entryCount); + entryCount++; + } + prop.put(DISABLED + EDIT + "Itemlist", entryCount); + + // create selection of sublist + entryCount = 0; + int end = -1; + int start = -1; + if (sortedlist.length > 0) { + while (end < sortedlist.length) { + if (size > 0) { + start = entryCount * size; + end = (entryCount + 1) * size; + } else { + start = 0; + end = sortedlist.length; + } + prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_value", start); + prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_fvalue", start + 1); + if (end > sortedlist.length) { + end = sortedlist.length; + } + prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_tvalue", end); + if (start == offset) { + prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_selected", 1); + } + entryCount++; + } + } else { + prop.put(DISABLED + EDIT + "subListOffset_0_value", 0); + prop.put(DISABLED + EDIT + "subListOffset_0_fvalue", 0); + prop.put(DISABLED + EDIT + "subListOffset_0_tvalue", 0); + entryCount++; + } + prop.put(DISABLED + EDIT + "subListOffset", entryCount); + + // create selection of list size + final int[] sizes = {10,25,50,100,250,-1}; + for (int i = 0; i < sizes.length; i++) { + prop.put(DISABLED + EDIT + "subListSize_" + i + "_value", sizes[i]); + if (sizes[i] == -1) { + prop.put(DISABLED + EDIT + "subListSize_" + i + "_text", "all"); + } else { + prop.put(DISABLED + EDIT + "subListSize_" + i + "_text", sizes[i]); + } + if (sizes[i] == size) { + prop.put(DISABLED + EDIT + "subListSize_" + i + "_selected", 1); + } + } + prop.put(DISABLED + EDIT + "subListSize", sizes.length); + } + + // List BlackLists + int blacklistCount = 0; + int blacklistMoveCount = 0; + if (dirlist != null) { + + for (final String element : dirlist) { + prop.putXML(DISABLED + BLACKLIST + blacklistCount + "_name", element); + prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", "0"); + + if (element.equals(blacklistToUse)) { //current List + prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", "1"); + + for (int blTypes=0; blTypes < BlacklistType.values().length; blTypes++) { + prop.putXML(DISABLED + "currentActiveFor_" + blTypes + "_blTypeName",BlacklistType.values()[blTypes].toString()); + prop.put(DISABLED + "currentActiveFor_" + blTypes + "_checked", + ListManager.listSetContains(BlacklistType.values()[blTypes] + ".BlackLists", element) ? "0" : "1"); + } + prop.put(DISABLED + "currentActiveFor", BlacklistType.values().length); + + } else { + prop.putXML(DISABLED + EDIT + BLACKLIST_MOVE + blacklistMoveCount + "_name", element); + blacklistMoveCount++; + } + + if (ListManager.listSetContains(BLACKLIST_SHARED, element)) { + prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", "1"); + } else { + prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", "0"); + } + + int activeCount = 0; + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", element)) { + prop.putHTML(DISABLED + BLACKLIST + blacklistCount + "_active_" + activeCount + "_blTypeName", supportedBlacklistType.toString()); + activeCount++; + } + } + prop.put(DISABLED + BLACKLIST + blacklistCount + "_active", activeCount); + blacklistCount++; + } + } + prop.put(DISABLED + "blackLists", blacklistCount); + prop.put(DISABLED + EDIT + "blackListsMove", blacklistMoveCount); + + prop.putXML(DISABLED + "currentBlacklist", (blacklistToUse==null) ? "" : blacklistToUse); + prop.putXML(DISABLED + EDIT + "currentBlacklist", (blacklistToUse==null) ? "" : blacklistToUse); + prop.put("disabled", (blacklistToUse == null) ? "1" : "0"); + return prop; + } + + + + /** + * This method adds a new entry to the chosen blacklist. + * @param blacklistToUse the name of the blacklist the entry is to be added to + * @param newEntry the entry that is to be added + * @param header + * @param supportedBlacklistTypes + * @return null if no error occurred, else a String to put into LOCATION + */ + private static String addBlacklistEntry( + final String blacklistToUse, + final String newEntry, + final RequestHeader header, + final BlacklistType[] supportedBlacklistTypes) { + + if (blacklistToUse == null || blacklistToUse.isEmpty()) { + return ""; + } + + if (newEntry == null || newEntry.isEmpty()) { + return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; + } + + addBlacklistEntry(ListManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); + SearchEventCache.cleanupEvents(true); + return null; + } + + + /** + * This method deletes a blacklist entry. + * @param blacklistToUse the name of the blacklist the entry is to be deleted from + * @param oldEntry the entry that is to be deleted + * @param header + * @param supportedBlacklistTypes + * @return null if no error occured, else a String to put into LOCATION + */ + private static String deleteBlacklistEntry( + final String blacklistToUse, + final String oldEntry, + final RequestHeader header, + final BlacklistType[] supportedBlacklistTypes) { + + if (blacklistToUse == null || blacklistToUse.isEmpty()) { + return ""; + } + + if (oldEntry == null || oldEntry.isEmpty()) { + return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; + } + + deleteBlacklistEntry(ListManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes); + SearchEventCache.cleanupEvents(true); + return null; + } + + /** + * This method deletes a blacklist entry. + * @param blacklistToUse the name of the blacklist the entry is to be deleted from + * @param oldEntry the entry that is to be deleted + * @param supportedBlacklistTypes + */ + private static void deleteBlacklistEntry( + final File listsPath, + final String blacklistToUse, + String oldEntry, + final BlacklistType[] supportedBlacklistTypes) { + + // load blacklist data from file + final List list = FileUtils.getListArray(new File(listsPath, blacklistToUse)); + + // delete the old entry from file + if (list != null) { + for (final String entry : list) { + if (entry.equals(oldEntry)) { + list.remove(entry); + break; + } + } + FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()])); + } + + // remove the entry from the running blacklist engine + int pos = oldEntry.indexOf('/',0); + if (pos < 0) { + // add default empty path pattern + pos = oldEntry.length(); + oldEntry = oldEntry + "/.*"; + } + for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) { + if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) { + Switchboard.urlBlacklist.remove(supportedBlacklistType,oldEntry.substring(0, pos), oldEntry.substring(pos + 1)); + } + } + } + + + + /** + * This method adds a new entry to the chosen blacklist. + * @param blacklistToUse the name of the blacklist the entry is to be added to + * @param newEntry the entry that is to be added + * @param supportedBlacklistTypes + */ + private static void addBlacklistEntry( + final File listsPath, + final String blacklistToUse, + String newEntry, + final BlacklistType[] supportedBlacklistTypes) { + + // ignore empty entries + if(newEntry == null || newEntry.isEmpty()) { + Log.logWarning("Blacklist", "skipped adding an empty entry"); + return; + } + + if (newEntry.startsWith("http://") ){ + newEntry = newEntry.substring(7); + } else if (newEntry.startsWith("https://")) { + newEntry = newEntry.substring(8); + } + + if (newEntry.indexOf("*") < 0) { + // user did not use any wild cards and just submitted a word + + addBlacklistEntry0(listsPath, blacklistToUse, ".*" + newEntry + ".*/.*", supportedBlacklistTypes); + addBlacklistEntry0(listsPath, blacklistToUse, ".*.*/.*" + newEntry + ".*", supportedBlacklistTypes); + + } else { + + int pos = newEntry.indexOf('/',0); + if (pos < 0) { + // add default empty path pattern + newEntry = newEntry + "/.*"; + } + + addBlacklistEntry0(listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); + } + } + + private static void addBlacklistEntry0( + final File listsPath, + final String blacklistToUse, + String newEntry, + final BlacklistType[] supportedBlacklistTypes) { + + if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) { + // append the line to the file + PrintWriter pw = null; + try { + pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true)); + pw.println(newEntry); + pw.close(); + } catch (final IOException e) { + Log.logException(e); + } finally { + if (pw != null) { + try { + pw.close(); + } catch (final Exception e) { + Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage()); + } + + } + } + + // add to blacklist + int pos = newEntry.indexOf('/',0); + for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) { + if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) { + Switchboard.urlBlacklist.add(supportedBlacklistType, newEntry.substring(0, pos), newEntry.substring(pos + 1)); + } + } + } + } +} diff --git a/htroot/RankingSolr_p.html b/htroot/RankingSolr_p.html index eb7eaf673..5dd27b17a 100644 --- a/htroot/RankingSolr_p.html +++ b/htroot/RankingSolr_p.html @@ -8,20 +8,20 @@ #%env/templates/header.template%# #%env/templates/submenuSearchConfiguration.template%#

Solr Ranking Configuration

-

These are ranking attributes for Solr. This ranking applies for internal and remote Solr acess.

+

These are ranking attributes for Solr. This ranking applies for internal and remote Solr access.

Solr Double Content Detection

Double-Content detection is done using a ranking on a 'unique'-Field, named 'fuzzy_signature_unique_b'. This field is set during parsing and is influenced by two attributes for the TextProfileSignature class.

-
-
+
+
This is the minimum length of a word which shall be considered as element of the signature. Should be either 2 or 3.
-
-
+
+
The quantRate is a measurement for the number of words that take part in a signature computation. The higher the number, the less words are used for the signature. For minTokenLen = 2 the quantRate value should not be below 0.24; for minTokenLen = 3 the quantRate value must be not below 0.5. @@ -38,9 +38,9 @@
Solr Boosts
#{boosts}# -
+
- +
#{/boosts}#
diff --git a/htroot/YaCySearchPluginFF.html b/htroot/YaCySearchPluginFF.html index c19206b4d..2df71af41 100644 --- a/htroot/YaCySearchPluginFF.html +++ b/htroot/YaCySearchPluginFF.html @@ -1,61 +1,61 @@ - - - - YaCy '#[clientname]#': Quick Crawl Link - #%env/templates/metas.template%# - - - - -
- - - - - - -
- - YaCy-Logo - - -
#[clientname]#: Firefox Search Plugin
- 		-
-
-
-
- - - - - - - - - -
- YaCy Firefox Search-Plugin Installation: -

Simply click on the link shown below to integrate the YaCy Firefox Search-Plugin into your browser.

-

In Mozilla Firefox, you can the Search-Plugin via the search box on the toolbar.
In Mozilla (Seamonkey) you can access the Search-Plugin via the Sidebar or the Location Bar.

-
Install the YaCy search plugin.
- - - + + + + YaCy '#[clientname]#': Quick Crawl Link + #%env/templates/metas.template%# + + + + +
+ + + + + + +
+ + YaCy-Logo + + +
#[clientname]#: Firefox Search Plugin
+ 		+
+
+
+
+ + + + + + + + + +
+ YaCy Firefox Search-Plugin Installation: +

Simply click on the link shown below to integrate the YaCy Firefox Search-Plugin into your browser.

+

In Mozilla Firefox, you can the Search-Plugin via the search box on the toolbar.
In Mozilla (Seamonkey) you can access the Search-Plugin via the Sidebar or the Location Bar.

+
Install the YaCy search plugin.
+ + + \ No newline at end of file diff --git a/htroot/opensearchdescription.xml b/htroot/opensearchdescription.xml index 46705b913..3d38509bd 100644 --- a/htroot/opensearchdescription.xml +++ b/htroot/opensearchdescription.xml @@ -4,7 +4,7 @@ xmlns:suggestions="http://www.opensearch.org/specifications/opensearch/extensions/suggestions/1.1"> #(compareyacy)#::Compare #(/compareyacy)#YaCy/#[clientname]# YaCy.net - #[SearchPageGreeting]# - http://#[thisaddress]#/env/grafics/yacy.gif + http://#[thisaddress]#/env/grafics/yacy.png data:image/x-icon;base64,AAABAAEAEBAAAAAAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD////////Chiu+fBm+fRq+fRq+fRq+fRq+fRq8ehXMmk////////////////////////////++exjUqGfZsnjYsXbYsXbYsXbZsnjTp2W+fBr///////////////////////////+9exfs2r/69/L58+z58+z58+z59e/z59bChSn///////////////////////////+9fBnnz638+fX69O369e748un38Ob59O3Ll0fVrGz////////////////////////Jk0HJlUjXuYrWt4jVtYPr28T58+r59OzPn1fPnlX////////////////////////Rolu8eRXCiTPEjTvCijbNnlj169359e7Zs3vLlkf////////////////////////UqmrAgSLt3MP27eH1693169327+T59Ozo0rG/gB////////////////////////////+/gCDv4Mn48+v38Of59Oz58+v48+vr2b2+fRv///////////////////////////+9ehXr17n58+z48ejo0bDp07T48+zx49DBhCj///////////////////////////+/fhzjyJ/59Oz59OzQoVvNmU759Oz58+vKlUbXrnH////////////////////////LlkfUqmn59Oz48+vZtHzNm1D48+v59OzPoFjOnlX////////////////////////OnVPOnFL59e348+vq1rnEiC7u38j69vDXsHTIkT3////////////////////////UqWjIkD327+T58+3s2bzAgSLp07T79/Pmzqq+fBn///////////////////////////+/fx7v4Mr7+PTx5NDHkD3kyqP8+vjs2r+9exf///////////////////////////++fBjQoFnVq2zTp2bGjTjJk0LWrW7SpWC+exf////////////////////////////Nm1C6dQy7eBG8eBK9exe8ehS7dxC6dQzLlkj////////////AH///wB8x+sAf///AD/+PwA///8AP///AD///4A9PU+APoz/gB///4AdxvOAHoz/gB3368AfjU/AHT1PwB/+P en-us diff --git a/htroot/www/welcome.html b/htroot/www/welcome.html index 2c8050e7d..82c66dbeb 100644 --- a/htroot/www/welcome.html +++ b/htroot/www/welcome.html @@ -1,69 +1,69 @@ - - - -YaCy: Default Page for Individual Peer Content - - - - - - - - - - - - -
- - - - - - -
- - YaCy-Logo - - -
Individual Web Page
- 		- - - -
-
-

- - - - - - -
-

Welcome to your own web page
in the YaCy Network!


- -

-THIS IS A DEMONSTRATION PAGE FOR YOUR OWN INDIVIDUAL WEB SERVER! -PLEASE REPLACE THIS PAGE BY PUTTING A FILE index.html INTO THE PATH -<YaCy-application-home>#[wwwpath]# -

- -

-This is peer '#[peername]#', running on host #[hostname]#.
-Your are accessing this page from the host '#[clientip]#'.
- -Every user of YaCy #[couldcan]# access this page -using the URL http://#[peeraddress]#/www/ -or http://www.#[peerdomain]#.yacy from within the YaCy network.

- -

#[seniorinfo]#

- -

We integrated an easy mechanism for web page authoring -which can also be used for simple file-sharing. -Please open the sample page http://share.#[peerdomain]#.yacy -and set upload/download accounts to author and access content on this peer.

- -
- - + + + +YaCy: Default Page for Individual Peer Content + + + + + + + + + + + + +
+ + + + + + +
+ + YaCy-Logo + + +
Individual Web Page
+ 		+
+ +
+
+
+

+ + + + + + +
+

Welcome to your own web page
in the YaCy Network!


+ +

+THIS IS A DEMONSTRATION PAGE FOR YOUR OWN INDIVIDUAL WEB SERVER! +PLEASE REPLACE THIS PAGE BY PUTTING A FILE index.html INTO THE PATH +<YaCy-application-home>#[wwwpath]# +

+ +

+This is peer '#[peername]#', running on host #[hostname]#.
+Your are accessing this page from the host '#[clientip]#'.
+ +Every user of YaCy #[couldcan]# access this page +using the URL http://#[peeraddress]#/www/ +or http://www.#[peerdomain]#.yacy from within the YaCy network.

+ +

#[seniorinfo]#

+ +

We integrated an easy mechanism for web page authoring +which can also be used for simple file-sharing. +Please open the sample page http://share.#[peerdomain]#.yacy +and set upload/download accounts to author and access content on this peer.

+ +
+ + diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 06eeda3ef..ba4625a4c 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -1,1050 +1,1050 @@ -// yacysearch.java -// ----------------------- -// part of the AnomicHTTPD caching proxy -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://www.anomic.de -// Frankfurt, Germany, 2004 -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// You must compile this file with -// javac -classpath .:../classes yacysearch.java -// if the shell's current path is HTROOT - -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -import net.yacy.cora.document.analysis.Classification; -import net.yacy.cora.document.analysis.Classification.ContentDomain; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.UTF8; -import net.yacy.cora.federate.yacy.CacheStrategy; -import net.yacy.cora.geo.GeoLocation; -import net.yacy.cora.lod.vocabulary.Tagging; -import net.yacy.cora.protocol.Domains; -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.protocol.ResponseHeader; -import net.yacy.data.DidYouMean; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.document.Condenser; -import net.yacy.document.Document; -import net.yacy.document.LibraryProvider; -import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; -import net.yacy.kelondro.data.meta.URIMetadataNode; -import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.Bitfield; -import net.yacy.kelondro.util.Formatter; -import net.yacy.kelondro.util.ISO639; -import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.util.SetTools; -import net.yacy.peers.EventChannel; -import net.yacy.peers.NewsPool; -import net.yacy.peers.graphics.ProfilingGraph; -import net.yacy.repository.Blacklist.BlacklistType; -import net.yacy.search.EventTracker; -import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; -import net.yacy.search.index.Segment; -import net.yacy.search.query.AccessTracker; -import net.yacy.search.query.QueryGoal; -import net.yacy.search.query.QueryParams; -import net.yacy.search.query.SearchEvent; -import net.yacy.search.query.SearchEventCache; -import net.yacy.search.query.SearchEventType; -import net.yacy.search.ranking.RankingProfile; -import net.yacy.search.snippet.TextSnippet; -import net.yacy.server.serverCore; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; -import net.yacy.server.servletProperties; - -public class yacysearch { - - public static serverObjects respond( - final RequestHeader header, - final serverObjects post, - final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - sb.localSearchLastAccess = System.currentTimeMillis(); - - final boolean searchAllowed = - sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header); - - boolean authenticated = sb.adminAuthenticated(header) >= 2; - if ( !authenticated ) { - final UserDB.Entry user = sb.userDB.getUser(header); - authenticated = (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT)); - } - final boolean localhostAccess = header.accessFromLocalhost(); - final String promoteSearchPageGreeting = - (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig( - "network.unit.description", - "") : env.getConfig(SwitchboardConstants.GREETING, ""); - final String client = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search - - // get query - final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); - String querystring = originalquerystring.replace('+', ' ').trim(); - CacheStrategy snippetFetchStrategy = (post == null) ? null : CacheStrategy.parse(post.get("verify", sb.getConfig("search.verify", ""))); - if (authenticated && originalquerystring.length() == 0) sb.index.fulltext().commit(); - - final servletProperties prop = new servletProperties(); - prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); - - //get focus option - prop.put("focus", ((post == null) ? true : post.get("focus", "1").equals("1")) ? 1 : 0); - - // produce vocabulary navigation sidebars - Collection vocabularies = LibraryProvider.autotagging.getVocabularies(); - int j = 0; - for (Tagging v: vocabularies) { - prop.put("sidebarVocabulary_" + j + "_vocabulary", v.getName()); - j++; - } - prop.put("sidebarVocabulary", j); - - // get segment - Segment indexSegment = sb.index; - - final String EXT = header.get("EXT", ""); - final boolean rss = EXT.equals("rss"); - final boolean json = EXT.equals("json"); - prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting); - prop.put( - "promoteSearchPageGreeting.homepage", - sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, "")); - prop.put( - "promoteSearchPageGreeting.smallImage", - sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, "")); - if ( post == null || indexSegment == null || env == null || !searchAllowed ) { - // we create empty entries for template strings - prop.put("searchagain", "0"); - prop.put("former", ""); - prop.put("count", "10"); - prop.put("offset", "0"); - prop.put("resource", "global"); - prop.put("urlmaskfilter", (post == null) ? ".*" : post.get("urlmaskfilter", ".*")); - prop.put("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", "")); - prop.put("tenant", (post == null) ? "" : post.get("tenant", "")); - prop.put("indexof", "off"); - prop.put("constraint", ""); - prop.put("cat", "href"); - prop.put("depth", "0"); - prop.put( - "search.verify", - (post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh")); - prop.put( - "search.navigation", - (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all")); - prop.put("contentdom", "text"); - prop.put("contentdomCheckText", "1"); - prop.put("contentdomCheckAudio", "0"); - prop.put("contentdomCheckVideo", "0"); - prop.put("contentdomCheckImage", "0"); - prop.put("contentdomCheckApp", "0"); - prop.put("excluded", "0"); - prop.put("results", ""); - prop.put("resultTable", "0"); - prop.put("num-results", searchAllowed ? "0" : "4"); - prop.put("num-results_totalcount", 0); - prop.put("num-results_offset", 0); - prop.put("num-results_itemsPerPage", 10); - prop.put("geoinfo", "0"); - prop.put("rss_queryenc", ""); - prop.put("meanCount", 5); - return prop; - } - - // check for JSONP - if ( post.containsKey("callback") ) { - final String jsonp = post.get("callback") + "(["; - prop.put("jsonp-start", jsonp); - prop.put("jsonp-end", "])"); - } else { - prop.put("jsonp-start", ""); - prop.put("jsonp-end", ""); - } - - // Adding CORS Access header for yacysearch.rss output - if ( rss ) { - final ResponseHeader outgoingHeader = new ResponseHeader(200); - outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*"); - prop.setOutgoingHeader(outgoingHeader); - } - - // collect search attributes - - int itemsPerPage = - Math.min( - (authenticated) - ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() - ? 100 - : 5000) : (snippetFetchStrategy != null - && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 1000), - post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative - int startRecord = post.getInt("startRecord", post.getInt("offset", 0)); - - boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0; - final boolean indexof = (post != null && post.get("indexof", "").equals("on")); - - String prefermask = (post == null) ? "" : post.get("prefermaskfilter", ""); - if ( !prefermask.isEmpty() && prefermask.indexOf(".*", 0) < 0 ) { - prefermask = ".*" + prefermask + ".*"; - } - - Bitfield constraint = - (post != null && post.containsKey("constraint") && !post.get("constraint", "").isEmpty()) - ? new Bitfield(4, post.get("constraint", "______")) - : null; - if ( indexof ) { - constraint = new Bitfield(4); - constraint.set(Condenser.flag_cat_indexof, true); - } - - // SEARCH - final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER); - final boolean indexReceiveGranted = - sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true) - || sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true) - || clustersearch; - global = global && indexReceiveGranted; // if the user does not want indexes from remote peers, it cannot be a global searchnn - final boolean intranetMode = sb.isIntranetMode() || sb.isAllIPMode(); - - // increase search statistic counter - if ( !global ) { - // we count only searches on the local peer here, because global searches - // are counted on the target peer to preserve privacy of the searcher - if ( authenticated ) { - // local or authenticated search requests are counted separately - // because they are not part of a public available peer statistic - sb.searchQueriesRobinsonFromLocal++; - } else { - // robinson-searches from non-authenticated requests are public - // and may be part of the public available statistic - sb.searchQueriesRobinsonFromRemote++; - } - } - - // find search domain - final Classification.ContentDomain contentdom = - ContentDomain.contentdomParser(post == null ? "all" : post.get("contentdom", "all")); - - // patch until better search profiles are available - if (contentdom == ContentDomain.IMAGE && (itemsPerPage == 10 || itemsPerPage == 100)) { - itemsPerPage = 64; - } else if ( contentdom != ContentDomain.IMAGE && itemsPerPage > 50 && itemsPerPage < 100 ) { - itemsPerPage = 10; - } - - // check the search tracker - TreeSet trackerHandles = sb.localSearchTracker.get(client); - if ( trackerHandles == null ) { - trackerHandles = new TreeSet(); - } - boolean block = false; - if ( Domains.matchesList(client, sb.networkBlacklist) ) { - global = false; - if ( snippetFetchStrategy != null ) { - snippetFetchStrategy = null; - } - block = true; - Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM " - + client - + " gets no permission to search"); - } else if ( Domains.matchesList(client, sb.networkWhitelist) ) { - Log.logInfo("LOCAL_SEARCH", "ACCESS CONTROL: WHITELISTED CLIENT FROM " - + client - + " gets no search restrictions"); - } else if ( !authenticated && !localhostAccess && !intranetMode ) { - // in case that we do a global search or we want to fetch snippets, we check for DoS cases - synchronized ( trackerHandles ) { - final int accInThreeSeconds = - trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size(); - final int accInOneMinute = - trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size(); - final int accInTenMinutes = - trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size(); - // protections against too strong YaCy network load, reduces remote search - if ( global ) { - if ( accInTenMinutes >= 60 || accInOneMinute >= 6 || accInThreeSeconds >= 1 ) { - global = false; - Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " - + client - + ": " - + accInThreeSeconds - + "/3s, " - + accInOneMinute - + "/60s, " - + accInTenMinutes - + "/600s, " - + " requests, disallowed global search"); - } - } - // protection against too many remote server snippet loads (protects traffic on server) - if ( snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ) { - if ( accInTenMinutes >= 20 || accInOneMinute >= 4 || accInThreeSeconds >= 1 ) { - snippetFetchStrategy = CacheStrategy.CACHEONLY; - Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " - + client - + ": " - + accInThreeSeconds - + "/3s, " - + accInOneMinute - + "/60s, " - + accInTenMinutes - + "/600s, " - + " requests, disallowed remote snippet loading"); - } - } - // general load protection - if ( accInTenMinutes >= 3000 || accInOneMinute >= 600 || accInThreeSeconds >= 60 ) { - block = true; - Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " - + client - + ": " - + accInThreeSeconds - + "/3s, " - + accInOneMinute - + "/60s, " - + accInTenMinutes - + "/600s, " - + " requests, disallowed search"); - } - } - } - - if ( !block && (post == null || post.get("cat", "href").equals("href")) ) { - String urlmask = null; - String protocol = null; - String tld = null; - String ext = null; - - // check available memory and clean up if necessary - if ( !MemoryControl.request(8000000L, false) ) { - indexSegment.fulltext().clearCache(); - SearchEventCache.cleanupEvents(false); - } - - final RankingProfile ranking = sb.getRanking(); - final StringBuilder modifier = new StringBuilder(20); - - int stp = querystring.indexOf('*'); - if (stp >= 0) { - querystring = querystring.substring(0, stp) + Segment.catchallString + querystring.substring(stp + 1); - } - if ( querystring.indexOf("/near", 0) >= 0 ) { - querystring = querystring.replace("/near", ""); - ranking.allZero(); // switch off all attributes - ranking.coeff_worddistance = RankingProfile.COEFF_MAX; - modifier.append("/near "); - } - if ( querystring.indexOf("/date", 0) >= 0 ) { - querystring = querystring.replace("/date", ""); - ranking.allZero(); // switch off all attributes - ranking.coeff_date = RankingProfile.COEFF_MAX; - modifier.append("/date "); - } - if ( querystring.indexOf("/https", 0) >= 0 ) { - querystring = querystring.replace("/https", ""); - protocol = "https"; - modifier.append("/https "); - } else if ( querystring.indexOf("/http", 0) >= 0 ) { - querystring = querystring.replace("/http", ""); - protocol = "http"; - modifier.append("/http "); - } - if ( querystring.indexOf("/ftp", 0) >= 0 ) { - querystring = querystring.replace("/ftp", ""); - protocol = "ftp"; - modifier.append("/ftp "); - } - if ( querystring.indexOf("/smb", 0) >= 0 ) { - querystring = querystring.replace("/smb", ""); - protocol = "smb"; - modifier.append("/smb "); - } - - if ( querystring.indexOf("/file", 0) >= 0 ) { - querystring = querystring.replace("/file", ""); - protocol = "file"; - modifier.append("/file "); - } - - if ( querystring.indexOf("/location", 0) >= 0 ) { - querystring = querystring.replace("/location", ""); - if ( constraint == null ) { - constraint = new Bitfield(4); - } - constraint.set(Condenser.flag_cat_haslocation, true); - modifier.append("/location "); - } - - final int lrp = querystring.indexOf("/language/", 0); - String language = ""; - if ( lrp >= 0 ) { - if ( querystring.length() >= (lrp + 12) ) { - language = querystring.substring(lrp + 10, lrp + 12); - } - querystring = querystring.replace("/language/" + language, ""); - language = language.toLowerCase(); - modifier.append("/language/").append(language).append(' '); - } - - final int inurl = querystring.indexOf("inurl:", 0); - if ( inurl >= 0 ) { - int ftb = querystring.indexOf(' ', inurl); - if ( ftb == -1 ) { - ftb = querystring.length(); - } - final String urlstr = querystring.substring(inurl + 6, ftb); - querystring = querystring.replace("inurl:" + urlstr, ""); - if ( !urlstr.isEmpty() ) { - urlmask = urlmask == null ? ".*" + urlstr + ".*" : urlmask + urlstr + ".*"; - } - modifier.append("inurl:").append(urlstr).append(' '); - } - - final int filetype = querystring.indexOf("filetype:", 0); - if ( filetype >= 0 ) { - int ftb = querystring.indexOf(' ', filetype); - if ( ftb == -1 ) { - ftb = querystring.length(); - } - ext = querystring.substring(filetype + 9, ftb); - querystring = querystring.replace("filetype:" + ext, ""); - while ( !ext.isEmpty() && ext.charAt(0) == '.' ) { - ext = ext.substring(1); - } - modifier.append("filetype:").append(ext).append(' '); - if (ext.isEmpty()) ext = null; - } - - int voc = 0; - Collection metatags = new ArrayList(1); - while ((voc = querystring.indexOf("/vocabulary/", 0)) >= 0) { - String vocabulary = ""; - int ve = querystring.indexOf(' ', voc + 12); - if (ve < 0) { - vocabulary = querystring.substring(voc); - querystring = querystring.substring(0, voc).trim(); - } else { - vocabulary = querystring.substring(voc, ve); - querystring = querystring.substring(0, voc) + querystring.substring(ve); - } - modifier.append(vocabulary).append(' '); - vocabulary = vocabulary.substring(12); - int p = vocabulary.indexOf('/'); - if (p > 0) { - String k = vocabulary.substring(0, p); - String v = vocabulary.substring(p + 1); - metatags.add(LibraryProvider.autotagging.metatag(k, v)); - } - } - - int radius = 0; - double lon = 0.0d, lat = 0.0d, rad = 0.0d; - if ((radius = querystring.indexOf("/radius/")) >= 0) { - int ve = querystring.indexOf(' ', radius + 8); - String geo = ""; - if (ve < 0) { - geo = querystring.substring(radius); - querystring = querystring.substring(0, radius).trim(); - } else { - geo = querystring.substring(radius, ve); - querystring = querystring.substring(0, radius) + querystring.substring(ve); - } - geo = geo.substring(8); - String[] sp = geo.split("/"); - if (sp.length == 3) try { - lat = Double.parseDouble(sp[0]); - lon = Double.parseDouble(sp[1]); - rad = Double.parseDouble(sp[2]); - } catch (NumberFormatException e) { - lon = 0.0d; lat = 0.0d; rad = 0.0d; - } - } - - final int site = querystring.indexOf("site:", 0); - String sitehash = null; - String sitehost = null; - if ( site >= 0 ) { - int ftb = querystring.indexOf(' ', site); - if ( ftb == -1 ) { - ftb = querystring.length(); - } - sitehost = querystring.substring(site + 5, ftb); - querystring = querystring.replace("site:" + sitehost, ""); - while ( sitehost.length() > 0 && sitehost.charAt(0) == '.' ) { - sitehost = sitehost.substring(1); - } - while ( sitehost.endsWith(".") ) { - sitehost = sitehost.substring(0, sitehost.length() - 1); - } - sitehash = DigestURI.hosthash(sitehost); - modifier.append("site:").append(sitehost).append(' '); - } - - final int heuristicBlekko = querystring.indexOf("/heuristic/blekko", 0); - if ( heuristicBlekko >= 0 ) { - querystring = querystring.replace("/heuristic/blekko", ""); - modifier.append("/heuristic/blekko "); - } - - final int heuristicTwitter = querystring.indexOf("/heuristic/twitter", 0); - if ( heuristicBlekko >= 0 ) { - querystring = querystring.replace("/heuristic/twitter", ""); - modifier.append("/heuristic/twitter "); - } - - final int authori = querystring.indexOf("author:", 0); - String author = null; - if ( authori >= 0 ) { - // check if the author was given with single quotes or without - final boolean quotes = (querystring.charAt(authori + 7) == '('); - if ( quotes ) { - int ftb = querystring.indexOf(')', authori + 8); - if (ftb == -1) ftb = querystring.length() + 1; - author = querystring.substring(authori + 8, ftb); - querystring = querystring.replace("author:(" + author + ")", ""); - modifier.append("author:(").append(author).append(") "); - } else { - int ftb = querystring.indexOf(' ', authori); - if ( ftb == -1 ) { - ftb = querystring.length(); - } - author = querystring.substring(authori + 7, ftb); - querystring = querystring.replace("author:" + author, ""); - modifier.append("author:").append(author).append(' '); - } - } - - final int tldp = querystring.indexOf("tld:", 0); - if (tldp >= 0) { - int ftb = querystring.indexOf(' ', tldp); - if (ftb == -1) ftb = querystring.length(); - tld = querystring.substring(tldp + 4, ftb); - querystring = querystring.replace("tld:" + tld, ""); - modifier.append("tld:").append(tld).append(' '); - while ( tld.length() > 0 && tld.charAt(0) == '.' ) { - tld = tld.substring(1); - } - if (tld.length() == 0) tld = null; - } - if (urlmask == null || urlmask.isEmpty()) urlmask = ".*"; //if no urlmask was given - - // read the language from the language-restrict option 'lr' - // if no one is given, use the user agent or the system language as default - language = (post == null) ? language : post.get("lr", language); - if ( language.startsWith("lang_") ) { - language = language.substring(5); - } - if ( !ISO639.exists(language) ) { - // find out language of the user by reading of the user-agent string - String agent = header.get(HeaderFramework.ACCEPT_LANGUAGE); - if ( agent == null ) { - agent = System.getProperty("user.language"); - } - language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent); - if ( language == null ) { - language = "en"; - } - } - - // the query - final QueryGoal qg = new QueryGoal(originalquerystring, querystring.trim()); - final int maxDistance = (querystring.indexOf('"', 0) >= 0) ? qg.getAllHashes().size() - 1 : Integer.MAX_VALUE; - - // filter out stopwords - final SortedSet filtered = SetTools.joinConstructiveByTest(qg.getIncludeStrings(), Switchboard.stopwords); - if ( !filtered.isEmpty() ) { - SetTools.excludeDestructiveByTestSmallInLarge(qg.getIncludeStrings(), Switchboard.stopwords); - } - - // if a minus-button was hit, remove a special reference first - if ( post != null && post.containsKey("deleteref") ) { - try { - if ( !sb.verifyAuthentication(header) ) { - prop.authenticationRequired(); - return prop; - } - - // delete the index entry locally - final String delHash = post.get("deleteref", ""); // urlhash - indexSegment.termIndex().remove(qg.getIncludeHashes(), delHash.getBytes()); - - // make new news message with negative voting - if ( !sb.isRobinsonMode() ) { - final Map map = new HashMap(); - map.put("urlhash", delHash); - map.put("vote", "negative"); - map.put("refid", ""); - sb.peers.newsPool.publishMyNews( - sb.peers.mySeed(), - NewsPool.CATEGORY_SURFTIPP_VOTE_ADD, - map); - } - - // delete the search history since this still shows the entry - SearchEventCache.delete(delHash); - } catch ( final IOException e ) { - Log.logException(e); - } - } - - // if a plus-button was hit, create new voting message - if ( post != null && post.containsKey("recommendref") ) { - if ( !sb.verifyAuthentication(header) ) { - prop.authenticationRequired(); - return prop; - } - final String recommendHash = post.get("recommendref", ""); // urlhash - final URIMetadataNode urlentry = indexSegment.fulltext().getMetadata(UTF8.getBytes(recommendHash)); - if ( urlentry != null ) { - Document[] documents = null; - try { - documents = - sb.loader.loadDocuments( - sb.loader.request(urlentry.url(), true, false), - CacheStrategy.IFEXIST, - Integer.MAX_VALUE, BlacklistType.SEARCH, TextSnippet.snippetMinLoadDelay); - } catch ( final IOException e ) { - } catch ( final Parser.Failure e ) { - } - if ( documents != null ) { - // create a news message - final Map map = new HashMap(); - map.put("url", urlentry.url().toNormalform(true).replace(',', '|')); - map.put("title", urlentry.dc_title().replace(',', ' ')); - map.put("description", documents[0].dc_title().replace(',', ' ')); - map.put("author", documents[0].dc_creator()); - map.put("tags", documents[0].dc_subject(' ')); - sb.peers.newsPool.publishMyNews( - sb.peers.mySeed(), - NewsPool.CATEGORY_SURFTIPP_ADD, - map); - documents[0].close(); - } - } - } - - // if a bookmarks-button was hit, create new bookmark entry - if ( post != null && post.containsKey("bookmarkref") ) { - if ( !sb.verifyAuthentication(header) ) { - prop.authenticationRequired(); - return prop; - } - final String bookmarkHash = post.get("bookmarkref", ""); // urlhash - final DigestURI url = indexSegment.fulltext().getURL(UTF8.getBytes(bookmarkHash)); - if ( url != null ) { - try { - sb.tables.bookmarks.createBookmark( - sb.loader, - url, - YMarkTables.USER_ADMIN, - true, - "searchresult", - "/search"); - } catch ( final Throwable e ) { - } - } - } - - // check filters - try { - Pattern.compile(urlmask); - } catch ( final PatternSyntaxException ex ) { - SearchEvent.log.logWarning("Illegal URL mask, not a valid regex: " + urlmask); - prop.put("urlmaskerror", 1); - prop.putHTML("urlmaskerror_urlmask", urlmask); - urlmask = ".*"; - } - - try { - Pattern.compile(prefermask); - } catch ( final PatternSyntaxException ex ) { - SearchEvent.log.logWarning("Illegal prefer mask, not a valid regex: " + prefermask); - prop.put("prefermaskerror", 1); - prop.putHTML("prefermaskerror_prefermask", prefermask); - prefermask = ""; - } - - // do the search - final QueryParams theQuery = - new QueryParams( - qg, - modifier.toString().trim(), - maxDistance, - prefermask, - contentdom, - language, - metatags, - snippetFetchStrategy, - itemsPerPage, - startRecord, - urlmask, protocol, tld, ext, - clustersearch && global ? QueryParams.Searchdom.CLUSTER : (global && indexReceiveGranted - ? QueryParams.Searchdom.GLOBAL - : QueryParams.Searchdom.LOCAL), - 20, - constraint, - true, - sitehash, - sitehost, - DigestURI.hosthashess(sb.getConfig("search.excludehosth", "")), - author, - DigestURI.TLD_any_zone_filter, - client, - authenticated, - indexSegment, - ranking, - header.get(RequestHeader.USER_AGENT, ""), - sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) - && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) - && sb.peers.mySeed().getFlagAcceptRemoteIndex(), - false, - lat, lon, rad); - EventTracker.delete(EventTracker.EClass.SEARCH); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch( - theQuery.id(true), - SearchEventType.INITIALIZATION, - "", - 0, - 0), false); - - // tell all threads to do nothing for a specific time - sb.intermissionAllThreads(3000); - - // filter out words that appear in bluelist - theQuery.getQueryGoal().filterOut(Switchboard.blueList); - - // log - Log.logInfo( - "LOCAL_SEARCH", - "INIT WORD SEARCH: " - + theQuery.getQueryGoal().getOriginalQueryString(false) - + ":" - + QueryParams.hashSet2hashString(theQuery.getQueryGoal().getIncludeHashes()) - + " - " - + theQuery.neededResults() - + " links to be computed, " - + theQuery.itemsPerPage() - + " lines to be displayed"); - EventChannel.channels(EventChannel.LOCALSEARCH).addMessage( - new RSSMessage("Local Search Request", theQuery.getQueryGoal().getOriginalQueryString(false), "")); - final long timestamp = System.currentTimeMillis(); - - // create a new search event - if ( SearchEventCache.getEvent(theQuery.id(false)) == null ) { - theQuery.setOffset(0); // in case that this is a new search, always start without a offset - startRecord = 0; - } - final SearchEvent theSearch = - SearchEventCache.getEvent( - theQuery, - sb.peers, - sb.tables, - (sb.isRobinsonMode()) ? sb.clusterhashes : null, - false, - sb.loader, - (int) sb.getConfigLong( - SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, - sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)), - sb.getConfigLong( - SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, - sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)), - (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), - (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); - - if ( startRecord == 0 ) { - if ( sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated ) { - sb.heuristicSite(theSearch, sitehost); - } - if ( (heuristicBlekko >= 0 || sb.getConfigBool("heuristic.blekko", false)) && authenticated ) { - sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko"); - } - if ( (heuristicTwitter >= 0 || sb.getConfigBool("heuristic.twitter", false)) && authenticated ) { - sb.heuristicRSS("http://search.twitter.com/search.rss?rpp=50&q=$", theSearch, "twitter"); - } - } - - // log - Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " - + theQuery.getQueryGoal().getOriginalQueryString(false) - + " - " - + "local_rwi_available(" + theSearch.query.local_rwi_available.get() + "), " - + "local_rwi_stored(" + theSearch.query.local_rwi_stored.get() + "), " - + "local_solr_available(" + theSearch.query.local_solr_available.get() + "), " - + "local_solr_stored(" + theSearch.query.local_solr_stored.get() + "), " - + "remote_available(" + theSearch.query.remote_available.get() + "), " - + "remote_stored(" + theSearch.query.remote_stored.get() + "), " - + "remote_peerCount(" + theSearch.query.remote_peerCount.get() + "), " - + "local_sortout(" + theSearch.query.misses.size() + "), " - + (System.currentTimeMillis() - timestamp) - + " ms"); - - // prepare search statistics - theQuery.searchtime = System.currentTimeMillis() - timestamp; - theQuery.urlretrievaltime = theSearch.getURLRetrievalTime(); - theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime(); - AccessTracker.add(AccessTracker.Location.local, theQuery); - - // check suggestions - final int meanMax = (post != null) ? post.getInt("meanCount", 0) : 0; - - prop.put("meanCount", meanMax); - if ( meanMax > 0 && !json && !rss ) { - final DidYouMean didYouMean = new DidYouMean(indexSegment, new StringBuilder(querystring)); - final Iterator meanIt = didYouMean.getSuggestions(100, 5).iterator(); - int meanCount = 0; - String suggestion; - try { - meanCollect: while ( meanCount < meanMax && meanIt.hasNext() ) { - try { - suggestion = meanIt.next().toString(); - prop.put("didYouMean_suggestions_" + meanCount + "_word", suggestion); - prop.put( - "didYouMean_suggestions_" + meanCount + "_url", - QueryParams.navurl( - "html", - 0, - theQuery, - suggestion).toString()); - prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|"); - meanCount++; - } catch (ConcurrentModificationException e) {break meanCollect;} - } - } catch (ConcurrentModificationException e) {} - prop.put("didYouMean_suggestions_" + (meanCount - 1) + "_sep", ""); - prop.put("didYouMean", meanCount > 0 ? 1 : 0); - prop.put("didYouMean_suggestions", meanCount); - } else { - prop.put("didYouMean", 0); - } - - // find geographic info - final SortedSet coordinates = LibraryProvider.geoLoc.find(originalquerystring, false); - if ( coordinates == null || coordinates.isEmpty() || startRecord > 0 ) { - prop.put("geoinfo", "0"); - } else { - int i = 0; - for ( final GeoLocation c : coordinates ) { - prop.put("geoinfo_loc_" + i + "_lon", Math.round(c.lon() * 10000.0f) / 10000.0f); - prop.put("geoinfo_loc_" + i + "_lat", Math.round(c.lat() * 10000.0f) / 10000.0f); - prop.put("geoinfo_loc_" + i + "_name", c.getName()); - i++; - if ( i >= 10 ) { - break; - } - } - prop.put("geoinfo_loc", i); - prop.put("geoinfo", "1"); - } - - // update the search tracker - try { - synchronized ( trackerHandles ) { - trackerHandles.add(theQuery.starttime); - while ( trackerHandles.size() > 600 ) { - if ( !trackerHandles.remove(trackerHandles.first()) ) { - break; - } - } - } - sb.localSearchTracker.put(client, trackerHandles); - if ( sb.localSearchTracker.size() > 100 ) { - sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement()); - } - if ( MemoryControl.shortStatus() ) { - sb.localSearchTracker.clear(); - } - } catch ( final Exception e ) { - Log.logException(e); - } - - prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1); - prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.query.itemsPerPage > theSearch.query.getResultCount() ? startRecord + theSearch.query.getResultCount() % theSearch.query.itemsPerPage : startRecord + theSearch.query.itemsPerPage, true)); - prop.put("num-results_itemsPerPage", Formatter.number(itemsPerPage)); - prop.put("num-results_totalcount", Formatter.number(theSearch.query.getResultCount())); - prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0"); - prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true)); - prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.query.misses.size(), true)); - prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true)); - prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true)); - prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); - - // compose page navigation - final StringBuilder resnav = new StringBuilder(200); - final int thispage = startRecord / theQuery.itemsPerPage(); - if ( thispage == 0 ) { - resnav - .append("\"arrowleft\" "); - } else { - resnav.append("\"arrowleft\" "); - } - final int numberofpages = Math.min(10, 1 + ((theSearch.query.getResultCount() - 1) / theQuery.itemsPerPage())); - - for ( int i = 0; i < numberofpages; i++ ) { - if ( i == thispage ) { - resnav.append("\"page"); "); - } else { - resnav.append("\"page"); "); - } - } - if ( thispage >= numberofpages ) { - resnav - .append("\"arrowright\""); - } else { - resnav.append("\"arrowright\""); - } - final String resnavs = resnav.toString(); - prop.put("num-results_resnav", resnavs); - prop.put("pageNavBottom", (theSearch.query.getResultCount() - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom - prop.put("pageNavBottom_resnav", resnavs); - - // generate the search result lines; the content will be produced by another servlet - for ( int i = 0; i < theQuery.itemsPerPage(); i++ ) { - prop.put("results_" + i + "_item", startRecord + i); - prop.put("results_" + i + "_eventID", theQuery.id(false)); - } - prop.put("results", theQuery.itemsPerPage()); - prop - .put( - "resultTable", - (contentdom == ContentDomain.APP || contentdom == ContentDomain.AUDIO || contentdom == ContentDomain.VIDEO) - ? 1 - : 0); - prop.put("eventID", theQuery.id(false)); // for bottomline - - // process result of search - if ( !filtered.isEmpty() ) { - prop.put("excluded", "1"); - prop.putHTML("excluded_stopwords", filtered.toString()); - } else { - prop.put("excluded", "0"); - } - - if ( prop == null || prop.isEmpty() ) { - if ( post.get("query", post.get("search", "")).length() < 2 ) { - prop.put("num-results", "2"); // no results - at least 2 chars - } else { - prop.put("num-results", "1"); // no results - } - } else { - prop.put("num-results", "3"); - } - - prop.put("cat", "href"); - prop.put("depth", "0"); - - // adding some additional properties needed for the rss feed - String hostName = header.get("Host", Domains.LOCALHOST); - if ( hostName.indexOf(':', 0) == -1 ) { - hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090")); - } - prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html"); - prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.gif"); - prop.put("thisaddress", hostName); - } - - prop.put("searchagain", global ? "1" : "0"); - prop.putHTML("former", originalquerystring); - prop.put("count", itemsPerPage); - prop.put("offset", startRecord); - prop.put("resource", global ? "global" : "local"); - prop.putHTML("prefermaskfilter", prefermask); - prop.put("indexof", (indexof) ? "on" : "off"); - prop.put("constraint", (constraint == null) ? "" : constraint.exportB64()); - prop.put("search.verify", snippetFetchStrategy == null - ? sb.getConfig("search.verify", "iffresh") - : snippetFetchStrategy.toName()); - prop.put( - "search.navigation", - (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all")); - prop.put("contentdom", (post == null ? "text" : post.get("contentdom", "text"))); - prop.put( - "searchdomswitches", - sb.getConfigBool("search.text", true) - || sb.getConfigBool("search.audio", true) - || sb.getConfigBool("search.video", true) - || sb.getConfigBool("search.image", true) - || sb.getConfigBool("search.app", true) ? 1 : 0); - prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0); - prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0); - prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0); - prop.put("searchdomswitches_searchimage", sb.getConfigBool("search.image", true) ? 1 : 0); - prop.put("searchdomswitches_searchapp", sb.getConfigBool("search.app", true) ? 1 : 0); - prop.put("searchdomswitches_searchtext_check", (contentdom == ContentDomain.TEXT || contentdom == ContentDomain.ALL) ? "1" : "0"); - prop.put("searchdomswitches_searchaudio_check", (contentdom == ContentDomain.AUDIO) ? "1" : "0"); - prop.put("searchdomswitches_searchvideo_check", (contentdom == ContentDomain.VIDEO) ? "1" : "0"); - prop.put("searchdomswitches_searchimage_check", (contentdom == ContentDomain.IMAGE) ? "1" : "0"); - prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0"); - - // copy properties for "more options" link - prop.put("searchdomswitches_count", prop.get("count")); - prop.put("searchdomswitches_urlmaskfilter", prop.get("urlmaskfilter")); - prop.put("searchdomswitches_prefermaskfilter", prop.get("prefermaskfilter")); - prop.put("searchdomswitches_cat", prop.get("cat")); - prop.put("searchdomswitches_constraint", prop.get("constraint")); - prop.put("searchdomswitches_contentdom", prop.get("contentdom")); - prop.put("searchdomswitches_former", prop.get("former")); - prop.put("searchdomswitches_meanCount", prop.get("meanCount")); - - // for RSS: don't HTML encode some elements - prop.putXML("rss_query", originalquerystring); - prop.putXML("rss_queryenc", originalquerystring.replace(' ', '+')); - - sb.localSearchLastAccess = System.currentTimeMillis(); - - // hostname and port (assume locahost if nothing helps) - final InetAddress hostIP = Domains.myPublicLocalIP(); - prop.put("myhost", hostIP != null ? hostIP.getHostAddress() : Domains.LOCALHOST); - prop.put("myport", serverCore.getPortNr(sb.getConfig("port", "8090"))); - - // return rewrite properties - return prop; - } -} +// yacysearch.java +// ----------------------- +// part of the AnomicHTTPD caching proxy +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// You must compile this file with +// javac -classpath .:../classes yacysearch.java +// if the shell's current path is HTROOT + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.UTF8; +import net.yacy.cora.federate.yacy.CacheStrategy; +import net.yacy.cora.geo.GeoLocation; +import net.yacy.cora.lod.vocabulary.Tagging; +import net.yacy.cora.protocol.Domains; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.protocol.ResponseHeader; +import net.yacy.data.DidYouMean; +import net.yacy.data.UserDB; +import net.yacy.data.ymark.YMarkTables; +import net.yacy.document.Condenser; +import net.yacy.document.Document; +import net.yacy.document.LibraryProvider; +import net.yacy.document.Parser; +import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.data.meta.URIMetadataNode; +import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.Bitfield; +import net.yacy.kelondro.util.Formatter; +import net.yacy.kelondro.util.ISO639; +import net.yacy.kelondro.util.MemoryControl; +import net.yacy.kelondro.util.SetTools; +import net.yacy.peers.EventChannel; +import net.yacy.peers.NewsPool; +import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.repository.Blacklist.BlacklistType; +import net.yacy.search.EventTracker; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; +import net.yacy.search.index.Segment; +import net.yacy.search.query.AccessTracker; +import net.yacy.search.query.QueryGoal; +import net.yacy.search.query.QueryParams; +import net.yacy.search.query.SearchEvent; +import net.yacy.search.query.SearchEventCache; +import net.yacy.search.query.SearchEventType; +import net.yacy.search.ranking.RankingProfile; +import net.yacy.search.snippet.TextSnippet; +import net.yacy.server.serverCore; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; +import net.yacy.server.servletProperties; + +public class yacysearch { + + public static serverObjects respond( + final RequestHeader header, + final serverObjects post, + final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + sb.localSearchLastAccess = System.currentTimeMillis(); + + final boolean searchAllowed = + sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header); + + boolean authenticated = sb.adminAuthenticated(header) >= 2; + if ( !authenticated ) { + final UserDB.Entry user = sb.userDB.getUser(header); + authenticated = (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT)); + } + final boolean localhostAccess = header.accessFromLocalhost(); + final String promoteSearchPageGreeting = + (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig( + "network.unit.description", + "") : env.getConfig(SwitchboardConstants.GREETING, ""); + final String client = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search + + // get query + final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); + String querystring = originalquerystring.replace('+', ' ').trim(); + CacheStrategy snippetFetchStrategy = (post == null) ? null : CacheStrategy.parse(post.get("verify", sb.getConfig("search.verify", ""))); + if (authenticated && originalquerystring.length() == 0) sb.index.fulltext().commit(); + + final servletProperties prop = new servletProperties(); + prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); + + //get focus option + prop.put("focus", ((post == null) ? true : post.get("focus", "1").equals("1")) ? 1 : 0); + + // produce vocabulary navigation sidebars + Collection vocabularies = LibraryProvider.autotagging.getVocabularies(); + int j = 0; + for (Tagging v: vocabularies) { + prop.put("sidebarVocabulary_" + j + "_vocabulary", v.getName()); + j++; + } + prop.put("sidebarVocabulary", j); + + // get segment + Segment indexSegment = sb.index; + + final String EXT = header.get("EXT", ""); + final boolean rss = EXT.equals("rss"); + final boolean json = EXT.equals("json"); + prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting); + prop.put( + "promoteSearchPageGreeting.homepage", + sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, "")); + prop.put( + "promoteSearchPageGreeting.smallImage", + sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, "")); + if ( post == null || indexSegment == null || env == null || !searchAllowed ) { + // we create empty entries for template strings + prop.put("searchagain", "0"); + prop.put("former", ""); + prop.put("count", "10"); + prop.put("offset", "0"); + prop.put("resource", "global"); + prop.put("urlmaskfilter", (post == null) ? ".*" : post.get("urlmaskfilter", ".*")); + prop.put("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", "")); + prop.put("tenant", (post == null) ? "" : post.get("tenant", "")); + prop.put("indexof", "off"); + prop.put("constraint", ""); + prop.put("cat", "href"); + prop.put("depth", "0"); + prop.put( + "search.verify", + (post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh")); + prop.put( + "search.navigation", + (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all")); + prop.put("contentdom", "text"); + prop.put("contentdomCheckText", "1"); + prop.put("contentdomCheckAudio", "0"); + prop.put("contentdomCheckVideo", "0"); + prop.put("contentdomCheckImage", "0"); + prop.put("contentdomCheckApp", "0"); + prop.put("excluded", "0"); + prop.put("results", ""); + prop.put("resultTable", "0"); + prop.put("num-results", searchAllowed ? "0" : "4"); + prop.put("num-results_totalcount", 0); + prop.put("num-results_offset", 0); + prop.put("num-results_itemsPerPage", 10); + prop.put("geoinfo", "0"); + prop.put("rss_queryenc", ""); + prop.put("meanCount", 5); + return prop; + } + + // check for JSONP + if ( post.containsKey("callback") ) { + final String jsonp = post.get("callback") + "(["; + prop.put("jsonp-start", jsonp); + prop.put("jsonp-end", "])"); + } else { + prop.put("jsonp-start", ""); + prop.put("jsonp-end", ""); + } + + // Adding CORS Access header for yacysearch.rss output + if ( rss ) { + final ResponseHeader outgoingHeader = new ResponseHeader(200); + outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*"); + prop.setOutgoingHeader(outgoingHeader); + } + + // collect search attributes + + int itemsPerPage = + Math.min( + (authenticated) + ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() + ? 100 + : 5000) : (snippetFetchStrategy != null + && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 1000), + post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative + int startRecord = post.getInt("startRecord", post.getInt("offset", 0)); + + boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0; + final boolean indexof = (post != null && post.get("indexof", "").equals("on")); + + String prefermask = (post == null) ? "" : post.get("prefermaskfilter", ""); + if ( !prefermask.isEmpty() && prefermask.indexOf(".*", 0) < 0 ) { + prefermask = ".*" + prefermask + ".*"; + } + + Bitfield constraint = + (post != null && post.containsKey("constraint") && !post.get("constraint", "").isEmpty()) + ? new Bitfield(4, post.get("constraint", "______")) + : null; + if ( indexof ) { + constraint = new Bitfield(4); + constraint.set(Condenser.flag_cat_indexof, true); + } + + // SEARCH + final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER); + final boolean indexReceiveGranted = + sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true) + || sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true) + || clustersearch; + global = global && indexReceiveGranted; // if the user does not want indexes from remote peers, it cannot be a global searchnn + final boolean intranetMode = sb.isIntranetMode() || sb.isAllIPMode(); + + // increase search statistic counter + if ( !global ) { + // we count only searches on the local peer here, because global searches + // are counted on the target peer to preserve privacy of the searcher + if ( authenticated ) { + // local or authenticated search requests are counted separately + // because they are not part of a public available peer statistic + sb.searchQueriesRobinsonFromLocal++; + } else { + // robinson-searches from non-authenticated requests are public + // and may be part of the public available statistic + sb.searchQueriesRobinsonFromRemote++; + } + } + + // find search domain + final Classification.ContentDomain contentdom = + ContentDomain.contentdomParser(post == null ? "all" : post.get("contentdom", "all")); + + // patch until better search profiles are available + if (contentdom == ContentDomain.IMAGE && (itemsPerPage == 10 || itemsPerPage == 100)) { + itemsPerPage = 64; + } else if ( contentdom != ContentDomain.IMAGE && itemsPerPage > 50 && itemsPerPage < 100 ) { + itemsPerPage = 10; + } + + // check the search tracker + TreeSet trackerHandles = sb.localSearchTracker.get(client); + if ( trackerHandles == null ) { + trackerHandles = new TreeSet(); + } + boolean block = false; + if ( Domains.matchesList(client, sb.networkBlacklist) ) { + global = false; + if ( snippetFetchStrategy != null ) { + snippetFetchStrategy = null; + } + block = true; + Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM " + + client + + " gets no permission to search"); + } else if ( Domains.matchesList(client, sb.networkWhitelist) ) { + Log.logInfo("LOCAL_SEARCH", "ACCESS CONTROL: WHITELISTED CLIENT FROM " + + client + + " gets no search restrictions"); + } else if ( !authenticated && !localhostAccess && !intranetMode ) { + // in case that we do a global search or we want to fetch snippets, we check for DoS cases + synchronized ( trackerHandles ) { + final int accInThreeSeconds = + trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size(); + final int accInOneMinute = + trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size(); + final int accInTenMinutes = + trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size(); + // protections against too strong YaCy network load, reduces remote search + if ( global ) { + if ( accInTenMinutes >= 60 || accInOneMinute >= 6 || accInThreeSeconds >= 1 ) { + global = false; + Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + + client + + ": " + + accInThreeSeconds + + "/3s, " + + accInOneMinute + + "/60s, " + + accInTenMinutes + + "/600s, " + + " requests, disallowed global search"); + } + } + // protection against too many remote server snippet loads (protects traffic on server) + if ( snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ) { + if ( accInTenMinutes >= 20 || accInOneMinute >= 4 || accInThreeSeconds >= 1 ) { + snippetFetchStrategy = CacheStrategy.CACHEONLY; + Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + + client + + ": " + + accInThreeSeconds + + "/3s, " + + accInOneMinute + + "/60s, " + + accInTenMinutes + + "/600s, " + + " requests, disallowed remote snippet loading"); + } + } + // general load protection + if ( accInTenMinutes >= 3000 || accInOneMinute >= 600 || accInThreeSeconds >= 60 ) { + block = true; + Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + + client + + ": " + + accInThreeSeconds + + "/3s, " + + accInOneMinute + + "/60s, " + + accInTenMinutes + + "/600s, " + + " requests, disallowed search"); + } + } + } + + if ( !block && (post == null || post.get("cat", "href").equals("href")) ) { + String urlmask = null; + String protocol = null; + String tld = null; + String ext = null; + + // check available memory and clean up if necessary + if ( !MemoryControl.request(8000000L, false) ) { + indexSegment.fulltext().clearCache(); + SearchEventCache.cleanupEvents(false); + } + + final RankingProfile ranking = sb.getRanking(); + final StringBuilder modifier = new StringBuilder(20); + + int stp = querystring.indexOf('*'); + if (stp >= 0) { + querystring = querystring.substring(0, stp) + Segment.catchallString + querystring.substring(stp + 1); + } + if ( querystring.indexOf("/near", 0) >= 0 ) { + querystring = querystring.replace("/near", ""); + ranking.allZero(); // switch off all attributes + ranking.coeff_worddistance = RankingProfile.COEFF_MAX; + modifier.append("/near "); + } + if ( querystring.indexOf("/date", 0) >= 0 ) { + querystring = querystring.replace("/date", ""); + ranking.allZero(); // switch off all attributes + ranking.coeff_date = RankingProfile.COEFF_MAX; + modifier.append("/date "); + } + if ( querystring.indexOf("/https", 0) >= 0 ) { + querystring = querystring.replace("/https", ""); + protocol = "https"; + modifier.append("/https "); + } else if ( querystring.indexOf("/http", 0) >= 0 ) { + querystring = querystring.replace("/http", ""); + protocol = "http"; + modifier.append("/http "); + } + if ( querystring.indexOf("/ftp", 0) >= 0 ) { + querystring = querystring.replace("/ftp", ""); + protocol = "ftp"; + modifier.append("/ftp "); + } + if ( querystring.indexOf("/smb", 0) >= 0 ) { + querystring = querystring.replace("/smb", ""); + protocol = "smb"; + modifier.append("/smb "); + } + + if ( querystring.indexOf("/file", 0) >= 0 ) { + querystring = querystring.replace("/file", ""); + protocol = "file"; + modifier.append("/file "); + } + + if ( querystring.indexOf("/location", 0) >= 0 ) { + querystring = querystring.replace("/location", ""); + if ( constraint == null ) { + constraint = new Bitfield(4); + } + constraint.set(Condenser.flag_cat_haslocation, true); + modifier.append("/location "); + } + + final int lrp = querystring.indexOf("/language/", 0); + String language = ""; + if ( lrp >= 0 ) { + if ( querystring.length() >= (lrp + 12) ) { + language = querystring.substring(lrp + 10, lrp + 12); + } + querystring = querystring.replace("/language/" + language, ""); + language = language.toLowerCase(); + modifier.append("/language/").append(language).append(' '); + } + + final int inurl = querystring.indexOf("inurl:", 0); + if ( inurl >= 0 ) { + int ftb = querystring.indexOf(' ', inurl); + if ( ftb == -1 ) { + ftb = querystring.length(); + } + final String urlstr = querystring.substring(inurl + 6, ftb); + querystring = querystring.replace("inurl:" + urlstr, ""); + if ( !urlstr.isEmpty() ) { + urlmask = urlmask == null ? ".*" + urlstr + ".*" : urlmask + urlstr + ".*"; + } + modifier.append("inurl:").append(urlstr).append(' '); + } + + final int filetype = querystring.indexOf("filetype:", 0); + if ( filetype >= 0 ) { + int ftb = querystring.indexOf(' ', filetype); + if ( ftb == -1 ) { + ftb = querystring.length(); + } + ext = querystring.substring(filetype + 9, ftb); + querystring = querystring.replace("filetype:" + ext, ""); + while ( !ext.isEmpty() && ext.charAt(0) == '.' ) { + ext = ext.substring(1); + } + modifier.append("filetype:").append(ext).append(' '); + if (ext.isEmpty()) ext = null; + } + + int voc = 0; + Collection metatags = new ArrayList(1); + while ((voc = querystring.indexOf("/vocabulary/", 0)) >= 0) { + String vocabulary = ""; + int ve = querystring.indexOf(' ', voc + 12); + if (ve < 0) { + vocabulary = querystring.substring(voc); + querystring = querystring.substring(0, voc).trim(); + } else { + vocabulary = querystring.substring(voc, ve); + querystring = querystring.substring(0, voc) + querystring.substring(ve); + } + modifier.append(vocabulary).append(' '); + vocabulary = vocabulary.substring(12); + int p = vocabulary.indexOf('/'); + if (p > 0) { + String k = vocabulary.substring(0, p); + String v = vocabulary.substring(p + 1); + metatags.add(LibraryProvider.autotagging.metatag(k, v)); + } + } + + int radius = 0; + double lon = 0.0d, lat = 0.0d, rad = 0.0d; + if ((radius = querystring.indexOf("/radius/")) >= 0) { + int ve = querystring.indexOf(' ', radius + 8); + String geo = ""; + if (ve < 0) { + geo = querystring.substring(radius); + querystring = querystring.substring(0, radius).trim(); + } else { + geo = querystring.substring(radius, ve); + querystring = querystring.substring(0, radius) + querystring.substring(ve); + } + geo = geo.substring(8); + String[] sp = geo.split("/"); + if (sp.length == 3) try { + lat = Double.parseDouble(sp[0]); + lon = Double.parseDouble(sp[1]); + rad = Double.parseDouble(sp[2]); + } catch (NumberFormatException e) { + lon = 0.0d; lat = 0.0d; rad = 0.0d; + } + } + + final int site = querystring.indexOf("site:", 0); + String sitehash = null; + String sitehost = null; + if ( site >= 0 ) { + int ftb = querystring.indexOf(' ', site); + if ( ftb == -1 ) { + ftb = querystring.length(); + } + sitehost = querystring.substring(site + 5, ftb); + querystring = querystring.replace("site:" + sitehost, ""); + while ( sitehost.length() > 0 && sitehost.charAt(0) == '.' ) { + sitehost = sitehost.substring(1); + } + while ( sitehost.endsWith(".") ) { + sitehost = sitehost.substring(0, sitehost.length() - 1); + } + sitehash = DigestURI.hosthash(sitehost); + modifier.append("site:").append(sitehost).append(' '); + } + + final int heuristicBlekko = querystring.indexOf("/heuristic/blekko", 0); + if ( heuristicBlekko >= 0 ) { + querystring = querystring.replace("/heuristic/blekko", ""); + modifier.append("/heuristic/blekko "); + } + + final int heuristicTwitter = querystring.indexOf("/heuristic/twitter", 0); + if ( heuristicBlekko >= 0 ) { + querystring = querystring.replace("/heuristic/twitter", ""); + modifier.append("/heuristic/twitter "); + } + + final int authori = querystring.indexOf("author:", 0); + String author = null; + if ( authori >= 0 ) { + // check if the author was given with single quotes or without + final boolean quotes = (querystring.charAt(authori + 7) == '('); + if ( quotes ) { + int ftb = querystring.indexOf(')', authori + 8); + if (ftb == -1) ftb = querystring.length() + 1; + author = querystring.substring(authori + 8, ftb); + querystring = querystring.replace("author:(" + author + ")", ""); + modifier.append("author:(").append(author).append(") "); + } else { + int ftb = querystring.indexOf(' ', authori); + if ( ftb == -1 ) { + ftb = querystring.length(); + } + author = querystring.substring(authori + 7, ftb); + querystring = querystring.replace("author:" + author, ""); + modifier.append("author:").append(author).append(' '); + } + } + + final int tldp = querystring.indexOf("tld:", 0); + if (tldp >= 0) { + int ftb = querystring.indexOf(' ', tldp); + if (ftb == -1) ftb = querystring.length(); + tld = querystring.substring(tldp + 4, ftb); + querystring = querystring.replace("tld:" + tld, ""); + modifier.append("tld:").append(tld).append(' '); + while ( tld.length() > 0 && tld.charAt(0) == '.' ) { + tld = tld.substring(1); + } + if (tld.length() == 0) tld = null; + } + if (urlmask == null || urlmask.isEmpty()) urlmask = ".*"; //if no urlmask was given + + // read the language from the language-restrict option 'lr' + // if no one is given, use the user agent or the system language as default + language = (post == null) ? language : post.get("lr", language); + if ( language.startsWith("lang_") ) { + language = language.substring(5); + } + if ( !ISO639.exists(language) ) { + // find out language of the user by reading of the user-agent string + String agent = header.get(HeaderFramework.ACCEPT_LANGUAGE); + if ( agent == null ) { + agent = System.getProperty("user.language"); + } + language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent); + if ( language == null ) { + language = "en"; + } + } + + // the query + final QueryGoal qg = new QueryGoal(originalquerystring, querystring.trim()); + final int maxDistance = (querystring.indexOf('"', 0) >= 0) ? qg.getAllHashes().size() - 1 : Integer.MAX_VALUE; + + // filter out stopwords + final SortedSet filtered = SetTools.joinConstructiveByTest(qg.getIncludeStrings(), Switchboard.stopwords); + if ( !filtered.isEmpty() ) { + SetTools.excludeDestructiveByTestSmallInLarge(qg.getIncludeStrings(), Switchboard.stopwords); + } + + // if a minus-button was hit, remove a special reference first + if ( post != null && post.containsKey("deleteref") ) { + try { + if ( !sb.verifyAuthentication(header) ) { + prop.authenticationRequired(); + return prop; + } + + // delete the index entry locally + final String delHash = post.get("deleteref", ""); // urlhash + indexSegment.termIndex().remove(qg.getIncludeHashes(), delHash.getBytes()); + + // make new news message with negative voting + if ( !sb.isRobinsonMode() ) { + final Map map = new HashMap(); + map.put("urlhash", delHash); + map.put("vote", "negative"); + map.put("refid", ""); + sb.peers.newsPool.publishMyNews( + sb.peers.mySeed(), + NewsPool.CATEGORY_SURFTIPP_VOTE_ADD, + map); + } + + // delete the search history since this still shows the entry + SearchEventCache.delete(delHash); + } catch ( final IOException e ) { + Log.logException(e); + } + } + + // if a plus-button was hit, create new voting message + if ( post != null && post.containsKey("recommendref") ) { + if ( !sb.verifyAuthentication(header) ) { + prop.authenticationRequired(); + return prop; + } + final String recommendHash = post.get("recommendref", ""); // urlhash + final URIMetadataNode urlentry = indexSegment.fulltext().getMetadata(UTF8.getBytes(recommendHash)); + if ( urlentry != null ) { + Document[] documents = null; + try { + documents = + sb.loader.loadDocuments( + sb.loader.request(urlentry.url(), true, false), + CacheStrategy.IFEXIST, + Integer.MAX_VALUE, BlacklistType.SEARCH, TextSnippet.snippetMinLoadDelay); + } catch ( final IOException e ) { + } catch ( final Parser.Failure e ) { + } + if ( documents != null ) { + // create a news message + final Map map = new HashMap(); + map.put("url", urlentry.url().toNormalform(true).replace(',', '|')); + map.put("title", urlentry.dc_title().replace(',', ' ')); + map.put("description", documents[0].dc_title().replace(',', ' ')); + map.put("author", documents[0].dc_creator()); + map.put("tags", documents[0].dc_subject(' ')); + sb.peers.newsPool.publishMyNews( + sb.peers.mySeed(), + NewsPool.CATEGORY_SURFTIPP_ADD, + map); + documents[0].close(); + } + } + } + + // if a bookmarks-button was hit, create new bookmark entry + if ( post != null && post.containsKey("bookmarkref") ) { + if ( !sb.verifyAuthentication(header) ) { + prop.authenticationRequired(); + return prop; + } + final String bookmarkHash = post.get("bookmarkref", ""); // urlhash + final DigestURI url = indexSegment.fulltext().getURL(UTF8.getBytes(bookmarkHash)); + if ( url != null ) { + try { + sb.tables.bookmarks.createBookmark( + sb.loader, + url, + YMarkTables.USER_ADMIN, + true, + "searchresult", + "/search"); + } catch ( final Throwable e ) { + } + } + } + + // check filters + try { + Pattern.compile(urlmask); + } catch ( final PatternSyntaxException ex ) { + SearchEvent.log.logWarning("Illegal URL mask, not a valid regex: " + urlmask); + prop.put("urlmaskerror", 1); + prop.putHTML("urlmaskerror_urlmask", urlmask); + urlmask = ".*"; + } + + try { + Pattern.compile(prefermask); + } catch ( final PatternSyntaxException ex ) { + SearchEvent.log.logWarning("Illegal prefer mask, not a valid regex: " + prefermask); + prop.put("prefermaskerror", 1); + prop.putHTML("prefermaskerror_prefermask", prefermask); + prefermask = ""; + } + + // do the search + final QueryParams theQuery = + new QueryParams( + qg, + modifier.toString().trim(), + maxDistance, + prefermask, + contentdom, + language, + metatags, + snippetFetchStrategy, + itemsPerPage, + startRecord, + urlmask, protocol, tld, ext, + clustersearch && global ? QueryParams.Searchdom.CLUSTER : (global && indexReceiveGranted + ? QueryParams.Searchdom.GLOBAL + : QueryParams.Searchdom.LOCAL), + 20, + constraint, + true, + sitehash, + sitehost, + DigestURI.hosthashess(sb.getConfig("search.excludehosth", "")), + author, + DigestURI.TLD_any_zone_filter, + client, + authenticated, + indexSegment, + ranking, + header.get(RequestHeader.USER_AGENT, ""), + sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) + && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) + && sb.peers.mySeed().getFlagAcceptRemoteIndex(), + false, + lat, lon, rad); + EventTracker.delete(EventTracker.EClass.SEARCH); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch( + theQuery.id(true), + SearchEventType.INITIALIZATION, + "", + 0, + 0), false); + + // tell all threads to do nothing for a specific time + sb.intermissionAllThreads(3000); + + // filter out words that appear in bluelist + theQuery.getQueryGoal().filterOut(Switchboard.blueList); + + // log + Log.logInfo( + "LOCAL_SEARCH", + "INIT WORD SEARCH: " + + theQuery.getQueryGoal().getOriginalQueryString(false) + + ":" + + QueryParams.hashSet2hashString(theQuery.getQueryGoal().getIncludeHashes()) + + " - " + + theQuery.neededResults() + + " links to be computed, " + + theQuery.itemsPerPage() + + " lines to be displayed"); + EventChannel.channels(EventChannel.LOCALSEARCH).addMessage( + new RSSMessage("Local Search Request", theQuery.getQueryGoal().getOriginalQueryString(false), "")); + final long timestamp = System.currentTimeMillis(); + + // create a new search event + if ( SearchEventCache.getEvent(theQuery.id(false)) == null ) { + theQuery.setOffset(0); // in case that this is a new search, always start without a offset + startRecord = 0; + } + final SearchEvent theSearch = + SearchEventCache.getEvent( + theQuery, + sb.peers, + sb.tables, + (sb.isRobinsonMode()) ? sb.clusterhashes : null, + false, + sb.loader, + (int) sb.getConfigLong( + SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, + sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)), + sb.getConfigLong( + SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, + sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)), + (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), + (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); + + if ( startRecord == 0 ) { + if ( sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated ) { + sb.heuristicSite(theSearch, sitehost); + } + if ( (heuristicBlekko >= 0 || sb.getConfigBool("heuristic.blekko", false)) && authenticated ) { + sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko"); + } + if ( (heuristicTwitter >= 0 || sb.getConfigBool("heuristic.twitter", false)) && authenticated ) { + sb.heuristicRSS("http://search.twitter.com/search.rss?rpp=50&q=$", theSearch, "twitter"); + } + } + + // log + Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + + theQuery.getQueryGoal().getOriginalQueryString(false) + + " - " + + "local_rwi_available(" + theSearch.query.local_rwi_available.get() + "), " + + "local_rwi_stored(" + theSearch.query.local_rwi_stored.get() + "), " + + "local_solr_available(" + theSearch.query.local_solr_available.get() + "), " + + "local_solr_stored(" + theSearch.query.local_solr_stored.get() + "), " + + "remote_available(" + theSearch.query.remote_available.get() + "), " + + "remote_stored(" + theSearch.query.remote_stored.get() + "), " + + "remote_peerCount(" + theSearch.query.remote_peerCount.get() + "), " + + "local_sortout(" + theSearch.query.misses.size() + "), " + + (System.currentTimeMillis() - timestamp) + + " ms"); + + // prepare search statistics + theQuery.searchtime = System.currentTimeMillis() - timestamp; + theQuery.urlretrievaltime = theSearch.getURLRetrievalTime(); + theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime(); + AccessTracker.add(AccessTracker.Location.local, theQuery); + + // check suggestions + final int meanMax = (post != null) ? post.getInt("meanCount", 0) : 0; + + prop.put("meanCount", meanMax); + if ( meanMax > 0 && !json && !rss ) { + final DidYouMean didYouMean = new DidYouMean(indexSegment, new StringBuilder(querystring)); + final Iterator meanIt = didYouMean.getSuggestions(100, 5).iterator(); + int meanCount = 0; + String suggestion; + try { + meanCollect: while ( meanCount < meanMax && meanIt.hasNext() ) { + try { + suggestion = meanIt.next().toString(); + prop.put("didYouMean_suggestions_" + meanCount + "_word", suggestion); + prop.put( + "didYouMean_suggestions_" + meanCount + "_url", + QueryParams.navurl( + "html", + 0, + theQuery, + suggestion).toString()); + prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|"); + meanCount++; + } catch (ConcurrentModificationException e) {break meanCollect;} + } + } catch (ConcurrentModificationException e) {} + prop.put("didYouMean_suggestions_" + (meanCount - 1) + "_sep", ""); + prop.put("didYouMean", meanCount > 0 ? 1 : 0); + prop.put("didYouMean_suggestions", meanCount); + } else { + prop.put("didYouMean", 0); + } + + // find geographic info + final SortedSet coordinates = LibraryProvider.geoLoc.find(originalquerystring, false); + if ( coordinates == null || coordinates.isEmpty() || startRecord > 0 ) { + prop.put("geoinfo", "0"); + } else { + int i = 0; + for ( final GeoLocation c : coordinates ) { + prop.put("geoinfo_loc_" + i + "_lon", Math.round(c.lon() * 10000.0f) / 10000.0f); + prop.put("geoinfo_loc_" + i + "_lat", Math.round(c.lat() * 10000.0f) / 10000.0f); + prop.put("geoinfo_loc_" + i + "_name", c.getName()); + i++; + if ( i >= 10 ) { + break; + } + } + prop.put("geoinfo_loc", i); + prop.put("geoinfo", "1"); + } + + // update the search tracker + try { + synchronized ( trackerHandles ) { + trackerHandles.add(theQuery.starttime); + while ( trackerHandles.size() > 600 ) { + if ( !trackerHandles.remove(trackerHandles.first()) ) { + break; + } + } + } + sb.localSearchTracker.put(client, trackerHandles); + if ( sb.localSearchTracker.size() > 100 ) { + sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement()); + } + if ( MemoryControl.shortStatus() ) { + sb.localSearchTracker.clear(); + } + } catch ( final Exception e ) { + Log.logException(e); + } + + prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1); + prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.query.itemsPerPage > theSearch.query.getResultCount() ? startRecord + theSearch.query.getResultCount() % theSearch.query.itemsPerPage : startRecord + theSearch.query.itemsPerPage, true)); + prop.put("num-results_itemsPerPage", Formatter.number(itemsPerPage)); + prop.put("num-results_totalcount", Formatter.number(theSearch.query.getResultCount())); + prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0"); + prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true)); + prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.query.misses.size(), true)); + prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true)); + prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true)); + prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); + + // compose page navigation + final StringBuilder resnav = new StringBuilder(200); + final int thispage = startRecord / theQuery.itemsPerPage(); + if ( thispage == 0 ) { + resnav + .append("\"arrowleft\" "); + } else { + resnav.append("\"arrowleft\" "); + } + final int numberofpages = Math.min(10, 1 + ((theSearch.query.getResultCount() - 1) / theQuery.itemsPerPage())); + + for ( int i = 0; i < numberofpages; i++ ) { + if ( i == thispage ) { + resnav.append("\"page"); "); + } else { + resnav.append("\"page"); "); + } + } + if ( thispage >= numberofpages ) { + resnav + .append("\"arrowright\""); + } else { + resnav.append("\"arrowright\""); + } + final String resnavs = resnav.toString(); + prop.put("num-results_resnav", resnavs); + prop.put("pageNavBottom", (theSearch.query.getResultCount() - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom + prop.put("pageNavBottom_resnav", resnavs); + + // generate the search result lines; the content will be produced by another servlet + for ( int i = 0; i < theQuery.itemsPerPage(); i++ ) { + prop.put("results_" + i + "_item", startRecord + i); + prop.put("results_" + i + "_eventID", theQuery.id(false)); + } + prop.put("results", theQuery.itemsPerPage()); + prop + .put( + "resultTable", + (contentdom == ContentDomain.APP || contentdom == ContentDomain.AUDIO || contentdom == ContentDomain.VIDEO) + ? 1 + : 0); + prop.put("eventID", theQuery.id(false)); // for bottomline + + // process result of search + if ( !filtered.isEmpty() ) { + prop.put("excluded", "1"); + prop.putHTML("excluded_stopwords", filtered.toString()); + } else { + prop.put("excluded", "0"); + } + + if ( prop == null || prop.isEmpty() ) { + if ( post.get("query", post.get("search", "")).length() < 2 ) { + prop.put("num-results", "2"); // no results - at least 2 chars + } else { + prop.put("num-results", "1"); // no results + } + } else { + prop.put("num-results", "3"); + } + + prop.put("cat", "href"); + prop.put("depth", "0"); + + // adding some additional properties needed for the rss feed + String hostName = header.get("Host", Domains.LOCALHOST); + if ( hostName.indexOf(':', 0) == -1 ) { + hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090")); + } + prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html"); + prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.png"); + prop.put("thisaddress", hostName); + } + + prop.put("searchagain", global ? "1" : "0"); + prop.putHTML("former", originalquerystring); + prop.put("count", itemsPerPage); + prop.put("offset", startRecord); + prop.put("resource", global ? "global" : "local"); + prop.putHTML("prefermaskfilter", prefermask); + prop.put("indexof", (indexof) ? "on" : "off"); + prop.put("constraint", (constraint == null) ? "" : constraint.exportB64()); + prop.put("search.verify", snippetFetchStrategy == null + ? sb.getConfig("search.verify", "iffresh") + : snippetFetchStrategy.toName()); + prop.put( + "search.navigation", + (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all")); + prop.put("contentdom", (post == null ? "text" : post.get("contentdom", "text"))); + prop.put( + "searchdomswitches", + sb.getConfigBool("search.text", true) + || sb.getConfigBool("search.audio", true) + || sb.getConfigBool("search.video", true) + || sb.getConfigBool("search.image", true) + || sb.getConfigBool("search.app", true) ? 1 : 0); + prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0); + prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0); + prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0); + prop.put("searchdomswitches_searchimage", sb.getConfigBool("search.image", true) ? 1 : 0); + prop.put("searchdomswitches_searchapp", sb.getConfigBool("search.app", true) ? 1 : 0); + prop.put("searchdomswitches_searchtext_check", (contentdom == ContentDomain.TEXT || contentdom == ContentDomain.ALL) ? "1" : "0"); + prop.put("searchdomswitches_searchaudio_check", (contentdom == ContentDomain.AUDIO) ? "1" : "0"); + prop.put("searchdomswitches_searchvideo_check", (contentdom == ContentDomain.VIDEO) ? "1" : "0"); + prop.put("searchdomswitches_searchimage_check", (contentdom == ContentDomain.IMAGE) ? "1" : "0"); + prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0"); + + // copy properties for "more options" link + prop.put("searchdomswitches_count", prop.get("count")); + prop.put("searchdomswitches_urlmaskfilter", prop.get("urlmaskfilter")); + prop.put("searchdomswitches_prefermaskfilter", prop.get("prefermaskfilter")); + prop.put("searchdomswitches_cat", prop.get("cat")); + prop.put("searchdomswitches_constraint", prop.get("constraint")); + prop.put("searchdomswitches_contentdom", prop.get("contentdom")); + prop.put("searchdomswitches_former", prop.get("former")); + prop.put("searchdomswitches_meanCount", prop.get("meanCount")); + + // for RSS: don't HTML encode some elements + prop.putXML("rss_query", originalquerystring); + prop.putXML("rss_queryenc", originalquerystring.replace(' ', '+')); + + sb.localSearchLastAccess = System.currentTimeMillis(); + + // hostname and port (assume locahost if nothing helps) + final InetAddress hostIP = Domains.myPublicLocalIP(); + prop.put("myhost", hostIP != null ? hostIP.getHostAddress() : Domains.LOCALHOST); + prop.put("myport", serverCore.getPortNr(sb.getConfig("port", "8090"))); + + // return rewrite properties + return prop; + } +} diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index c62717026..cb101934c 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -127,7 +127,7 @@ public class yacysearch_location { prop.put("kml_date822", HeaderFramework.formatRFC1123(new Date())); prop.put("kml_promoteSearchPageGreeting", promoteSearchPageGreeting); - prop.put("kml_rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.gif"); + prop.put("kml_rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.png"); prop.put("kml_searchBaseURL", "http://" + hostName + "/yacysearch_location.rss"); prop.putXML("kml_rss_query", originalquerystring); prop.put("kml_rss_queryenc", originalquerystring.replace(' ', '+')); diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.java index a70d63738..6c53c2be9 100644 --- a/source/net/yacy/crawler/retrieval/Response.java +++ b/source/net/yacy/crawler/retrieval/Response.java @@ -231,12 +231,6 @@ public class Response { return docDate; } - public String language() { - // please avoid this method if a condenser document is available, because the condenser has a built-in language detection - // this here is only a guess using the TLD - return url().language(); - } - public CrawlProfile profile() { return this.profile; } diff --git a/source/net/yacy/kelondro/data/word/WordReferenceVars.java b/source/net/yacy/kelondro/data/word/WordReferenceVars.java index f33998f81..87b7d5110 100644 --- a/source/net/yacy/kelondro/data/word/WordReferenceVars.java +++ b/source/net/yacy/kelondro/data/word/WordReferenceVars.java @@ -114,7 +114,6 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc final Bitfield flags, // attributes to the url and to the word according the url final double termfrequency ) { - if (language == null || language.length != 2) language = default_language; //final int mddct = MicroDate.microDateDays(updatetime); this.flags = flags; //this.freshUntil = Math.max(0, mddlm + (mddct - mddlm) * 2);