diff --git a/.classpath b/.classpath index 1f79155a6..b27797779 100644 --- a/.classpath +++ b/.classpath @@ -13,7 +13,7 @@ - + @@ -45,9 +45,9 @@ - - - + + + diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist index f16752825..178fb54bb 100644 --- a/addon/YaCy.app/Contents/Info.plist +++ b/addon/YaCy.app/Contents/Info.plist @@ -48,14 +48,14 @@ $JAVAROOT/lib/commons-io-2.1.jar $JAVAROOT/lib/commons-jxpath-1.3.jar $JAVAROOT/lib/commons-lang-2.6.jar - $JAVAROOT/lib/commons-logging-1.1.1.jar + $JAVAROOT/lib/commons-logging-1.1.3.jar $JAVAROOT/lib/fontbox-1.8.2.jar $JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar $JAVAROOT/lib/guava-13.0.1.jar $JAVAROOT/lib/htmllexer.jar - $JAVAROOT/lib/httpclient-4.2.5.jar - $JAVAROOT/lib/httpcore-4.2.4.jar - $JAVAROOT/lib/httpmime-4.2.5.jar + $JAVAROOT/lib/httpclient-4.3.jar + $JAVAROOT/lib/httpcore-4.3.jar + $JAVAROOT/lib/httpmime-4.3.jar $JAVAROOT/lib/icu4j-core.jar $JAVAROOT/lib/iri-0.8.jar $JAVAROOT/lib/J7Zip-modified.jar diff --git a/build.xml b/build.xml index 344202fbd..f7f4c90a1 100644 --- a/build.xml +++ b/build.xml @@ -172,14 +172,14 @@ - + - - - + + + diff --git a/lib/commons-logging-1.1.1.jar b/lib/commons-logging-1.1.1.jar deleted file mode 100644 index 8758a96b7..000000000 Binary files a/lib/commons-logging-1.1.1.jar and /dev/null differ diff --git a/lib/httpclient-4.2.5.License b/lib/commons-logging-1.1.3.License similarity index 100% rename from lib/httpclient-4.2.5.License rename to lib/commons-logging-1.1.3.License diff --git a/lib/commons-logging-1.1.3.jar b/lib/commons-logging-1.1.3.jar new file mode 100644 index 000000000..ab5125407 Binary files /dev/null and b/lib/commons-logging-1.1.3.jar differ diff --git a/lib/dependencies.txt b/lib/dependencies.txt index 053d6d846..25ea9a53d 100644 --- a/lib/dependencies.txt +++ b/lib/dependencies.txt @@ -22,8 +22,8 @@ commons-io-2.1.jar commons-lang-2.6.jar geronimo-stax-api_1.0_spec-1.0.1.jar guava-r05.jar -httpclient-4.2.3.jar -httpcore-4.2.3.jar +httpclient-4.3.jar +httpcore-4.3.jar jcl-over-slf4j-1.6.1.jar log4j-over-slf4j-1.6.1.jar lucene-analyzers-3.6.0.jar diff --git a/lib/httpclient-4.2.5.jar b/lib/httpclient-4.2.5.jar deleted file mode 100644 index 5310588ef..000000000 Binary files a/lib/httpclient-4.2.5.jar and /dev/null differ diff --git a/lib/httpmime-4.2.5.License b/lib/httpclient-4.3.License similarity index 100% rename from lib/httpmime-4.2.5.License rename to lib/httpclient-4.3.License diff --git a/lib/httpclient-4.3.jar b/lib/httpclient-4.3.jar new file mode 100644 index 000000000..5c446f04d Binary files /dev/null and b/lib/httpclient-4.3.jar differ diff --git a/lib/httpcore-4.2.4.License b/lib/httpcore-4.2.4.License deleted file mode 100644 index 879d3fdd9..000000000 --- a/lib/httpcore-4.2.4.License +++ /dev/null @@ -1,240 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -========================================================================= - -This project contains annotations in the package org.apache.http.annotation -which are derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. -See http://www.jcip.net and the Creative Commons Attribution License -(http://creativecommons.org/licenses/by/2.5) -Full text: http://creativecommons.org/licenses/by/2.5/legalcode - -License - -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - - "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License. - "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License. - "Licensor" means the individual or entity that offers the Work under the terms of this License. - "Original Author" means the individual or entity who created the Work. - "Work" means the copyrightable work of authorship offered under the terms of this License. - "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. - -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: - - to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works; - to create and reproduce Derivative Works; - to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works; - to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works. - - For the avoidance of doubt, where the work is a musical composition: - Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work. - Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions). - Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions). - -The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: - - You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested. - If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit. - -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. Termination - - This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. - Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. - -8. Miscellaneous - - Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. - Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. - If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. - No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. - This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. diff --git a/lib/httpcore-4.2.4.jar b/lib/httpcore-4.2.4.jar deleted file mode 100644 index 9f45bd91c..000000000 Binary files a/lib/httpcore-4.2.4.jar and /dev/null differ diff --git a/lib/commons-logging-1.1.1.License b/lib/httpcore-4.3.License similarity index 88% rename from lib/commons-logging-1.1.1.License rename to lib/httpcore-4.3.License index 75b52484e..d9a10c0d8 100644 --- a/lib/commons-logging-1.1.1.License +++ b/lib/httpcore-4.3.License @@ -1,202 +1,176 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/lib/httpcore-4.3.jar b/lib/httpcore-4.3.jar new file mode 100644 index 000000000..e5da4578a Binary files /dev/null and b/lib/httpcore-4.3.jar differ diff --git a/lib/httpmime-4.2.5.jar b/lib/httpmime-4.2.5.jar deleted file mode 100644 index e63b24d70..000000000 Binary files a/lib/httpmime-4.2.5.jar and /dev/null differ diff --git a/lib/httpmime-4.3.License b/lib/httpmime-4.3.License new file mode 100644 index 000000000..d9a10c0d8 --- /dev/null +++ b/lib/httpmime-4.3.License @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/lib/httpmime-4.3.jar b/lib/httpmime-4.3.jar new file mode 100644 index 000000000..55dd8aecf Binary files /dev/null and b/lib/httpmime-4.3.jar differ diff --git a/nbproject/project.xml b/nbproject/project.xml index 9df73aceb..ca5a5d3bb 100644 --- a/nbproject/project.xml +++ b/nbproject/project.xml @@ -73,7 +73,7 @@ source htroot - lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.2.3.jar;lib/httpcore-4.2.3.jar;lib/httpmime-4.2.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.2.1.jar;lib/lucene-analyzers-phonetic-4.2.1.jar;lib/lucene-core-4.2.1.jar;lib/lucene-misc-4.2.1.jar;lib/lucene-spatial-4.2.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.2.1.jar;lib/solr-solrj-4.2.1.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar + lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.3.jar;lib/httpcore-4.3.jar;lib/httpmime-4.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.2.1.jar;lib/lucene-analyzers-phonetic-4.2.1.jar;lib/lucene-core-4.2.1.jar;lib/lucene-misc-4.2.1.jar;lib/lucene-spatial-4.2.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.2.1.jar;lib/solr-solrj-4.2.1.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar 1.6 diff --git a/source/net/yacy/cora/document/encoding/UTF8.java b/source/net/yacy/cora/document/encoding/UTF8.java index 1d6de94a1..06c5a8176 100644 --- a/source/net/yacy/cora/document/encoding/UTF8.java +++ b/source/net/yacy/cora/document/encoding/UTF8.java @@ -24,10 +24,10 @@ package net.yacy.cora.document.encoding; -import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.util.Comparator; +import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.content.StringBody; /** @@ -45,6 +45,7 @@ public class UTF8 implements Comparator { static { charset = Charset.forName("UTF-8"); } + private final static ContentType contentType = ContentType.TEXT_PLAIN.withCharset(charset); public static final UTF8 insensitiveUTF8Comparator = new UTF8(true); public static final UTF8 identityUTF8Comparator = new UTF8(false); @@ -103,12 +104,7 @@ public class UTF8 implements Comparator { } public final static StringBody StringBody(final String s) { - try { - return new StringBody(s == null ? "" : s, charset); - } catch (final UnsupportedEncodingException e) { - e.printStackTrace(); - return null; - } + return new StringBody(s == null ? "" : s, contentType); } /** diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index f9fa62cef..7032c37ec 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -37,24 +37,18 @@ import net.yacy.search.schema.WebgraphSchema; import org.apache.http.Header; import org.apache.http.HeaderElement; import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.GzipDecompressingEntity; -import org.apache.http.client.protocol.ClientContext; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.PoolingClientConnectionManager; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer; @@ -62,8 +56,8 @@ import org.apache.solr.client.solrj.impl.HttpSolrServer; public class RemoteInstance implements SolrInstance { private String solrurl; - private final DefaultHttpClient client; - + // old Stuff: private final DefaultHttpClient client; + private final CloseableHttpClient client; private final String defaultCoreName; private final HttpSolrServer defaultServer; private final Collection coreNames; @@ -133,26 +127,24 @@ public class RemoteInstance implements SolrInstance { } } if (solraccount.length() > 0) { - PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager - cm.setMaxTotal(100); - - this.client = new DefaultHttpClient(cm) { - @Override - protected HttpContext createHttpContext() { - HttpContext context = super.createHttpContext(); - AuthCache authCache = new BasicAuthCache(); - BasicScheme basicAuth = new BasicScheme(); - HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol()); - authCache.put(targetHost, basicAuth); - context.setAttribute(ClientContext.AUTH_CACHE, authCache); - this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry - return context; - } - }; - HttpParams params = this.client.getParams(); - HttpConnectionParams.setConnectionTimeout(params, timeout); - HttpConnectionParams.setSoTimeout(params, timeout); - this.client.addRequestInterceptor(new HttpRequestInterceptor() { + final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(100); + + final RequestConfig.Builder reqBuilder = RequestConfig.custom(); + reqBuilder.setSocketTimeout(timeout); + reqBuilder.setConnectTimeout(timeout); + reqBuilder.setConnectionRequestTimeout(timeout); + + final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw)); + + final HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setConnectionManager(cm); + builder.setDefaultRequestConfig(reqBuilder.build()); + builder.setDefaultCredentialsProvider(credsProvider); + builder.disableAutomaticRetries(); // no retries needed; we expect connections to fail; therefore we should not retry + // ask for gzip - why not using net.yacy.cora.protocol.http.GzipRequestInterceptor? + builder.addInterceptorLast(new HttpRequestInterceptor() { @Override public void process(final HttpRequest request, final HttpContext context) throws IOException { if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip"); @@ -160,7 +152,8 @@ public class RemoteInstance implements SolrInstance { } }); - this.client.addResponseInterceptor(new HttpResponseInterceptor() { + // uncompress gzip - why not using net.yacy.cora.protocol.http.GzipResponseInterceptor? + builder.addInterceptorLast(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws IOException { HttpEntity entity = response.getEntity(); @@ -178,9 +171,57 @@ public class RemoteInstance implements SolrInstance { } } }); - BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw)); - this.client.setCredentialsProvider(credsProvider); + this.client = builder.build(); + +// old Stuff: +// PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager +// cm.setMaxTotal(100); +// +// this.client = new DefaultHttpClient(cm) { +// @Override +// protected HttpContext createHttpContext() { +// Auth-caching HttpContext context = super.createHttpContext(); +// is done AuthCache authCache = new BasicAuthCache(); +// by default BasicScheme basicAuth = new BasicScheme(); +// since 4.1 HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol()); +// authCache.put(targetHost, basicAuth); +// context.setAttribute(ClientContext.AUTH_CACHE, authCache); +// this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry +// return context; +// } +// }; +// HttpParams params = this.client.getParams(); +// HttpConnectionParams.setConnectionTimeout(params, timeout); +// HttpConnectionParams.setSoTimeout(params, timeout); +// this.client.addRequestInterceptor(new HttpRequestInterceptor() { +// @Override +// public void process(final HttpRequest request, final HttpContext context) throws IOException { +// if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip"); +// if (!request.containsHeader("Connection")) request.addHeader("Connection", "close"); // prevent CLOSE_WAIT +// } +// +// }); +// this.client.addResponseInterceptor(new HttpResponseInterceptor() { +// @Override +// public void process(final HttpResponse response, final HttpContext context) throws IOException { +// HttpEntity entity = response.getEntity(); +// if (entity != null) { +// Header ceheader = entity.getContentEncoding(); +// if (ceheader != null) { +// HeaderElement[] codecs = ceheader.getElements(); +// for (HeaderElement codec : codecs) { +// if (codec.getName().equalsIgnoreCase("gzip")) { +// response.setEntity(new GzipDecompressingEntity(response.getEntity())); +// return; +// } +// } +// } +// } +// } +// }); +// BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); +// credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw)); +// this.client.setCredentialsProvider(credsProvider); } else { this.client = null; } @@ -248,7 +289,13 @@ public class RemoteInstance implements SolrInstance { @Override public void close() { - if (this.client != null) this.client.getConnectionManager().shutdown(); +// old Stuff: if (this.client != null) this.client.getConnectionManager().shutdown(); + if (this.client != null) + try { + this.client.close(); + } catch (final IOException e) { + // TODO Auto-generated catch block + } } } diff --git a/source/net/yacy/cora/protocol/ByteArrayBody.java b/source/net/yacy/cora/protocol/ByteArrayBody.java index 352cfffed..132b20f41 100644 --- a/source/net/yacy/cora/protocol/ByteArrayBody.java +++ b/source/net/yacy/cora/protocol/ByteArrayBody.java @@ -28,6 +28,7 @@ package net.yacy.cora.protocol; import java.io.IOException; import java.io.OutputStream; +import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MIME; import org.apache.http.entity.mime.content.AbstractContentBody; @@ -42,7 +43,7 @@ public class ByteArrayBody extends AbstractContentBody { * @param filename */ public ByteArrayBody(final byte[] bytes, final String filename) { - super("application/octet-stream"); + super(ContentType.APPLICATION_OCTET_STREAM); this.bytes = bytes; this.filename = filename; } diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 71b2591a4..d34dcb7be 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -28,12 +28,12 @@ package net.yacy.cora.protocol.http; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -49,7 +49,6 @@ import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.cora.protocol.http.ProxySettings.Protocol; import org.apache.http.Header; import org.apache.http.HeaderElement; @@ -58,41 +57,32 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; -import org.apache.http.HttpVersion; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.HttpClient; +import org.apache.http.client.config.CookieSpecs; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.params.CookiePolicy; -import org.apache.http.client.params.HttpClientParams; -import org.apache.http.client.protocol.ClientContext; -import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.config.SocketConfig; import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.params.ConnRouteParams; +import org.apache.http.conn.DnsResolver; +import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.mime.MultipartEntity; +import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.ContentBody; -import org.apache.http.entity.mime.content.StringBody; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.PoolingClientConnectionManager; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; -import org.apache.http.params.HttpProtocolParams; -import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.apache.http.util.ByteArrayBuffer; @@ -108,106 +98,126 @@ import org.apache.http.util.EntityUtils; public class HTTPClient { private final static int maxcon = 200; - private static IdledConnectionEvictor idledConnectionEvictor = null; - private static HttpClient httpClient = initConnectionManager(); - private static final CredentialsProvider credsProvider = new BasicCredentialsProvider(); + private static IdleConnectionMonitorThread connectionMonitor = null; + private final static RequestConfig dfltReqConf = initRequestConfig(); + private final static HttpClientBuilder clientBuilder = initClientBuilder(); + private final RequestConfig.Builder reqConfBuilder; private Set> headers = null; - private HttpResponse httpResponse = null; + private CloseableHttpResponse httpResponse = null; private HttpUriRequest currentRequest = null; private long upbytes = 0L; - private int timeout = 10000; - private ClientIdentification.Agent agent = null; private String host = null; - private boolean redirecting = true; private String realm = null; public HTTPClient(final ClientIdentification.Agent agent) { super(); - this.agent = agent; - this.timeout = agent.clientTimeout; - HttpProtocolParams.setUserAgent(httpClient.getParams(), agent.userAgent); + clientBuilder.setUserAgent(agent.userAgent); + reqConfBuilder = RequestConfig.copy(dfltReqConf); + reqConfBuilder.setSocketTimeout(agent.clientTimeout); + reqConfBuilder.setConnectTimeout(agent.clientTimeout); + reqConfBuilder.setConnectionRequestTimeout(agent.clientTimeout); } public HTTPClient(final ClientIdentification.Agent agent, final int timeout) { super(); - this.agent = agent; - this.timeout = timeout; - HttpProtocolParams.setUserAgent(httpClient.getParams(), agent.userAgent); + clientBuilder.setUserAgent(agent.userAgent); + reqConfBuilder = RequestConfig.copy(dfltReqConf); + reqConfBuilder.setSocketTimeout(timeout); + reqConfBuilder.setConnectTimeout(timeout); + reqConfBuilder.setConnectionRequestTimeout(timeout); } public static void setDefaultUserAgent(final String defaultAgent) { - HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent); + clientBuilder.setUserAgent(defaultAgent); } - - public static HttpClient initConnectionManager() { - // Create and initialize scheme registry - final SchemeRegistry schemeRegistry = new SchemeRegistry(); - schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); - schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory())); - - final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry); - - // Create and initialize HTTP parameters - final HttpParams httpParams = new BasicHttpParams(); - /** - * ConnectionManager settings - */ - // how much connections do we need? - default: 20 - clientConnectionManager.setMaxTotal(maxcon); - // for statistics same value should also be set here - ConnectionInfo.setMaxcount(maxcon); - // connections per host (2 default) - clientConnectionManager.setDefaultMaxPerRoute(2); - // Increase max connections for localhost - final HttpHost localhost = new HttpHost(Domains.LOCALHOST); - clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon); - /** - * HTTP protocol settings - */ - HttpProtocolParams.setVersion(httpParams, HttpVersion.HTTP_1_1); - // UserAgent - HttpProtocolParams.setUserAgent(httpParams, ClientIdentification.yacyInternetCrawlerAgent.userAgent); - HttpProtocolParams.setUseExpectContinue(httpParams, false); // IMPORTANT - if not set to 'false' then servers do not process the request until a time-out of 2 seconds - /** - * HTTP connection settings - */ + + private static RequestConfig initRequestConfig() { + final RequestConfig.Builder builder = RequestConfig.custom(); + // IMPORTANT - if not set to 'false' then servers do not process the request until a time-out of 2 seconds + builder.setExpectContinueEnabled(false); // timeout in milliseconds until a connection is established in milliseconds - HttpConnectionParams.setConnectionTimeout(httpParams, 6000); - // SO_LINGER affects the socket close operation in seconds - // HttpConnectionParams.setLinger(httpParams, 6); - // HttpConnectionParams.setSocketBufferSize(httpParams, 8192); + builder.setConnectionRequestTimeout(6000); + builder.setConnectTimeout(8000); // SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds - HttpConnectionParams.setSoTimeout(httpParams, 1000); + builder.setSocketTimeout(3000); // getting an I/O error when executing a request over a connection that has been closed at the server side - HttpConnectionParams.setStaleCheckingEnabled(httpParams, true); - // conserve bandwidth by minimizing the number of segments that are sent - HttpConnectionParams.setTcpNoDelay(httpParams, false); - // Defines whether the socket can be bound even though a previous connection is still in a timeout state. - HttpConnectionParams.setSoReuseaddr(httpParams, true); - - /** - * HTTP client settings - */ + builder.setStaleConnectionCheckEnabled(true); // ignore cookies, cause this may cause segfaults in default cookiestore and is not needed - HttpClientParams.setCookiePolicy(httpParams, CookiePolicy.IGNORE_COOKIES); - - httpClient = new DefaultHttpClient(clientConnectionManager, httpParams); + builder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); + builder.setRedirectsEnabled(true); + builder.setRelativeRedirectsAllowed(true); + return builder.build(); + } + + private static HttpClientBuilder initClientBuilder() { + final HttpClientBuilder builder = HttpClientBuilder.create(); + + builder.setConnectionManager(initPoolingConnectionManager()); + builder.setDefaultRequestConfig(dfltReqConf); + + // UserAgent + builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent); + + // remove retries; we expect connections to fail; therefore we should not retry + builder.disableAutomaticRetries(); // disable the cookiestore, cause this may cause segfaults and is not needed - ((DefaultHttpClient) httpClient).setCookieStore(null); + builder.setDefaultCookieStore(null); + builder.disableCookieManagement(); + // add cutom keep alive strategy - addCustomKeepAliveStrategy((DefaultHttpClient) httpClient); + builder.setKeepAliveStrategy(customKeepAliveStrategy()); + // ask for gzip - ((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); + builder.addInterceptorLast(new GzipRequestInterceptor()); // uncompress gzip - ((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor()); - // remove retries; we expect connections to fail; therefore we should not retry - ((DefaultHttpClient) httpClient).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); - if (idledConnectionEvictor == null) { - idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager); - idledConnectionEvictor.start(); + builder.addInterceptorLast(new GzipResponseInterceptor()); + // Proxy + builder.setRoutePlanner(ProxySettings.RoutePlanner); + builder.setDefaultCredentialsProvider(ProxySettings.CredsProvider); + + return builder; + } + + private static PoolingHttpClientConnectionManager initPoolingConnectionManager() { + final PlainConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory(); + final Registry registry = RegistryBuilder.create() + .register("http", plainsf) + .register("https", getSSLSocketFactory()) + .build(); + final PoolingHttpClientConnectionManager pooling = new PoolingHttpClientConnectionManager(registry, new DnsResolver(){ + @Override + public InetAddress[] resolve(final String host0)throws UnknownHostException { + final InetAddress ip = Domains.dnsResolve(host0); + if (ip == null) throw new UnknownHostException(host0); + return new InetAddress[]{ip}; + }}); + // how much connections do we need? - default: 20 + pooling.setMaxTotal(maxcon); + // for statistics same value should also be set here + ConnectionInfo.setMaxcount(maxcon); + // connections per host (2 default) + pooling.setDefaultMaxPerRoute(4); + // Increase max connections for localhost + final HttpHost localhost = new HttpHost(Domains.LOCALHOST); + pooling.setMaxPerRoute(new HttpRoute(localhost), maxcon); + + final SocketConfig socketConfig = SocketConfig.custom() + // Defines whether the socket can be bound even though a previous connection is still in a timeout state. + .setSoReuseAddress(true) + // SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds + .setSoTimeout(3000) + // conserve bandwidth by minimizing the number of segments that are sent + .setTcpNoDelay(false) + .build(); + pooling.setDefaultSocketConfig(socketConfig); + + if (connectionMonitor == null) { + connectionMonitor = new IdleConnectionMonitorThread(pooling); + connectionMonitor.start(); } - return httpClient; + + return pooling; } /** @@ -217,34 +227,29 @@ public class HTTPClient { * @throws InterruptedException */ public static void closeConnectionManager() throws InterruptedException { - if (idledConnectionEvictor != null) { + if (connectionMonitor != null) { // Shut down the evictor thread - idledConnectionEvictor.shutdown(); - idledConnectionEvictor.join(); + connectionMonitor.shutdown(); + connectionMonitor.join(); } - if (httpClient != null) { - // Shut down the connection manager - httpClient.getConnectionManager().shutdown(); - } - } - public static void setAuth(final String host, final int port, final String user, final String pw) { - final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw); - final AuthScope scope = new AuthScope(host, port); - credsProvider.setCredentials(scope, creds); - httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider); - } +// public static void setAuth(final String host, final int port, final String user, final String pw) { +// final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw); +// final AuthScope scope = new AuthScope(host, port); +// credsProvider.setCredentials(scope, creds); +// httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider); +// } - /** - * this method sets a host on which more than the default of 2 router per host are allowed - * - * @param the host to be raised in 'route per host' - */ - public static void setMaxRouteHost(final String host) { - final HttpHost mHost = new HttpHost(host); - ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50); - } +// /** +// * this method sets a host on which more than the default of 2 router per host are allowed +// * +// * @param the host to be raised in 'route per host' +// */ +// public static void setMaxRouteHost(final String host) { +// final HttpHost mHost = new HttpHost(host); +// ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50); +// } /** * This method sets the Header used for the request @@ -261,7 +266,9 @@ public class HTTPClient { * @param timeout in milliseconds */ public void setTimout(final int timeout) { - this.timeout = timeout; + reqConfBuilder.setSocketTimeout(timeout); + reqConfBuilder.setConnectTimeout(timeout); + reqConfBuilder.setConnectionRequestTimeout(timeout); } /** @@ -270,7 +277,7 @@ public class HTTPClient { * @param userAgent */ public void setUserAgent(final ClientIdentification.Agent agent) { - this.agent = agent; + clientBuilder.setUserAgent(agent.userAgent); } /** @@ -288,7 +295,8 @@ public class HTTPClient { * @param redirecting */ public void setRedirecting(final boolean redirecting) { - this.redirecting = redirecting; + reqConfBuilder.setRedirectsEnabled(redirecting); + reqConfBuilder.setRelativeRedirectsAllowed(redirecting); } /** @@ -354,7 +362,7 @@ public class HTTPClient { } httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service - return getContentBytes(httpGet, url.getHost(), maxBytes); + return getContentBytes(httpGet, maxBytes); } /** @@ -378,7 +386,7 @@ public class HTTPClient { httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service this.currentRequest = httpGet; - execute(httpGet, url.getHost()); + execute(httpGet); } /** @@ -393,7 +401,7 @@ public class HTTPClient { final HttpHead httpHead = new HttpHead(url.toNormalform(true)); httpHead.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service - execute(httpHead, url.getHost()); + execute(httpHead); finish(); ConnectionInfo.removeConnection(httpHead.hashCode()); return this.httpResponse; @@ -422,7 +430,7 @@ public class HTTPClient { this.upbytes = length; httpPost.setEntity(inputStreamEntity); this.currentRequest = httpPost; - execute(httpPost, host); + execute(httpPost); } /** @@ -454,10 +462,11 @@ public class HTTPClient { setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service if (vhost == null) setHost(Domains.LOCALHOST); - - final MultipartEntity multipartEntity = new MultipartEntity(); - for (final Entry part : post.entrySet()) - multipartEntity.addPart(part.getKey(), part.getValue()); + + final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + for (final Entry part : post.entrySet()) + entityBuilder.addPart(part.getKey(), part.getValue()); + final HttpEntity multipartEntity = entityBuilder.build(); // statistics this.upbytes = multipartEntity.getContentLength(); @@ -467,7 +476,7 @@ public class HTTPClient { httpPost.setEntity(multipartEntity); } - return getContentBytes(httpPost, url.getHost(), Integer.MAX_VALUE); + return getContentBytes(httpPost, Integer.MAX_VALUE); } /** @@ -491,7 +500,7 @@ public class HTTPClient { // statistics this.upbytes = length; httpPost.setEntity(inputStreamEntity); - return getContentBytes(httpPost, host, Integer.MAX_VALUE); + return getContentBytes(httpPost, Integer.MAX_VALUE); } /** @@ -580,9 +589,9 @@ public class HTTPClient { } } - private byte[] getContentBytes(final HttpUriRequest httpUriRequest, String host, final int maxBytes) throws IOException { + private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final int maxBytes) throws IOException { try { - execute(httpUriRequest, host); + execute(httpUriRequest); if (this.httpResponse == null) return null; // get the response body final HttpEntity httpEntity = this.httpResponse.getEntity(); @@ -602,11 +611,13 @@ public class HTTPClient { } } - private void execute(final HttpUriRequest httpUriRequest, String host) throws IOException { - final HttpContext httpContext = new BasicHttpContext(); + private void execute(final HttpUriRequest httpUriRequest) throws IOException { + final HttpClientContext context = HttpClientContext.create(); + context.setRequestConfig(reqConfBuilder.build()); + if (this.host != null) + context.setTargetHost(new HttpHost(this.host)); + setHeaders(httpUriRequest); - setParams(httpUriRequest.getParams()); - setProxy(httpUriRequest.getParams(), host); // statistics storeConnectionInfo(httpUriRequest); // execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF @@ -620,14 +631,17 @@ public class HTTPClient { } Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost()); + final long time = System.currentTimeMillis(); try { - final long time = System.currentTimeMillis(); - this.httpResponse = httpClient.execute(httpUriRequest, httpContext); + final CloseableHttpClient client = clientBuilder.build(); + this.httpResponse = client.execute(httpUriRequest, context); this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); } catch (final IOException e) { ConnectionInfo.removeConnection(httpUriRequest.hashCode()); httpUriRequest.abort(); - throw new IOException("Client can't execute: " + (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())); + throw new IOException("Client can't execute: " + + (e.getCause() == null ? e.getMessage() : e.getCause().getMessage()) + + " duration=" + Long.toString(System.currentTimeMillis() - time)); } } @@ -669,23 +683,6 @@ public class HTTPClient { httpUriRequest.setHeader("Authorization", "realm=" + this.realm); } - private void setParams(final HttpParams httpParams) { - HttpClientParams.setRedirecting(httpParams, this.redirecting); - HttpConnectionParams.setConnectionTimeout(httpParams, this.timeout); - HttpConnectionParams.setSoTimeout(httpParams, this.timeout); - if (this.agent != null) - HttpProtocolParams.setUserAgent(httpParams, this.agent.userAgent); - if (this.host != null) - httpParams.setParameter(HTTP.TARGET_HOST, this.host); - } - - private static void setProxy(final HttpParams httpParams, String host) { - if (ProxySettings.useForHost(host, Protocol.HTTP)) - ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); - // TODO find a better way for this - ProxySettings.setProxyCreds((DefaultHttpClient) httpClient); - } - private void storeConnectionInfo(final HttpUriRequest httpUriRequest) { final int port = httpUriRequest.getURI().getPort(); final String thost = httpUriRequest.getURI().getHost(); @@ -699,7 +696,7 @@ public class HTTPClient { this.upbytes)); } - private static SSLSocketFactory getSSLSocketFactory() { + private static SSLConnectionSocketFactory getSSLSocketFactory() { final TrustManager trustManager = new X509TrustManager() { @Override public void checkClientTrusted(final X509Certificate[] chain, final String authType) @@ -728,7 +725,9 @@ public class HTTPClient { // e.printStackTrace(); } - final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + final SSLConnectionSocketFactory sslSF = new SSLConnectionSocketFactory( + sslContext, + SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); return sslSF; } @@ -739,8 +738,8 @@ public class HTTPClient { * * @param defaultHttpClient */ - private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) { - defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() { + private static ConnectionKeepAliveStrategy customKeepAliveStrategy() { + return new ConnectionKeepAliveStrategy() { @Override public long getKeepAliveDuration(HttpResponse response, HttpContext context) { // Honor 'keep-alive' header @@ -762,7 +761,7 @@ public class HTTPClient { // Keep alive for 5 seconds only return 5 * 1000; } - }); + }; } /** @@ -773,13 +772,13 @@ public class HTTPClient { public static void main(final String[] args) { String url = null; // prepare Parts - final Map newparts = new LinkedHashMap(); - try { - newparts.put("foo", new StringBody("FooBar")); - newparts.put("bar", new StringBody("BarFoo")); - } catch (final UnsupportedEncodingException e) { - System.out.println(e.getStackTrace()); - } +// final Map newparts = new LinkedHashMap(); +// try { +// newparts.put("foo", new StringBody("FooBar")); +// newparts.put("bar", new StringBody("BarFoo")); +// } catch (final UnsupportedEncodingException e) { +// System.out.println(e.getStackTrace()); +// } final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setRedirecting(false); // Get some @@ -805,7 +804,7 @@ public class HTTPClient { // for (HeaderElement element: header.getElements()) // System.out.println("Element " + element.getName() + " : " + element.getValue()); } - System.out.println(client.getHttpResponse().getLocale()); +// System.out.println(client.getHttpResponse().getLocale()); System.out.println(client.getHttpResponse().getProtocolVersion()); System.out.println(client.getHttpResponse().getStatusLine()); // Post some @@ -822,49 +821,41 @@ public class HTTPClient { } } + public static class IdleConnectionMonitorThread extends Thread { + + private final HttpClientConnectionManager connMgr; + private volatile boolean shutdown; + + public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) { + super(); + this.connMgr = connMgr; + } - /** - * - * @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638 - * - */ - private static class IdledConnectionEvictor extends Thread { - - private final ClientConnectionManager clientConnectionManager; - - private volatile boolean shutdown; - - public IdledConnectionEvictor(final ClientConnectionManager clientConnectionManager) { - super(); - this.clientConnectionManager = clientConnectionManager; - } - - @Override - public void run() { - try { - while (!this.shutdown) { - synchronized (this) { - wait(5000); - // Close expired connections - this.clientConnectionManager.closeExpiredConnections(); - // Optionally, close connections - // that have been idle longer than 5 sec - // (some SOHO router act strange on >5sec idled connections) - this.clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS); - } - } - } catch (final InterruptedException ex) { - // terminate - } - } - - public void shutdown() { - this.shutdown = true; - synchronized (this) { - notifyAll(); - } - } - + @Override + public void run() { + try { + while (!shutdown) { + synchronized (this) { + wait(5000); + // Close expired connections + connMgr.closeExpiredConnections(); + // Optionally, close connections + // that have been idle longer than 30 sec + connMgr.closeIdleConnections(30, TimeUnit.SECONDS); + } + } + connMgr.shutdown(); + } catch (final InterruptedException ex) { + // terminate + } + } + + public void shutdown() { + shutdown = true; + synchronized (this) { + notifyAll(); + } + } } } diff --git a/source/net/yacy/cora/protocol/http/ProxySettings.java b/source/net/yacy/cora/protocol/http/ProxySettings.java index 6bb11623d..b5aec00c5 100644 --- a/source/net/yacy/cora/protocol/http/ProxySettings.java +++ b/source/net/yacy/cora/protocol/http/ProxySettings.java @@ -27,10 +27,16 @@ package net.yacy.cora.protocol.http; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.http.HttpException; import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.impl.client.AbstractHttpClient; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.protocol.HttpContext; /** * settings for a remote proxy @@ -71,12 +77,36 @@ public final class ProxySettings { return new HttpHost(host, port); } - public static void setProxyCreds(AbstractHttpClient httpClient) { - if (!use) return; - httpClient.getCredentialsProvider().setCredentials( - new AuthScope(host, port), - new UsernamePasswordCredentials(user, password)); - } + public static HttpRoutePlanner RoutePlanner = new HttpRoutePlanner() { + + @Override + public HttpRoute determineRoute(HttpHost target, HttpRequest request, HttpContext context) throws HttpException { + if (use) { + final Protocol protocol = "https".equalsIgnoreCase(target.getSchemeName())? Protocol.HTTPS : Protocol.HTTP; + if (useForHost(target.getHostName(), protocol)) + return new HttpRoute(target, null, getProxyHost(), protocol == Protocol.HTTPS); + } + return new HttpRoute(target); // direct + } + }; + + public static CredentialsProvider CredsProvider = new CredentialsProvider() { + + @Override + public void clear() { + } + + @Override + public Credentials getCredentials(AuthScope scope) { + if (host != null && host.equals(scope.getHost()) && port == scope.getPort()) + return new UsernamePasswordCredentials(user, password); + return null; + } + + @Override + public void setCredentials(AuthScope arg0, Credentials arg1) { + } + }; /** * tell if a remote proxy will be used for the given host