bump to httpClient-4.3

pull/1/head
sixcooler 12 years ago
parent 14442efa6d
commit 15b1bb2513

@ -13,7 +13,7 @@
<classpathentry excluding="collection1/" kind="src" path="htroot/solr"/>
<classpathentry kind="src" path="htroot/gsa"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.3.jar"/>
<classpathentry kind="lib" path="lib/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="lib/activation.jar"/>
@ -45,9 +45,9 @@
<classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.2.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.2.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.2.5.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.2.4.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.2.5.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.3.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.3.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.3.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.5.jar"/>
<classpathentry kind="lib" path="lib/metadata-extractor-2.6.2.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-4.4.0.jar"/>

@ -48,14 +48,14 @@
<string>$JAVAROOT/lib/commons-io-2.1.jar</string>
<string>$JAVAROOT/lib/commons-jxpath-1.3.jar</string>
<string>$JAVAROOT/lib/commons-lang-2.6.jar</string>
<string>$JAVAROOT/lib/commons-logging-1.1.1.jar</string>
<string>$JAVAROOT/lib/commons-logging-1.1.3.jar</string>
<string>$JAVAROOT/lib/fontbox-1.8.2.jar</string>
<string>$JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar</string>
<string>$JAVAROOT/lib/guava-13.0.1.jar</string>
<string>$JAVAROOT/lib/htmllexer.jar</string>
<string>$JAVAROOT/lib/httpclient-4.2.5.jar</string>
<string>$JAVAROOT/lib/httpcore-4.2.4.jar</string>
<string>$JAVAROOT/lib/httpmime-4.2.5.jar</string>
<string>$JAVAROOT/lib/httpclient-4.3.jar</string>
<string>$JAVAROOT/lib/httpcore-4.3.jar</string>
<string>$JAVAROOT/lib/httpmime-4.3.jar</string>
<string>$JAVAROOT/lib/icu4j-core.jar</string>
<string>$JAVAROOT/lib/iri-0.8.jar</string>
<string>$JAVAROOT/lib/J7Zip-modified.jar</string>

@ -172,14 +172,14 @@
<pathelement location="${lib}/commons-io-2.1.jar" />
<pathelement location="${lib}/commons-jxpath-1.3.jar" />
<pathelement location="${lib}/commons-lang-2.6.jar" />
<pathelement location="${lib}/commons-logging-1.1.1.jar" />
<pathelement location="${lib}/commons-logging-1.1.3.jar" />
<pathelement location="${lib}/fontbox-1.8.2.jar" />
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/guava-13.0.1.jar" />
<pathelement location="${lib}/htmllexer.jar" />
<pathelement location="${lib}/httpclient-4.2.5.jar" />
<pathelement location="${lib}/httpcore-4.2.4.jar" />
<pathelement location="${lib}/httpmime-4.2.5.jar" />
<pathelement location="${lib}/httpclient-4.3.jar" />
<pathelement location="${lib}/httpcore-4.3.jar" />
<pathelement location="${lib}/httpmime-4.3.jar" />
<pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/iri-0.8.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" />

Binary file not shown.

Binary file not shown.

@ -22,8 +22,8 @@ commons-io-2.1.jar
commons-lang-2.6.jar
geronimo-stax-api_1.0_spec-1.0.1.jar
guava-r05.jar
httpclient-4.2.3.jar
httpcore-4.2.3.jar
httpclient-4.3.jar
httpcore-4.3.jar
jcl-over-slf4j-1.6.1.jar
log4j-over-slf4j-1.6.1.jar
lucene-analyzers-3.6.0.jar

Binary file not shown.

Binary file not shown.

@ -1,240 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
=========================================================================
This project contains annotations in the package org.apache.http.annotation
which are derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls.
See http://www.jcip.net and the Creative Commons Attribution License
(http://creativecommons.org/licenses/by/2.5)
Full text: http://creativecommons.org/licenses/by/2.5/legalcode
License
THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS.
1. Definitions
"Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License.
"Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License.
"Licensor" means the individual or entity that offers the Work under the terms of this License.
"Original Author" means the individual or entity who created the Work.
"Work" means the copyrightable work of authorship offered under the terms of this License.
"You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation.
2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws.
3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below:
to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works;
to create and reproduce Derivative Works;
to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works;
to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works.
For the avoidance of doubt, where the work is a musical composition:
Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work.
Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions).
Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions).
The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved.
4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions:
You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested.
If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit.
5. Representations, Warranties and Disclaimer
UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
7. Termination
This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License.
Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above.
8. Miscellaneous
Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License.
Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License.
If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.
No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent.
This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You.

Binary file not shown.

@ -1,4 +1,3 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@ -175,28 +174,3 @@
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Binary file not shown.

Binary file not shown.

@ -0,0 +1,176 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS

Binary file not shown.

@ -73,7 +73,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.2.3.jar;lib/httpcore-4.2.3.jar;lib/httpmime-4.2.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.2.1.jar;lib/lucene-analyzers-phonetic-4.2.1.jar;lib/lucene-core-4.2.1.jar;lib/lucene-misc-4.2.1.jar;lib/lucene-spatial-4.2.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.2.1.jar;lib/solr-solrj-4.2.1.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath>
<classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.3.jar;lib/httpcore-4.3.jar;lib/httpmime-4.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.2.1.jar;lib/lucene-analyzers-phonetic-4.2.1.jar;lib/lucene-core-4.2.1.jar;lib/lucene-misc-4.2.1.jar;lib/lucene-spatial-4.2.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.2.1.jar;lib/solr-solrj-4.2.1.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath>
<source-level>1.6</source-level>
</compilation-unit>
</java-data>

@ -24,10 +24,10 @@
package net.yacy.cora.document.encoding;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Comparator;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.content.StringBody;
/**
@ -45,6 +45,7 @@ public class UTF8 implements Comparator<String> {
static {
charset = Charset.forName("UTF-8");
}
private final static ContentType contentType = ContentType.TEXT_PLAIN.withCharset(charset);
public static final UTF8 insensitiveUTF8Comparator = new UTF8(true);
public static final UTF8 identityUTF8Comparator = new UTF8(false);
@ -103,12 +104,7 @@ public class UTF8 implements Comparator<String> {
}
public final static StringBody StringBody(final String s) {
try {
return new StringBody(s == null ? "" : s, charset);
} catch (final UnsupportedEncodingException e) {
e.printStackTrace();
return null;
}
return new StringBody(s == null ? "" : s, contentType);
}
/**

@ -37,24 +37,18 @@ import net.yacy.search.schema.WebgraphSchema;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@ -62,8 +56,8 @@ import org.apache.solr.client.solrj.impl.HttpSolrServer;
public class RemoteInstance implements SolrInstance {
private String solrurl;
private final DefaultHttpClient client;
// old Stuff: private final DefaultHttpClient client;
private final CloseableHttpClient client;
private final String defaultCoreName;
private final HttpSolrServer defaultServer;
private final Collection<String> coreNames;
@ -133,26 +127,24 @@ public class RemoteInstance implements SolrInstance {
}
}
if (solraccount.length() > 0) {
PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(100);
this.client = new DefaultHttpClient(cm) {
@Override
protected HttpContext createHttpContext() {
HttpContext context = super.createHttpContext();
AuthCache authCache = new BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
authCache.put(targetHost, basicAuth);
context.setAttribute(ClientContext.AUTH_CACHE, authCache);
this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
return context;
}
};
HttpParams params = this.client.getParams();
HttpConnectionParams.setConnectionTimeout(params, timeout);
HttpConnectionParams.setSoTimeout(params, timeout);
this.client.addRequestInterceptor(new HttpRequestInterceptor() {
final RequestConfig.Builder reqBuilder = RequestConfig.custom();
reqBuilder.setSocketTimeout(timeout);
reqBuilder.setConnectTimeout(timeout);
reqBuilder.setConnectionRequestTimeout(timeout);
final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setConnectionManager(cm);
builder.setDefaultRequestConfig(reqBuilder.build());
builder.setDefaultCredentialsProvider(credsProvider);
builder.disableAutomaticRetries(); // no retries needed; we expect connections to fail; therefore we should not retry
// ask for gzip - why not using net.yacy.cora.protocol.http.GzipRequestInterceptor?
builder.addInterceptorLast(new HttpRequestInterceptor() {
@Override
public void process(final HttpRequest request, final HttpContext context) throws IOException {
if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
@ -160,7 +152,8 @@ public class RemoteInstance implements SolrInstance {
}
});
this.client.addResponseInterceptor(new HttpResponseInterceptor() {
// uncompress gzip - why not using net.yacy.cora.protocol.http.GzipResponseInterceptor?
builder.addInterceptorLast(new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext context) throws IOException {
HttpEntity entity = response.getEntity();
@ -178,9 +171,57 @@ public class RemoteInstance implements SolrInstance {
}
}
});
BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
this.client.setCredentialsProvider(credsProvider);
this.client = builder.build();
// old Stuff:
// PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
// cm.setMaxTotal(100);
//
// this.client = new DefaultHttpClient(cm) {
// @Override
// protected HttpContext createHttpContext() {
// Auth-caching HttpContext context = super.createHttpContext();
// is done AuthCache authCache = new BasicAuthCache();
// by default BasicScheme basicAuth = new BasicScheme();
// since 4.1 HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
// authCache.put(targetHost, basicAuth);
// context.setAttribute(ClientContext.AUTH_CACHE, authCache);
// this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
// return context;
// }
// };
// HttpParams params = this.client.getParams();
// HttpConnectionParams.setConnectionTimeout(params, timeout);
// HttpConnectionParams.setSoTimeout(params, timeout);
// this.client.addRequestInterceptor(new HttpRequestInterceptor() {
// @Override
// public void process(final HttpRequest request, final HttpContext context) throws IOException {
// if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
// if (!request.containsHeader("Connection")) request.addHeader("Connection", "close"); // prevent CLOSE_WAIT
// }
//
// });
// this.client.addResponseInterceptor(new HttpResponseInterceptor() {
// @Override
// public void process(final HttpResponse response, final HttpContext context) throws IOException {
// HttpEntity entity = response.getEntity();
// if (entity != null) {
// Header ceheader = entity.getContentEncoding();
// if (ceheader != null) {
// HeaderElement[] codecs = ceheader.getElements();
// for (HeaderElement codec : codecs) {
// if (codec.getName().equalsIgnoreCase("gzip")) {
// response.setEntity(new GzipDecompressingEntity(response.getEntity()));
// return;
// }
// }
// }
// }
// }
// });
// BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
// credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
// this.client.setCredentialsProvider(credsProvider);
} else {
this.client = null;
}
@ -248,7 +289,13 @@ public class RemoteInstance implements SolrInstance {
@Override
public void close() {
if (this.client != null) this.client.getConnectionManager().shutdown();
// old Stuff: if (this.client != null) this.client.getConnectionManager().shutdown();
if (this.client != null)
try {
this.client.close();
} catch (final IOException e) {
// TODO Auto-generated catch block
}
}
}

@ -28,6 +28,7 @@ package net.yacy.cora.protocol;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.MIME;
import org.apache.http.entity.mime.content.AbstractContentBody;
@ -42,7 +43,7 @@ public class ByteArrayBody extends AbstractContentBody {
* @param filename
*/
public ByteArrayBody(final byte[] bytes, final String filename) {
super("application/octet-stream");
super(ContentType.APPLICATION_OCTET_STREAM);
this.bytes = bytes;
this.filename = filename;
}

@ -28,12 +28,12 @@ package net.yacy.cora.protocol.http;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@ -49,7 +49,6 @@ import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.http.ProxySettings.Protocol;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
@ -58,41 +57,32 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.ByteArrayBuffer;
@ -108,106 +98,126 @@ import org.apache.http.util.EntityUtils;
public class HTTPClient {
private final static int maxcon = 200;
private static IdledConnectionEvictor idledConnectionEvictor = null;
private static HttpClient httpClient = initConnectionManager();
private static final CredentialsProvider credsProvider = new BasicCredentialsProvider();
private static IdleConnectionMonitorThread connectionMonitor = null;
private final static RequestConfig dfltReqConf = initRequestConfig();
private final static HttpClientBuilder clientBuilder = initClientBuilder();
private final RequestConfig.Builder reqConfBuilder;
private Set<Entry<String, String>> headers = null;
private HttpResponse httpResponse = null;
private CloseableHttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
private long upbytes = 0L;
private int timeout = 10000;
private ClientIdentification.Agent agent = null;
private String host = null;
private boolean redirecting = true;
private String realm = null;
public HTTPClient(final ClientIdentification.Agent agent) {
super();
this.agent = agent;
this.timeout = agent.clientTimeout;
HttpProtocolParams.setUserAgent(httpClient.getParams(), agent.userAgent);
clientBuilder.setUserAgent(agent.userAgent);
reqConfBuilder = RequestConfig.copy(dfltReqConf);
reqConfBuilder.setSocketTimeout(agent.clientTimeout);
reqConfBuilder.setConnectTimeout(agent.clientTimeout);
reqConfBuilder.setConnectionRequestTimeout(agent.clientTimeout);
}
public HTTPClient(final ClientIdentification.Agent agent, final int timeout) {
super();
this.agent = agent;
this.timeout = timeout;
HttpProtocolParams.setUserAgent(httpClient.getParams(), agent.userAgent);
clientBuilder.setUserAgent(agent.userAgent);
reqConfBuilder = RequestConfig.copy(dfltReqConf);
reqConfBuilder.setSocketTimeout(timeout);
reqConfBuilder.setConnectTimeout(timeout);
reqConfBuilder.setConnectionRequestTimeout(timeout);
}
public static void setDefaultUserAgent(final String defaultAgent) {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
clientBuilder.setUserAgent(defaultAgent);
}
public static HttpClient initConnectionManager() {
// Create and initialize scheme registry
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
private static RequestConfig initRequestConfig() {
final RequestConfig.Builder builder = RequestConfig.custom();
// IMPORTANT - if not set to 'false' then servers do not process the request until a time-out of 2 seconds
builder.setExpectContinueEnabled(false);
// timeout in milliseconds until a connection is established in milliseconds
builder.setConnectionRequestTimeout(6000);
builder.setConnectTimeout(8000);
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
builder.setSocketTimeout(3000);
// getting an I/O error when executing a request over a connection that has been closed at the server side
builder.setStaleConnectionCheckEnabled(true);
// ignore cookies, cause this may cause segfaults in default cookiestore and is not needed
builder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
builder.setRedirectsEnabled(true);
builder.setRelativeRedirectsAllowed(true);
return builder.build();
}
final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry);
private static HttpClientBuilder initClientBuilder() {
final HttpClientBuilder builder = HttpClientBuilder.create();
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
/**
* ConnectionManager settings
*/
builder.setConnectionManager(initPoolingConnectionManager());
builder.setDefaultRequestConfig(dfltReqConf);
// UserAgent
builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
// remove retries; we expect connections to fail; therefore we should not retry
builder.disableAutomaticRetries();
// disable the cookiestore, cause this may cause segfaults and is not needed
builder.setDefaultCookieStore(null);
builder.disableCookieManagement();
// add cutom keep alive strategy
builder.setKeepAliveStrategy(customKeepAliveStrategy());
// ask for gzip
builder.addInterceptorLast(new GzipRequestInterceptor());
// uncompress gzip
builder.addInterceptorLast(new GzipResponseInterceptor());
// Proxy
builder.setRoutePlanner(ProxySettings.RoutePlanner);
builder.setDefaultCredentialsProvider(ProxySettings.CredsProvider);
return builder;
}
private static PoolingHttpClientConnectionManager initPoolingConnectionManager() {
final PlainConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", getSSLSocketFactory())
.build();
final PoolingHttpClientConnectionManager pooling = new PoolingHttpClientConnectionManager(registry, new DnsResolver(){
@Override
public InetAddress[] resolve(final String host0)throws UnknownHostException {
final InetAddress ip = Domains.dnsResolve(host0);
if (ip == null) throw new UnknownHostException(host0);
return new InetAddress[]{ip};
}});
// how much connections do we need? - default: 20
clientConnectionManager.setMaxTotal(maxcon);
pooling.setMaxTotal(maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// connections per host (2 default)
clientConnectionManager.setDefaultMaxPerRoute(2);
pooling.setDefaultMaxPerRoute(4);
// Increase max connections for localhost
final HttpHost localhost = new HttpHost(Domains.LOCALHOST);
clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon);
/**
* HTTP protocol settings
*/
HttpProtocolParams.setVersion(httpParams, HttpVersion.HTTP_1_1);
// UserAgent
HttpProtocolParams.setUserAgent(httpParams, ClientIdentification.yacyInternetCrawlerAgent.userAgent);
HttpProtocolParams.setUseExpectContinue(httpParams, false); // IMPORTANT - if not set to 'false' then servers do not process the request until a time-out of 2 seconds
/**
* HTTP connection settings
*/
// timeout in milliseconds until a connection is established in milliseconds
HttpConnectionParams.setConnectionTimeout(httpParams, 6000);
// SO_LINGER affects the socket close operation in seconds
// HttpConnectionParams.setLinger(httpParams, 6);
// HttpConnectionParams.setSocketBufferSize(httpParams, 8192);
pooling.setMaxPerRoute(new HttpRoute(localhost), maxcon);
final SocketConfig socketConfig = SocketConfig.custom()
// Defines whether the socket can be bound even though a previous connection is still in a timeout state.
.setSoReuseAddress(true)
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
HttpConnectionParams.setSoTimeout(httpParams, 1000);
// getting an I/O error when executing a request over a connection that has been closed at the server side
HttpConnectionParams.setStaleCheckingEnabled(httpParams, true);
.setSoTimeout(3000)
// conserve bandwidth by minimizing the number of segments that are sent
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// Defines whether the socket can be bound even though a previous connection is still in a timeout state.
HttpConnectionParams.setSoReuseaddr(httpParams, true);
.setTcpNoDelay(false)
.build();
pooling.setDefaultSocketConfig(socketConfig);
/**
* HTTP client settings
*/
// ignore cookies, cause this may cause segfaults in default cookiestore and is not needed
HttpClientParams.setCookiePolicy(httpParams, CookiePolicy.IGNORE_COOKIES);
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
// disable the cookiestore, cause this may cause segfaults and is not needed
((DefaultHttpClient) httpClient).setCookieStore(null);
// add cutom keep alive strategy
addCustomKeepAliveStrategy((DefaultHttpClient) httpClient);
// ask for gzip
((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
// uncompress gzip
((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
// remove retries; we expect connections to fail; therefore we should not retry
((DefaultHttpClient) httpClient).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
if (idledConnectionEvictor == null) {
idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
idledConnectionEvictor.start();
if (connectionMonitor == null) {
connectionMonitor = new IdleConnectionMonitorThread(pooling);
connectionMonitor.start();
}
return httpClient;
return pooling;
}
/**
@ -217,34 +227,29 @@ public class HTTPClient {
* @throws InterruptedException
*/
public static void closeConnectionManager() throws InterruptedException {
if (idledConnectionEvictor != null) {
if (connectionMonitor != null) {
// Shut down the evictor thread
idledConnectionEvictor.shutdown();
idledConnectionEvictor.join();
connectionMonitor.shutdown();
connectionMonitor.join();
}
if (httpClient != null) {
// Shut down the connection manager
httpClient.getConnectionManager().shutdown();
}
}
public static void setAuth(final String host, final int port, final String user, final String pw) {
final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw);
final AuthScope scope = new AuthScope(host, port);
credsProvider.setCredentials(scope, creds);
httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider);
}
// public static void setAuth(final String host, final int port, final String user, final String pw) {
// final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw);
// final AuthScope scope = new AuthScope(host, port);
// credsProvider.setCredentials(scope, creds);
// httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider);
// }
/**
* this method sets a host on which more than the default of 2 router per host are allowed
*
* @param the host to be raised in 'route per host'
*/
public static void setMaxRouteHost(final String host) {
final HttpHost mHost = new HttpHost(host);
((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
}
// /**
// * this method sets a host on which more than the default of 2 router per host are allowed
// *
// * @param the host to be raised in 'route per host'
// */
// public static void setMaxRouteHost(final String host) {
// final HttpHost mHost = new HttpHost(host);
// ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
// }
/**
* This method sets the Header used for the request
@ -261,7 +266,9 @@ public class HTTPClient {
* @param timeout in milliseconds
*/
public void setTimout(final int timeout) {
this.timeout = timeout;
reqConfBuilder.setSocketTimeout(timeout);
reqConfBuilder.setConnectTimeout(timeout);
reqConfBuilder.setConnectionRequestTimeout(timeout);
}
/**
@ -270,7 +277,7 @@ public class HTTPClient {
* @param userAgent
*/
public void setUserAgent(final ClientIdentification.Agent agent) {
this.agent = agent;
clientBuilder.setUserAgent(agent.userAgent);
}
/**
@ -288,7 +295,8 @@ public class HTTPClient {
* @param redirecting
*/
public void setRedirecting(final boolean redirecting) {
this.redirecting = redirecting;
reqConfBuilder.setRedirectsEnabled(redirecting);
reqConfBuilder.setRelativeRedirectsAllowed(redirecting);
}
/**
@ -354,7 +362,7 @@ public class HTTPClient {
}
httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
return getContentBytes(httpGet, url.getHost(), maxBytes);
return getContentBytes(httpGet, maxBytes);
}
/**
@ -378,7 +386,7 @@ public class HTTPClient {
httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
this.currentRequest = httpGet;
execute(httpGet, url.getHost());
execute(httpGet);
}
/**
@ -393,7 +401,7 @@ public class HTTPClient {
final HttpHead httpHead = new HttpHead(url.toNormalform(true));
httpHead.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
execute(httpHead, url.getHost());
execute(httpHead);
finish();
ConnectionInfo.removeConnection(httpHead.hashCode());
return this.httpResponse;
@ -422,7 +430,7 @@ public class HTTPClient {
this.upbytes = length;
httpPost.setEntity(inputStreamEntity);
this.currentRequest = httpPost;
execute(httpPost, host);
execute(httpPost);
}
/**
@ -455,9 +463,10 @@ public class HTTPClient {
setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (vhost == null) setHost(Domains.LOCALHOST);
final MultipartEntity multipartEntity = new MultipartEntity();
final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
for (final Entry<String,ContentBody> part : post.entrySet())
multipartEntity.addPart(part.getKey(), part.getValue());
entityBuilder.addPart(part.getKey(), part.getValue());
final HttpEntity multipartEntity = entityBuilder.build();
// statistics
this.upbytes = multipartEntity.getContentLength();
@ -467,7 +476,7 @@ public class HTTPClient {
httpPost.setEntity(multipartEntity);
}
return getContentBytes(httpPost, url.getHost(), Integer.MAX_VALUE);
return getContentBytes(httpPost, Integer.MAX_VALUE);
}
/**
@ -491,7 +500,7 @@ public class HTTPClient {
// statistics
this.upbytes = length;
httpPost.setEntity(inputStreamEntity);
return getContentBytes(httpPost, host, Integer.MAX_VALUE);
return getContentBytes(httpPost, Integer.MAX_VALUE);
}
/**
@ -580,9 +589,9 @@ public class HTTPClient {
}
}
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, String host, final int maxBytes) throws IOException {
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final int maxBytes) throws IOException {
try {
execute(httpUriRequest, host);
execute(httpUriRequest);
if (this.httpResponse == null) return null;
// get the response body
final HttpEntity httpEntity = this.httpResponse.getEntity();
@ -602,11 +611,13 @@ public class HTTPClient {
}
}
private void execute(final HttpUriRequest httpUriRequest, String host) throws IOException {
final HttpContext httpContext = new BasicHttpContext();
private void execute(final HttpUriRequest httpUriRequest) throws IOException {
final HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(reqConfBuilder.build());
if (this.host != null)
context.setTargetHost(new HttpHost(this.host));
setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams(), host);
// statistics
storeConnectionInfo(httpUriRequest);
// execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF
@ -620,14 +631,17 @@ public class HTTPClient {
}
Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost());
try {
final long time = System.currentTimeMillis();
this.httpResponse = httpClient.execute(httpUriRequest, httpContext);
try {
final CloseableHttpClient client = clientBuilder.build();
this.httpResponse = client.execute(httpUriRequest, context);
this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (final IOException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort();
throw new IOException("Client can't execute: " + (e.getCause() == null ? e.getMessage() : e.getCause().getMessage()));
throw new IOException("Client can't execute: "
+ (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())
+ " duration=" + Long.toString(System.currentTimeMillis() - time));
}
}
@ -669,23 +683,6 @@ public class HTTPClient {
httpUriRequest.setHeader("Authorization", "realm=" + this.realm);
}
private void setParams(final HttpParams httpParams) {
HttpClientParams.setRedirecting(httpParams, this.redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, this.timeout);
HttpConnectionParams.setSoTimeout(httpParams, this.timeout);
if (this.agent != null)
HttpProtocolParams.setUserAgent(httpParams, this.agent.userAgent);
if (this.host != null)
httpParams.setParameter(HTTP.TARGET_HOST, this.host);
}
private static void setProxy(final HttpParams httpParams, String host) {
if (ProxySettings.useForHost(host, Protocol.HTTP))
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((DefaultHttpClient) httpClient);
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost();
@ -699,7 +696,7 @@ public class HTTPClient {
this.upbytes));
}
private static SSLSocketFactory getSSLSocketFactory() {
private static SSLConnectionSocketFactory getSSLSocketFactory() {
final TrustManager trustManager = new X509TrustManager() {
@Override
public void checkClientTrusted(final X509Certificate[] chain, final String authType)
@ -728,7 +725,9 @@ public class HTTPClient {
// e.printStackTrace();
}
final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
final SSLConnectionSocketFactory sslSF = new SSLConnectionSocketFactory(
sslContext,
SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
return sslSF;
}
@ -739,8 +738,8 @@ public class HTTPClient {
*
* @param defaultHttpClient
*/
private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) {
defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
private static ConnectionKeepAliveStrategy customKeepAliveStrategy() {
return new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// Honor 'keep-alive' header
@ -762,7 +761,7 @@ public class HTTPClient {
// Keep alive for 5 seconds only
return 5 * 1000;
}
});
};
}
/**
@ -773,13 +772,13 @@ public class HTTPClient {
public static void main(final String[] args) {
String url = null;
// prepare Parts
final Map<String,ContentBody> newparts = new LinkedHashMap<String,ContentBody>();
try {
newparts.put("foo", new StringBody("FooBar"));
newparts.put("bar", new StringBody("BarFoo"));
} catch (final UnsupportedEncodingException e) {
System.out.println(e.getStackTrace());
}
// final Map<String,ContentBody> newparts = new LinkedHashMap<String,ContentBody>();
// try {
// newparts.put("foo", new StringBody("FooBar"));
// newparts.put("bar", new StringBody("BarFoo"));
// } catch (final UnsupportedEncodingException e) {
// System.out.println(e.getStackTrace());
// }
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setRedirecting(false);
// Get some
@ -805,7 +804,7 @@ public class HTTPClient {
// for (HeaderElement element: header.getElements())
// System.out.println("Element " + element.getName() + " : " + element.getValue());
}
System.out.println(client.getHttpResponse().getLocale());
// System.out.println(client.getHttpResponse().getLocale());
System.out.println(client.getHttpResponse().getProtocolVersion());
System.out.println(client.getHttpResponse().getStatusLine());
// Post some
@ -822,49 +821,41 @@ public class HTTPClient {
}
}
public static class IdleConnectionMonitorThread extends Thread {
/**
*
* @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638
*
*/
private static class IdledConnectionEvictor extends Thread {
private final ClientConnectionManager clientConnectionManager;
private final HttpClientConnectionManager connMgr;
private volatile boolean shutdown;
public IdledConnectionEvictor(final ClientConnectionManager clientConnectionManager) {
public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
super();
this.clientConnectionManager = clientConnectionManager;
this.connMgr = connMgr;
}
@Override
public void run() {
try {
while (!this.shutdown) {
while (!shutdown) {
synchronized (this) {
wait(5000);
// Close expired connections
this.clientConnectionManager.closeExpiredConnections();
connMgr.closeExpiredConnections();
// Optionally, close connections
// that have been idle longer than 5 sec
// (some SOHO router act strange on >5sec idled connections)
this.clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS);
// that have been idle longer than 30 sec
connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
}
}
connMgr.shutdown();
} catch (final InterruptedException ex) {
// terminate
}
}
public void shutdown() {
this.shutdown = true;
shutdown = true;
synchronized (this) {
notifyAll();
}
}
}
}

@ -27,10 +27,16 @@ package net.yacy.cora.protocol.http;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.routing.HttpRoutePlanner;
import org.apache.http.protocol.HttpContext;
/**
* settings for a remote proxy
@ -71,12 +77,36 @@ public final class ProxySettings {
return new HttpHost(host, port);
}
public static void setProxyCreds(AbstractHttpClient httpClient) {
if (!use) return;
httpClient.getCredentialsProvider().setCredentials(
new AuthScope(host, port),
new UsernamePasswordCredentials(user, password));
public static HttpRoutePlanner RoutePlanner = new HttpRoutePlanner() {
@Override
public HttpRoute determineRoute(HttpHost target, HttpRequest request, HttpContext context) throws HttpException {
if (use) {
final Protocol protocol = "https".equalsIgnoreCase(target.getSchemeName())? Protocol.HTTPS : Protocol.HTTP;
if (useForHost(target.getHostName(), protocol))
return new HttpRoute(target, null, getProxyHost(), protocol == Protocol.HTTPS);
}
return new HttpRoute(target); // direct
}
};
public static CredentialsProvider CredsProvider = new CredentialsProvider() {
@Override
public void clear() {
}
@Override
public Credentials getCredentials(AuthScope scope) {
if (host != null && host.equals(scope.getHost()) && port == scope.getPort())
return new UsernamePasswordCredentials(user, password);
return null;
}
@Override
public void setCredentials(AuthScope arg0, Credentials arg1) {
}
};
/**
* tell if a remote proxy will be used for the given host

Loading…
Cancel
Save