bump to httpClient-4.3

pull/1/head
sixcooler 12 years ago
parent 14442efa6d
commit 15b1bb2513

@ -13,7 +13,7 @@
<classpathentry excluding="collection1/" kind="src" path="htroot/solr"/> <classpathentry excluding="collection1/" kind="src" path="htroot/solr"/>
<classpathentry kind="src" path="htroot/gsa"/> <classpathentry kind="src" path="htroot/gsa"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/> <classpathentry kind="lib" path="lib/commons-logging-1.1.3.jar"/>
<classpathentry kind="lib" path="lib/J7Zip-modified.jar"/> <classpathentry kind="lib" path="lib/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/> <classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="lib/activation.jar"/> <classpathentry kind="lib" path="lib/activation.jar"/>
@ -45,9 +45,9 @@
<classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/> <classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.2.jar"/> <classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.2.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.2.jar"/> <classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.2.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.2.5.jar"/> <classpathentry kind="lib" path="lib/httpclient-4.3.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.2.4.jar"/> <classpathentry kind="lib" path="lib/httpcore-4.3.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.2.5.jar"/> <classpathentry kind="lib" path="lib/httpmime-4.3.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.5.jar"/> <classpathentry kind="lib" path="lib/noggit-0.5.jar"/>
<classpathentry kind="lib" path="lib/metadata-extractor-2.6.2.jar"/> <classpathentry kind="lib" path="lib/metadata-extractor-2.6.2.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-4.4.0.jar"/> <classpathentry kind="lib" path="lib/lucene-analyzers-common-4.4.0.jar"/>

@ -48,14 +48,14 @@
<string>$JAVAROOT/lib/commons-io-2.1.jar</string> <string>$JAVAROOT/lib/commons-io-2.1.jar</string>
<string>$JAVAROOT/lib/commons-jxpath-1.3.jar</string> <string>$JAVAROOT/lib/commons-jxpath-1.3.jar</string>
<string>$JAVAROOT/lib/commons-lang-2.6.jar</string> <string>$JAVAROOT/lib/commons-lang-2.6.jar</string>
<string>$JAVAROOT/lib/commons-logging-1.1.1.jar</string> <string>$JAVAROOT/lib/commons-logging-1.1.3.jar</string>
<string>$JAVAROOT/lib/fontbox-1.8.2.jar</string> <string>$JAVAROOT/lib/fontbox-1.8.2.jar</string>
<string>$JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar</string> <string>$JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar</string>
<string>$JAVAROOT/lib/guava-13.0.1.jar</string> <string>$JAVAROOT/lib/guava-13.0.1.jar</string>
<string>$JAVAROOT/lib/htmllexer.jar</string> <string>$JAVAROOT/lib/htmllexer.jar</string>
<string>$JAVAROOT/lib/httpclient-4.2.5.jar</string> <string>$JAVAROOT/lib/httpclient-4.3.jar</string>
<string>$JAVAROOT/lib/httpcore-4.2.4.jar</string> <string>$JAVAROOT/lib/httpcore-4.3.jar</string>
<string>$JAVAROOT/lib/httpmime-4.2.5.jar</string> <string>$JAVAROOT/lib/httpmime-4.3.jar</string>
<string>$JAVAROOT/lib/icu4j-core.jar</string> <string>$JAVAROOT/lib/icu4j-core.jar</string>
<string>$JAVAROOT/lib/iri-0.8.jar</string> <string>$JAVAROOT/lib/iri-0.8.jar</string>
<string>$JAVAROOT/lib/J7Zip-modified.jar</string> <string>$JAVAROOT/lib/J7Zip-modified.jar</string>

@ -172,14 +172,14 @@
<pathelement location="${lib}/commons-io-2.1.jar" /> <pathelement location="${lib}/commons-io-2.1.jar" />
<pathelement location="${lib}/commons-jxpath-1.3.jar" /> <pathelement location="${lib}/commons-jxpath-1.3.jar" />
<pathelement location="${lib}/commons-lang-2.6.jar" /> <pathelement location="${lib}/commons-lang-2.6.jar" />
<pathelement location="${lib}/commons-logging-1.1.1.jar" /> <pathelement location="${lib}/commons-logging-1.1.3.jar" />
<pathelement location="${lib}/fontbox-1.8.2.jar" /> <pathelement location="${lib}/fontbox-1.8.2.jar" />
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" /> <pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/guava-13.0.1.jar" /> <pathelement location="${lib}/guava-13.0.1.jar" />
<pathelement location="${lib}/htmllexer.jar" /> <pathelement location="${lib}/htmllexer.jar" />
<pathelement location="${lib}/httpclient-4.2.5.jar" /> <pathelement location="${lib}/httpclient-4.3.jar" />
<pathelement location="${lib}/httpcore-4.2.4.jar" /> <pathelement location="${lib}/httpcore-4.3.jar" />
<pathelement location="${lib}/httpmime-4.2.5.jar" /> <pathelement location="${lib}/httpmime-4.3.jar" />
<pathelement location="${lib}/icu4j-core.jar" /> <pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/iri-0.8.jar" /> <pathelement location="${lib}/iri-0.8.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" /> <pathelement location="${lib}/J7Zip-modified.jar" />

Binary file not shown.

Binary file not shown.

@ -22,8 +22,8 @@ commons-io-2.1.jar
commons-lang-2.6.jar commons-lang-2.6.jar
geronimo-stax-api_1.0_spec-1.0.1.jar geronimo-stax-api_1.0_spec-1.0.1.jar
guava-r05.jar guava-r05.jar
httpclient-4.2.3.jar httpclient-4.3.jar
httpcore-4.2.3.jar httpcore-4.3.jar
jcl-over-slf4j-1.6.1.jar jcl-over-slf4j-1.6.1.jar
log4j-over-slf4j-1.6.1.jar log4j-over-slf4j-1.6.1.jar
lucene-analyzers-3.6.0.jar lucene-analyzers-3.6.0.jar

Binary file not shown.

Binary file not shown.

@ -1,240 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
=========================================================================
This project contains annotations in the package org.apache.http.annotation
which are derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls.
See http://www.jcip.net and the Creative Commons Attribution License
(http://creativecommons.org/licenses/by/2.5)
Full text: http://creativecommons.org/licenses/by/2.5/legalcode
License
THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS.
1. Definitions
"Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License.
"Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License.
"Licensor" means the individual or entity that offers the Work under the terms of this License.
"Original Author" means the individual or entity who created the Work.
"Work" means the copyrightable work of authorship offered under the terms of this License.
"You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation.
2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws.
3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below:
to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works;
to create and reproduce Derivative Works;
to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works;
to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works.
For the avoidance of doubt, where the work is a musical composition:
Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work.
Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions).
Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions).
The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved.
4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions:
You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested.
If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit.
5. Representations, Warranties and Disclaimer
UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
7. Termination
This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License.
Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above.
8. Miscellaneous
Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License.
Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License.
If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.
No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent.
This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You.

Binary file not shown.

@ -1,202 +1,176 @@
Apache License
Apache License Version 2.0, January 2004
Version 2.0, January 2004 http://www.apache.org/licenses/
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common
other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition,
control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the
"control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or
direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation
including but not limited to software source code, documentation source, and configuration files.
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but
transformation or translation of a Source form, including but not limited to compiled object code, generated documentation,
not limited to compiled object code, generated documentation, and conversions to other media types.
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a
Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work
copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the
form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications
editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes
represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain
of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of,
separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions
the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally
to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner
submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of
or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted"
the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent
means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to
to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems,
communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the
and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but
Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise
excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and
on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual,
this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable
worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of,
copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the
publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual,
this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable
worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made,
(except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work,
use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable
where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their
by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s)
Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You
with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a
institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work
cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct
or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses
or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate
granted to You under this License for that Work shall terminate as of the date such litigation is filed.
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without
Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You
modifications, and in Source or Object form, provided that You meet the following conditions:
meet the following conditions:
(a) You must give any other recipients of the Work or
(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
(b) You must cause any modified files to carry prominent notices stating that You changed the files; and
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and
that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work,
attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of
excluding those notices that do not pertain to any part of the Derivative Works; and
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must
distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained
include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not
within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one
pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed
of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or
as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or,
documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and
within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents
wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and
of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution
do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside
notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided
or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed
that such additional attribution notices cannot be construed as modifying the License.
as modifying the License.
You may add Your own copyright statement to Your modifications and
You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions
may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or
for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use,
for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with
reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work
any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of
by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions.
this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify
Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed
the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor,
names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the
except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each
agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions
implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the
PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any
appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise,
whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly
unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be
negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special,
liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a
incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the
result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill,
Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all
work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor
other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer,
the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity,
and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this
or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only
License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf
on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify,
of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability
defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason
incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Binary file not shown.

Binary file not shown.

@ -0,0 +1,176 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS

Binary file not shown.

@ -73,7 +73,7 @@
<compilation-unit> <compilation-unit>
<package-root>source</package-root> <package-root>source</package-root>
<package-root>htroot</package-root> <package-root>htroot</package-root>
<classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.2.3.jar;lib/httpcore-4.2.3.jar;lib/httpmime-4.2.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.2.1.jar;lib/lucene-analyzers-phonetic-4.2.1.jar;lib/lucene-core-4.2.1.jar;lib/lucene-misc-4.2.1.jar;lib/lucene-spatial-4.2.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.2.1.jar;lib/solr-solrj-4.2.1.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath> <classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.3.jar;lib/httpcore-4.3.jar;lib/httpmime-4.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.2.1.jar;lib/lucene-analyzers-phonetic-4.2.1.jar;lib/lucene-core-4.2.1.jar;lib/lucene-misc-4.2.1.jar;lib/lucene-spatial-4.2.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.2.1.jar;lib/solr-solrj-4.2.1.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath>
<source-level>1.6</source-level> <source-level>1.6</source-level>
</compilation-unit> </compilation-unit>
</java-data> </java-data>

@ -24,10 +24,10 @@
package net.yacy.cora.document.encoding; package net.yacy.cora.document.encoding;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.util.Comparator; import java.util.Comparator;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.content.StringBody; import org.apache.http.entity.mime.content.StringBody;
/** /**
@ -45,6 +45,7 @@ public class UTF8 implements Comparator<String> {
static { static {
charset = Charset.forName("UTF-8"); charset = Charset.forName("UTF-8");
} }
private final static ContentType contentType = ContentType.TEXT_PLAIN.withCharset(charset);
public static final UTF8 insensitiveUTF8Comparator = new UTF8(true); public static final UTF8 insensitiveUTF8Comparator = new UTF8(true);
public static final UTF8 identityUTF8Comparator = new UTF8(false); public static final UTF8 identityUTF8Comparator = new UTF8(false);
@ -103,12 +104,7 @@ public class UTF8 implements Comparator<String> {
} }
public final static StringBody StringBody(final String s) { public final static StringBody StringBody(final String s) {
try { return new StringBody(s == null ? "" : s, contentType);
return new StringBody(s == null ? "" : s, charset);
} catch (final UnsupportedEncodingException e) {
e.printStackTrace();
return null;
}
} }
/** /**

@ -37,24 +37,18 @@ import net.yacy.search.schema.WebgraphSchema;
import org.apache.http.Header; import org.apache.http.Header;
import org.apache.http.HeaderElement; import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest; import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor; import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache; import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer;
@ -62,8 +56,8 @@ import org.apache.solr.client.solrj.impl.HttpSolrServer;
public class RemoteInstance implements SolrInstance { public class RemoteInstance implements SolrInstance {
private String solrurl; private String solrurl;
private final DefaultHttpClient client; // old Stuff: private final DefaultHttpClient client;
private final CloseableHttpClient client;
private final String defaultCoreName; private final String defaultCoreName;
private final HttpSolrServer defaultServer; private final HttpSolrServer defaultServer;
private final Collection<String> coreNames; private final Collection<String> coreNames;
@ -133,26 +127,24 @@ public class RemoteInstance implements SolrInstance {
} }
} }
if (solraccount.length() > 0) { if (solraccount.length() > 0) {
PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(100); cm.setMaxTotal(100);
this.client = new DefaultHttpClient(cm) { final RequestConfig.Builder reqBuilder = RequestConfig.custom();
@Override reqBuilder.setSocketTimeout(timeout);
protected HttpContext createHttpContext() { reqBuilder.setConnectTimeout(timeout);
HttpContext context = super.createHttpContext(); reqBuilder.setConnectionRequestTimeout(timeout);
AuthCache authCache = new BasicAuthCache();
BasicScheme basicAuth = new BasicScheme(); final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol()); credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
authCache.put(targetHost, basicAuth);
context.setAttribute(ClientContext.AUTH_CACHE, authCache); final HttpClientBuilder builder = HttpClientBuilder.create();
this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry builder.setConnectionManager(cm);
return context; builder.setDefaultRequestConfig(reqBuilder.build());
} builder.setDefaultCredentialsProvider(credsProvider);
}; builder.disableAutomaticRetries(); // no retries needed; we expect connections to fail; therefore we should not retry
HttpParams params = this.client.getParams(); // ask for gzip - why not using net.yacy.cora.protocol.http.GzipRequestInterceptor?
HttpConnectionParams.setConnectionTimeout(params, timeout); builder.addInterceptorLast(new HttpRequestInterceptor() {
HttpConnectionParams.setSoTimeout(params, timeout);
this.client.addRequestInterceptor(new HttpRequestInterceptor() {
@Override @Override
public void process(final HttpRequest request, final HttpContext context) throws IOException { public void process(final HttpRequest request, final HttpContext context) throws IOException {
if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip"); if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
@ -160,7 +152,8 @@ public class RemoteInstance implements SolrInstance {
} }
}); });
this.client.addResponseInterceptor(new HttpResponseInterceptor() { // uncompress gzip - why not using net.yacy.cora.protocol.http.GzipResponseInterceptor?
builder.addInterceptorLast(new HttpResponseInterceptor() {
@Override @Override
public void process(final HttpResponse response, final HttpContext context) throws IOException { public void process(final HttpResponse response, final HttpContext context) throws IOException {
HttpEntity entity = response.getEntity(); HttpEntity entity = response.getEntity();
@ -178,9 +171,57 @@ public class RemoteInstance implements SolrInstance {
} }
} }
}); });
BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); this.client = builder.build();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
this.client.setCredentialsProvider(credsProvider); // old Stuff:
// PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
// cm.setMaxTotal(100);
//
// this.client = new DefaultHttpClient(cm) {
// @Override
// protected HttpContext createHttpContext() {
// Auth-caching HttpContext context = super.createHttpContext();
// is done AuthCache authCache = new BasicAuthCache();
// by default BasicScheme basicAuth = new BasicScheme();
// since 4.1 HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
// authCache.put(targetHost, basicAuth);
// context.setAttribute(ClientContext.AUTH_CACHE, authCache);
// this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
// return context;
// }
// };
// HttpParams params = this.client.getParams();
// HttpConnectionParams.setConnectionTimeout(params, timeout);
// HttpConnectionParams.setSoTimeout(params, timeout);
// this.client.addRequestInterceptor(new HttpRequestInterceptor() {
// @Override
// public void process(final HttpRequest request, final HttpContext context) throws IOException {
// if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
// if (!request.containsHeader("Connection")) request.addHeader("Connection", "close"); // prevent CLOSE_WAIT
// }
//
// });
// this.client.addResponseInterceptor(new HttpResponseInterceptor() {
// @Override
// public void process(final HttpResponse response, final HttpContext context) throws IOException {
// HttpEntity entity = response.getEntity();
// if (entity != null) {
// Header ceheader = entity.getContentEncoding();
// if (ceheader != null) {
// HeaderElement[] codecs = ceheader.getElements();
// for (HeaderElement codec : codecs) {
// if (codec.getName().equalsIgnoreCase("gzip")) {
// response.setEntity(new GzipDecompressingEntity(response.getEntity()));
// return;
// }
// }
// }
// }
// }
// });
// BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
// credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
// this.client.setCredentialsProvider(credsProvider);
} else { } else {
this.client = null; this.client = null;
} }
@ -248,7 +289,13 @@ public class RemoteInstance implements SolrInstance {
@Override @Override
public void close() { public void close() {
if (this.client != null) this.client.getConnectionManager().shutdown(); // old Stuff: if (this.client != null) this.client.getConnectionManager().shutdown();
if (this.client != null)
try {
this.client.close();
} catch (final IOException e) {
// TODO Auto-generated catch block
}
} }
} }

@ -28,6 +28,7 @@ package net.yacy.cora.protocol;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.MIME; import org.apache.http.entity.mime.MIME;
import org.apache.http.entity.mime.content.AbstractContentBody; import org.apache.http.entity.mime.content.AbstractContentBody;
@ -42,7 +43,7 @@ public class ByteArrayBody extends AbstractContentBody {
* @param filename * @param filename
*/ */
public ByteArrayBody(final byte[] bytes, final String filename) { public ByteArrayBody(final byte[] bytes, final String filename) {
super("application/octet-stream"); super(ContentType.APPLICATION_OCTET_STREAM);
this.bytes = bytes; this.bytes = bytes;
this.filename = filename; this.filename = filename;
} }

@ -28,12 +28,12 @@ package net.yacy.cora.protocol.http;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.UnsupportedEncodingException; import java.net.InetAddress;
import java.net.UnknownHostException;
import java.security.KeyManagementException; import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException; import java.security.cert.CertificateException;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
@ -49,7 +49,6 @@ import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.http.ProxySettings.Protocol;
import org.apache.http.Header; import org.apache.http.Header;
import org.apache.http.HeaderElement; import org.apache.http.HeaderElement;
@ -58,41 +57,32 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion; import org.apache.http.client.config.CookieSpecs;
import org.apache.http.auth.AuthScope; import org.apache.http.client.config.RequestConfig;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.params.HttpClientParams; import org.apache.http.config.Registry;
import org.apache.http.client.protocol.ClientContext; import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ClientConnectionManager; import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.params.ConnRouteParams; import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory; import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.scheme.Scheme; import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.scheme.SchemeRegistry; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicHeaderElementIterator; import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.apache.http.util.ByteArrayBuffer; import org.apache.http.util.ByteArrayBuffer;
@ -108,106 +98,126 @@ import org.apache.http.util.EntityUtils;
public class HTTPClient { public class HTTPClient {
private final static int maxcon = 200; private final static int maxcon = 200;
private static IdledConnectionEvictor idledConnectionEvictor = null; private static IdleConnectionMonitorThread connectionMonitor = null;
private static HttpClient httpClient = initConnectionManager(); private final static RequestConfig dfltReqConf = initRequestConfig();
private static final CredentialsProvider credsProvider = new BasicCredentialsProvider(); private final static HttpClientBuilder clientBuilder = initClientBuilder();
private final RequestConfig.Builder reqConfBuilder;
private Set<Entry<String, String>> headers = null; private Set<Entry<String, String>> headers = null;
private HttpResponse httpResponse = null; private CloseableHttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null; private HttpUriRequest currentRequest = null;
private long upbytes = 0L; private long upbytes = 0L;
private int timeout = 10000;
private ClientIdentification.Agent agent = null;
private String host = null; private String host = null;
private boolean redirecting = true;
private String realm = null; private String realm = null;
public HTTPClient(final ClientIdentification.Agent agent) { public HTTPClient(final ClientIdentification.Agent agent) {
super(); super();
this.agent = agent; clientBuilder.setUserAgent(agent.userAgent);
this.timeout = agent.clientTimeout; reqConfBuilder = RequestConfig.copy(dfltReqConf);
HttpProtocolParams.setUserAgent(httpClient.getParams(), agent.userAgent); reqConfBuilder.setSocketTimeout(agent.clientTimeout);
reqConfBuilder.setConnectTimeout(agent.clientTimeout);
reqConfBuilder.setConnectionRequestTimeout(agent.clientTimeout);
} }
public HTTPClient(final ClientIdentification.Agent agent, final int timeout) { public HTTPClient(final ClientIdentification.Agent agent, final int timeout) {
super(); super();
this.agent = agent; clientBuilder.setUserAgent(agent.userAgent);
this.timeout = timeout; reqConfBuilder = RequestConfig.copy(dfltReqConf);
HttpProtocolParams.setUserAgent(httpClient.getParams(), agent.userAgent); reqConfBuilder.setSocketTimeout(timeout);
reqConfBuilder.setConnectTimeout(timeout);
reqConfBuilder.setConnectionRequestTimeout(timeout);
} }
public static void setDefaultUserAgent(final String defaultAgent) { public static void setDefaultUserAgent(final String defaultAgent) {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent); clientBuilder.setUserAgent(defaultAgent);
} }
public static HttpClient initConnectionManager() { private static RequestConfig initRequestConfig() {
// Create and initialize scheme registry final RequestConfig.Builder builder = RequestConfig.custom();
final SchemeRegistry schemeRegistry = new SchemeRegistry(); // IMPORTANT - if not set to 'false' then servers do not process the request until a time-out of 2 seconds
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); builder.setExpectContinueEnabled(false);
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry);
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
/**
* ConnectionManager settings
*/
// how much connections do we need? - default: 20
clientConnectionManager.setMaxTotal(maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// connections per host (2 default)
clientConnectionManager.setDefaultMaxPerRoute(2);
// Increase max connections for localhost
final HttpHost localhost = new HttpHost(Domains.LOCALHOST);
clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon);
/**
* HTTP protocol settings
*/
HttpProtocolParams.setVersion(httpParams, HttpVersion.HTTP_1_1);
// UserAgent
HttpProtocolParams.setUserAgent(httpParams, ClientIdentification.yacyInternetCrawlerAgent.userAgent);
HttpProtocolParams.setUseExpectContinue(httpParams, false); // IMPORTANT - if not set to 'false' then servers do not process the request until a time-out of 2 seconds
/**
* HTTP connection settings
*/
// timeout in milliseconds until a connection is established in milliseconds // timeout in milliseconds until a connection is established in milliseconds
HttpConnectionParams.setConnectionTimeout(httpParams, 6000); builder.setConnectionRequestTimeout(6000);
// SO_LINGER affects the socket close operation in seconds builder.setConnectTimeout(8000);
// HttpConnectionParams.setLinger(httpParams, 6);
// HttpConnectionParams.setSocketBufferSize(httpParams, 8192);
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds // SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
HttpConnectionParams.setSoTimeout(httpParams, 1000); builder.setSocketTimeout(3000);
// getting an I/O error when executing a request over a connection that has been closed at the server side // getting an I/O error when executing a request over a connection that has been closed at the server side
HttpConnectionParams.setStaleCheckingEnabled(httpParams, true); builder.setStaleConnectionCheckEnabled(true);
// conserve bandwidth by minimizing the number of segments that are sent
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// Defines whether the socket can be bound even though a previous connection is still in a timeout state.
HttpConnectionParams.setSoReuseaddr(httpParams, true);
/**
* HTTP client settings
*/
// ignore cookies, cause this may cause segfaults in default cookiestore and is not needed // ignore cookies, cause this may cause segfaults in default cookiestore and is not needed
HttpClientParams.setCookiePolicy(httpParams, CookiePolicy.IGNORE_COOKIES); builder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
builder.setRedirectsEnabled(true);
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams); builder.setRelativeRedirectsAllowed(true);
return builder.build();
}
private static HttpClientBuilder initClientBuilder() {
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setConnectionManager(initPoolingConnectionManager());
builder.setDefaultRequestConfig(dfltReqConf);
// UserAgent
builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
// remove retries; we expect connections to fail; therefore we should not retry
builder.disableAutomaticRetries();
// disable the cookiestore, cause this may cause segfaults and is not needed // disable the cookiestore, cause this may cause segfaults and is not needed
((DefaultHttpClient) httpClient).setCookieStore(null); builder.setDefaultCookieStore(null);
builder.disableCookieManagement();
// add cutom keep alive strategy // add cutom keep alive strategy
addCustomKeepAliveStrategy((DefaultHttpClient) httpClient); builder.setKeepAliveStrategy(customKeepAliveStrategy());
// ask for gzip // ask for gzip
((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); builder.addInterceptorLast(new GzipRequestInterceptor());
// uncompress gzip // uncompress gzip
((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor()); builder.addInterceptorLast(new GzipResponseInterceptor());
// remove retries; we expect connections to fail; therefore we should not retry // Proxy
((DefaultHttpClient) httpClient).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); builder.setRoutePlanner(ProxySettings.RoutePlanner);
if (idledConnectionEvictor == null) { builder.setDefaultCredentialsProvider(ProxySettings.CredsProvider);
idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
idledConnectionEvictor.start(); return builder;
}
private static PoolingHttpClientConnectionManager initPoolingConnectionManager() {
final PlainConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", getSSLSocketFactory())
.build();
final PoolingHttpClientConnectionManager pooling = new PoolingHttpClientConnectionManager(registry, new DnsResolver(){
@Override
public InetAddress[] resolve(final String host0)throws UnknownHostException {
final InetAddress ip = Domains.dnsResolve(host0);
if (ip == null) throw new UnknownHostException(host0);
return new InetAddress[]{ip};
}});
// how much connections do we need? - default: 20
pooling.setMaxTotal(maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// connections per host (2 default)
pooling.setDefaultMaxPerRoute(4);
// Increase max connections for localhost
final HttpHost localhost = new HttpHost(Domains.LOCALHOST);
pooling.setMaxPerRoute(new HttpRoute(localhost), maxcon);
final SocketConfig socketConfig = SocketConfig.custom()
// Defines whether the socket can be bound even though a previous connection is still in a timeout state.
.setSoReuseAddress(true)
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
.setSoTimeout(3000)
// conserve bandwidth by minimizing the number of segments that are sent
.setTcpNoDelay(false)
.build();
pooling.setDefaultSocketConfig(socketConfig);
if (connectionMonitor == null) {
connectionMonitor = new IdleConnectionMonitorThread(pooling);
connectionMonitor.start();
} }
return httpClient;
return pooling;
} }
/** /**
@ -217,34 +227,29 @@ public class HTTPClient {
* @throws InterruptedException * @throws InterruptedException
*/ */
public static void closeConnectionManager() throws InterruptedException { public static void closeConnectionManager() throws InterruptedException {
if (idledConnectionEvictor != null) { if (connectionMonitor != null) {
// Shut down the evictor thread // Shut down the evictor thread
idledConnectionEvictor.shutdown(); connectionMonitor.shutdown();
idledConnectionEvictor.join(); connectionMonitor.join();
} }
if (httpClient != null) {
// Shut down the connection manager
httpClient.getConnectionManager().shutdown();
}
} }
public static void setAuth(final String host, final int port, final String user, final String pw) { // public static void setAuth(final String host, final int port, final String user, final String pw) {
final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw); // final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw);
final AuthScope scope = new AuthScope(host, port); // final AuthScope scope = new AuthScope(host, port);
credsProvider.setCredentials(scope, creds); // credsProvider.setCredentials(scope, creds);
httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider); // httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider);
} // }
/** // /**
* this method sets a host on which more than the default of 2 router per host are allowed // * this method sets a host on which more than the default of 2 router per host are allowed
* // *
* @param the host to be raised in 'route per host' // * @param the host to be raised in 'route per host'
*/ // */
public static void setMaxRouteHost(final String host) { // public static void setMaxRouteHost(final String host) {
final HttpHost mHost = new HttpHost(host); // final HttpHost mHost = new HttpHost(host);
((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50); // ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
} // }
/** /**
* This method sets the Header used for the request * This method sets the Header used for the request
@ -261,7 +266,9 @@ public class HTTPClient {
* @param timeout in milliseconds * @param timeout in milliseconds
*/ */
public void setTimout(final int timeout) { public void setTimout(final int timeout) {
this.timeout = timeout; reqConfBuilder.setSocketTimeout(timeout);
reqConfBuilder.setConnectTimeout(timeout);
reqConfBuilder.setConnectionRequestTimeout(timeout);
} }
/** /**
@ -270,7 +277,7 @@ public class HTTPClient {
* @param userAgent * @param userAgent
*/ */
public void setUserAgent(final ClientIdentification.Agent agent) { public void setUserAgent(final ClientIdentification.Agent agent) {
this.agent = agent; clientBuilder.setUserAgent(agent.userAgent);
} }
/** /**
@ -288,7 +295,8 @@ public class HTTPClient {
* @param redirecting * @param redirecting
*/ */
public void setRedirecting(final boolean redirecting) { public void setRedirecting(final boolean redirecting) {
this.redirecting = redirecting; reqConfBuilder.setRedirectsEnabled(redirecting);
reqConfBuilder.setRelativeRedirectsAllowed(redirecting);
} }
/** /**
@ -354,7 +362,7 @@ public class HTTPClient {
} }
httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
return getContentBytes(httpGet, url.getHost(), maxBytes); return getContentBytes(httpGet, maxBytes);
} }
/** /**
@ -378,7 +386,7 @@ public class HTTPClient {
httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state httpGet.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
this.currentRequest = httpGet; this.currentRequest = httpGet;
execute(httpGet, url.getHost()); execute(httpGet);
} }
/** /**
@ -393,7 +401,7 @@ public class HTTPClient {
final HttpHead httpHead = new HttpHead(url.toNormalform(true)); final HttpHead httpHead = new HttpHead(url.toNormalform(true));
httpHead.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state httpHead.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
execute(httpHead, url.getHost()); execute(httpHead);
finish(); finish();
ConnectionInfo.removeConnection(httpHead.hashCode()); ConnectionInfo.removeConnection(httpHead.hashCode());
return this.httpResponse; return this.httpResponse;
@ -422,7 +430,7 @@ public class HTTPClient {
this.upbytes = length; this.upbytes = length;
httpPost.setEntity(inputStreamEntity); httpPost.setEntity(inputStreamEntity);
this.currentRequest = httpPost; this.currentRequest = httpPost;
execute(httpPost, host); execute(httpPost);
} }
/** /**
@ -454,10 +462,11 @@ public class HTTPClient {
setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (vhost == null) setHost(Domains.LOCALHOST); if (vhost == null) setHost(Domains.LOCALHOST);
final MultipartEntity multipartEntity = new MultipartEntity(); final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
for (final Entry<String,ContentBody> part : post.entrySet()) for (final Entry<String,ContentBody> part : post.entrySet())
multipartEntity.addPart(part.getKey(), part.getValue()); entityBuilder.addPart(part.getKey(), part.getValue());
final HttpEntity multipartEntity = entityBuilder.build();
// statistics // statistics
this.upbytes = multipartEntity.getContentLength(); this.upbytes = multipartEntity.getContentLength();
@ -467,7 +476,7 @@ public class HTTPClient {
httpPost.setEntity(multipartEntity); httpPost.setEntity(multipartEntity);
} }
return getContentBytes(httpPost, url.getHost(), Integer.MAX_VALUE); return getContentBytes(httpPost, Integer.MAX_VALUE);
} }
/** /**
@ -491,7 +500,7 @@ public class HTTPClient {
// statistics // statistics
this.upbytes = length; this.upbytes = length;
httpPost.setEntity(inputStreamEntity); httpPost.setEntity(inputStreamEntity);
return getContentBytes(httpPost, host, Integer.MAX_VALUE); return getContentBytes(httpPost, Integer.MAX_VALUE);
} }
/** /**
@ -580,9 +589,9 @@ public class HTTPClient {
} }
} }
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, String host, final int maxBytes) throws IOException { private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final int maxBytes) throws IOException {
try { try {
execute(httpUriRequest, host); execute(httpUriRequest);
if (this.httpResponse == null) return null; if (this.httpResponse == null) return null;
// get the response body // get the response body
final HttpEntity httpEntity = this.httpResponse.getEntity(); final HttpEntity httpEntity = this.httpResponse.getEntity();
@ -602,11 +611,13 @@ public class HTTPClient {
} }
} }
private void execute(final HttpUriRequest httpUriRequest, String host) throws IOException { private void execute(final HttpUriRequest httpUriRequest) throws IOException {
final HttpContext httpContext = new BasicHttpContext(); final HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(reqConfBuilder.build());
if (this.host != null)
context.setTargetHost(new HttpHost(this.host));
setHeaders(httpUriRequest); setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams(), host);
// statistics // statistics
storeConnectionInfo(httpUriRequest); storeConnectionInfo(httpUriRequest);
// execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF // execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF
@ -620,14 +631,17 @@ public class HTTPClient {
} }
Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost()); Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost());
final long time = System.currentTimeMillis();
try { try {
final long time = System.currentTimeMillis(); final CloseableHttpClient client = clientBuilder.build();
this.httpResponse = httpClient.execute(httpUriRequest, httpContext); this.httpResponse = client.execute(httpUriRequest, context);
this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (final IOException e) { } catch (final IOException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode()); ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort(); httpUriRequest.abort();
throw new IOException("Client can't execute: " + (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())); throw new IOException("Client can't execute: "
+ (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())
+ " duration=" + Long.toString(System.currentTimeMillis() - time));
} }
} }
@ -669,23 +683,6 @@ public class HTTPClient {
httpUriRequest.setHeader("Authorization", "realm=" + this.realm); httpUriRequest.setHeader("Authorization", "realm=" + this.realm);
} }
private void setParams(final HttpParams httpParams) {
HttpClientParams.setRedirecting(httpParams, this.redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, this.timeout);
HttpConnectionParams.setSoTimeout(httpParams, this.timeout);
if (this.agent != null)
HttpProtocolParams.setUserAgent(httpParams, this.agent.userAgent);
if (this.host != null)
httpParams.setParameter(HTTP.TARGET_HOST, this.host);
}
private static void setProxy(final HttpParams httpParams, String host) {
if (ProxySettings.useForHost(host, Protocol.HTTP))
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((DefaultHttpClient) httpClient);
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) { private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort(); final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost(); final String thost = httpUriRequest.getURI().getHost();
@ -699,7 +696,7 @@ public class HTTPClient {
this.upbytes)); this.upbytes));
} }
private static SSLSocketFactory getSSLSocketFactory() { private static SSLConnectionSocketFactory getSSLSocketFactory() {
final TrustManager trustManager = new X509TrustManager() { final TrustManager trustManager = new X509TrustManager() {
@Override @Override
public void checkClientTrusted(final X509Certificate[] chain, final String authType) public void checkClientTrusted(final X509Certificate[] chain, final String authType)
@ -728,7 +725,9 @@ public class HTTPClient {
// e.printStackTrace(); // e.printStackTrace();
} }
final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); final SSLConnectionSocketFactory sslSF = new SSLConnectionSocketFactory(
sslContext,
SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
return sslSF; return sslSF;
} }
@ -739,8 +738,8 @@ public class HTTPClient {
* *
* @param defaultHttpClient * @param defaultHttpClient
*/ */
private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) { private static ConnectionKeepAliveStrategy customKeepAliveStrategy() {
defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() { return new ConnectionKeepAliveStrategy() {
@Override @Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) { public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// Honor 'keep-alive' header // Honor 'keep-alive' header
@ -762,7 +761,7 @@ public class HTTPClient {
// Keep alive for 5 seconds only // Keep alive for 5 seconds only
return 5 * 1000; return 5 * 1000;
} }
}); };
} }
/** /**
@ -773,13 +772,13 @@ public class HTTPClient {
public static void main(final String[] args) { public static void main(final String[] args) {
String url = null; String url = null;
// prepare Parts // prepare Parts
final Map<String,ContentBody> newparts = new LinkedHashMap<String,ContentBody>(); // final Map<String,ContentBody> newparts = new LinkedHashMap<String,ContentBody>();
try { // try {
newparts.put("foo", new StringBody("FooBar")); // newparts.put("foo", new StringBody("FooBar"));
newparts.put("bar", new StringBody("BarFoo")); // newparts.put("bar", new StringBody("BarFoo"));
} catch (final UnsupportedEncodingException e) { // } catch (final UnsupportedEncodingException e) {
System.out.println(e.getStackTrace()); // System.out.println(e.getStackTrace());
} // }
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setRedirecting(false); client.setRedirecting(false);
// Get some // Get some
@ -805,7 +804,7 @@ public class HTTPClient {
// for (HeaderElement element: header.getElements()) // for (HeaderElement element: header.getElements())
// System.out.println("Element " + element.getName() + " : " + element.getValue()); // System.out.println("Element " + element.getName() + " : " + element.getValue());
} }
System.out.println(client.getHttpResponse().getLocale()); // System.out.println(client.getHttpResponse().getLocale());
System.out.println(client.getHttpResponse().getProtocolVersion()); System.out.println(client.getHttpResponse().getProtocolVersion());
System.out.println(client.getHttpResponse().getStatusLine()); System.out.println(client.getHttpResponse().getStatusLine());
// Post some // Post some
@ -822,49 +821,41 @@ public class HTTPClient {
} }
} }
public static class IdleConnectionMonitorThread extends Thread {
private final HttpClientConnectionManager connMgr;
private volatile boolean shutdown;
public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
super();
this.connMgr = connMgr;
}
/** @Override
* public void run() {
* @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638 try {
* while (!shutdown) {
*/ synchronized (this) {
private static class IdledConnectionEvictor extends Thread { wait(5000);
// Close expired connections
private final ClientConnectionManager clientConnectionManager; connMgr.closeExpiredConnections();
// Optionally, close connections
private volatile boolean shutdown; // that have been idle longer than 30 sec
connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
public IdledConnectionEvictor(final ClientConnectionManager clientConnectionManager) { }
super(); }
this.clientConnectionManager = clientConnectionManager; connMgr.shutdown();
} } catch (final InterruptedException ex) {
// terminate
@Override }
public void run() { }
try {
while (!this.shutdown) { public void shutdown() {
synchronized (this) { shutdown = true;
wait(5000); synchronized (this) {
// Close expired connections notifyAll();
this.clientConnectionManager.closeExpiredConnections(); }
// Optionally, close connections }
// that have been idle longer than 5 sec
// (some SOHO router act strange on >5sec idled connections)
this.clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS);
}
}
} catch (final InterruptedException ex) {
// terminate
}
}
public void shutdown() {
this.shutdown = true;
synchronized (this) {
notifyAll();
}
}
} }
} }

@ -27,10 +27,16 @@ package net.yacy.cora.protocol.http;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import org.apache.http.HttpException;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.AbstractHttpClient; import org.apache.http.client.CredentialsProvider;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.routing.HttpRoutePlanner;
import org.apache.http.protocol.HttpContext;
/** /**
* settings for a remote proxy * settings for a remote proxy
@ -71,12 +77,36 @@ public final class ProxySettings {
return new HttpHost(host, port); return new HttpHost(host, port);
} }
public static void setProxyCreds(AbstractHttpClient httpClient) { public static HttpRoutePlanner RoutePlanner = new HttpRoutePlanner() {
if (!use) return;
httpClient.getCredentialsProvider().setCredentials( @Override
new AuthScope(host, port), public HttpRoute determineRoute(HttpHost target, HttpRequest request, HttpContext context) throws HttpException {
new UsernamePasswordCredentials(user, password)); if (use) {
} final Protocol protocol = "https".equalsIgnoreCase(target.getSchemeName())? Protocol.HTTPS : Protocol.HTTP;
if (useForHost(target.getHostName(), protocol))
return new HttpRoute(target, null, getProxyHost(), protocol == Protocol.HTTPS);
}
return new HttpRoute(target); // direct
}
};
public static CredentialsProvider CredsProvider = new CredentialsProvider() {
@Override
public void clear() {
}
@Override
public Credentials getCredentials(AuthScope scope) {
if (host != null && host.equals(scope.getHost()) && port == scope.getPort())
return new UsernamePasswordCredentials(user, password);
return null;
}
@Override
public void setCredentials(AuthScope arg0, Credentials arg1) {
}
};
/** /**
* tell if a remote proxy will be used for the given host * tell if a remote proxy will be used for the given host

Loading…
Cancel
Save