diff --git a/.classpath b/.classpath index 2bf8103c3..06a5e7e44 100644 --- a/.classpath +++ b/.classpath @@ -35,12 +35,12 @@ - - + + diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist index 1d498c03b..12fce02c8 100644 --- a/addon/YaCy.app/Contents/Info.plist +++ b/addon/YaCy.app/Contents/Info.plist @@ -25,7 +25,7 @@ Java VMOptions - -Xmx600m -Xms180m -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 + -Xmx600m -Xms180m -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 WorkingDirectory $APP_PACKAGE/Contents/Resources/Java MainClass @@ -45,7 +45,7 @@ $JAVAROOT/lib/commons-io-1.4.jar $JAVAROOT/lib/commons-jxpath-1.3.jar $JAVAROOT/lib/commons-logging-1.1.1.jar - $JAVAROOT/lib/fontbox-1.1.0.jar + $JAVAROOT/lib/fontbox-1.2.1.jar $JAVAROOT/lib/activation.jar $JAVAROOT/lib/apache-mime4j-0.6.jar $JAVAROOT/lib/J7Zip-modified.jar @@ -58,7 +58,7 @@ $JAVAROOT/lib/log4j-1.2.15.jar $JAVAROOT/lib/metadata-extractor-2.4.0-beta-1.jar $JAVAROOT/lib/mysql-connector-java-5.1.12-bin.jar - $JAVAROOT/lib/pdfbox-1.1.0.jar + $JAVAROOT/lib/pdfbox-1.2.1.jar $JAVAROOT/lib/poi-3.6-20091214.jar $JAVAROOT/lib/poi-scratchpad-3.6-20091214.jar $JAVAROOT/lib/servlet-api.jar diff --git a/addon/yacyInit.m4 b/addon/yacyInit.m4 index 79d5bfedd..04326d25a 100644 --- a/addon/yacyInit.m4 +++ b/addon/yacyInit.m4 @@ -57,7 +57,7 @@ SHUTDOWN_TIMEOUT=50 # Default niceness if not set in config file NICE_VAL=0 -JAVA_ARGS="-server -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+UseAdaptiveSizePolicy -Djava.awt.headless=true -Dfile.encoding=UTF-8" +JAVA_ARGS="-server -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8" #check if system supports large memory pages and enable it if possible HUGEPAGESTOTAL="$(cat /proc/meminfo | grep HugePages_Total | sed s/[^0-9]//g)" if [ -n "$HUGEPAGESTOTAL" ] && [ $HUGEPAGESTOTAL -ne 0 ] diff --git a/build.xml b/build.xml index 924c4562e..e175f7605 100644 --- a/build.xml +++ b/build.xml @@ -189,7 +189,7 @@ - + @@ -200,7 +200,7 @@ - + diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.java index 32e46b6ae..b6d76e329 100644 --- a/htroot/Threaddump_p.java +++ b/htroot/Threaddump_p.java @@ -25,29 +25,19 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// You must compile this file with -// javac -classpath .:../Classes Blacklist_p.java -// if the shell's current path is HTROOT import java.io.File; -import java.io.IOException; import java.util.Date; import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.Map.Entry; import java.util.ArrayList; -import java.util.HashMap; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.logging.ThreadDumpGenerator; import net.yacy.kelondro.util.MemoryControl; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.tools.nxTools; import de.anomic.yacy.yacyBuildProperties; public class Threaddump_p { @@ -68,14 +58,14 @@ public class Threaddump_p { final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision(); Runtime runtime = Runtime.getRuntime(); - bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************"); - bufferappend(buffer, plain, ""); - bufferappend(buffer, plain, "YaCy Version: " + versionstring); - bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory())); - bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory())); - bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory())); - bufferappend(buffer, plain, ""); - bufferappend(buffer, plain, ""); + ThreadDumpGenerator.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************"); + ThreadDumpGenerator.bufferappend(buffer, plain, ""); + ThreadDumpGenerator.bufferappend(buffer, plain, "YaCy Version: " + versionstring); + ThreadDumpGenerator.bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory())); + ThreadDumpGenerator.bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory())); + ThreadDumpGenerator.bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory())); + ThreadDumpGenerator.bufferappend(buffer, plain, ""); + ThreadDumpGenerator.bufferappend(buffer, plain, ""); int multipleCount = 100; File appPath = sb.getAppPath(); @@ -86,27 +76,27 @@ public class Threaddump_p { traces.add(Thread.getAllStackTraces()); if (MemoryControl.available() < 20 * 1024 * 1024) break; } - appendStackTraceStats(appPath, buffer, traces, plain, null); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, null); /* - appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.BLOCKED); - appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.RUNNABLE); - appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.TIMED_WAITING); - appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.WAITING); - appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.NEW); - appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.TERMINATED); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.BLOCKED); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.RUNNABLE); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.TIMED_WAITING); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.WAITING); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.NEW); + ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.TERMINATED); */ } else { // generate a single thread dump final Map stackTraces = Thread.getAllStackTraces(); - appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.BLOCKED); - appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.RUNNABLE); - appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TIMED_WAITING); - appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.WAITING); - appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.NEW); - appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TERMINATED); + ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.BLOCKED); + ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.RUNNABLE); + ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TIMED_WAITING); + ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.WAITING); + ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.NEW); + ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TERMINATED); } - bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************"); + ThreadDumpGenerator.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************"); prop.put("plain_count", multipleCount); prop.put("plain_content", buffer.toString()); @@ -115,153 +105,6 @@ public class Threaddump_p { return prop; // return from serverObjects respond() } - private static void appendStackTraces(final File rootPath, final StringBuilder buffer, final Map stackTraces, final boolean plain, final Thread.State stateIn) { - bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString()); - bufferappend(buffer, plain, ""); - // collect single dumps - HashMap> dumps = dumpCollection(rootPath, stackTraces, plain, stateIn); - - // write dumps - for (final Entry> entry: dumps.entrySet()) { - SortedSet threads = entry.getValue(); - for (String t: threads) bufferappend(buffer, plain, t); - bufferappend(buffer, plain, entry.getKey()); - bufferappend(buffer, plain, ""); - } - bufferappend(buffer, plain, ""); - } - - private static void appendStackTraceStats(final File rootPath, final StringBuilder buffer, final ArrayList> traces, final boolean plain, final Thread.State stateIn) { - if (stateIn != null) { - bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString()); - bufferappend(buffer, plain, ""); - } - // collect single dumps - HashMap dumps = dumpStatistic(rootPath, traces, plain, stateIn); - - // write dumps - while (!dumps.isEmpty()) { - Entry e = removeMax(dumps); - bufferappend(buffer, plain, "Occurrences: " + e.getValue()); - bufferappend(buffer, plain, e.getKey()); - //bufferappend(buffer, plain, ""); - } - bufferappend(buffer, plain, ""); - } - - private static Entry removeMax(HashMap result) { - Entry max = null; - for (final Entry e: result.entrySet()) { - if (max == null || e.getValue().intValue() > max.getValue().intValue()) { - max = e; - } - } - result.remove(max.getKey()); - return max; - } - - private static HashMap dumpStatistic(final File rootPath, final ArrayList> stackTraces, final boolean plain, final Thread.State stateIn) { - Map trace; - HashMap result = new HashMap(); - HashMap> x; - int count; - for (int i = 0; i < stackTraces.size(); i++) { - trace = stackTraces.get(i); - x = dumpCollection(rootPath, trace, plain, stateIn); - for (final Entry> e: x.entrySet()) { - Integer c = result.get(e.getKey()); - count = e.getValue().size(); - if (c == null) result.put(e.getKey(), Integer.valueOf(count)); - else { - c = Integer.valueOf(c.intValue() + count); - result.put(e.getKey(), c); - } - } - } - return result; - } - - private static HashMap> dumpCollection(final File appPath, final Map stackTraces, final boolean plain, final Thread.State stateIn) { - final File classPath = new File(appPath, "source"); - - Thread thread; - // collect single dumps - HashMap> dumps = new HashMap>(); - for (final Entry entry: stackTraces.entrySet()) { - thread = entry.getKey(); - final StackTraceElement[] stackTraceElements = entry.getValue(); - StackTraceElement ste; - String line; - String tracename = ""; - File classFile; - if ((stateIn == null || stateIn.equals(thread.getState())) && stackTraceElements.length > 0) { - StringBuilder sb = new StringBuilder(3000); - if (plain) { - classFile = getClassFile(classPath, stackTraceElements[stackTraceElements.length - 1].getClassName()); - tracename = classFile.getName(); - if (tracename.endsWith(".java")) tracename = tracename.substring(0, tracename.length() - 5); - if (tracename.length() > 20) tracename = tracename.substring(0, 20); - while (tracename.length() < 20) tracename = tracename + "_"; - tracename = "[" + tracename + "] "; - } - String threadtitle = tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString(); - String className; - boolean cutcore = true; - for (int i = 0; i < stackTraceElements.length; i++) { - ste = stackTraceElements[i]; - className = ste.getClassName(); - if (cutcore && (className.startsWith("java.") || className.startsWith("sun."))) { - sb.setLength(0); - bufferappend(sb, plain, tracename + "at " + CharacterCoding.unicode2html(ste.toString(), true)); - } else { - cutcore = false; - if (i == 0) { - line = getLine(getClassFile(classPath, className), ste.getLineNumber()); - } else { - line = null; - } - if ((line != null) && (line.length() > 0)) { - bufferappend(sb, plain, tracename + "at " + CharacterCoding.unicode2html(ste.toString(), true) + " [" + line.trim() + "]"); - } else { - bufferappend(sb, plain, tracename + "at " + CharacterCoding.unicode2html(ste.toString(), true)); - } - } - } - String threaddump = sb.toString(); - SortedSet threads = dumps.get(threaddump); - if (threads == null) threads = new TreeSet(); - threads.add(threadtitle); - dumps.put(threaddump, threads); - } - } - return dumps; - } - private static File getClassFile(final File sourcePath, final String classname) { - final String classPath = classname.replace('.', '/') + ".java"; - final File file = new File(sourcePath, classPath); - return file; - } - - private static String getLine(final File file, final int line) { - // find class - if (!file.exists()) return ""; - try { - final String lineString = nxTools.line(FileUtils.read(file), line); - if (lineString == null) return "@ERROR"; - return lineString; - } catch (final IOException e) { - return "@EXCEPTION: " + e.getMessage(); - } - } - - private static void bufferappend(final StringBuilder buffer, final boolean plain, final String a) { - buffer.append(a); - if (plain) { - buffer.append("\n"); - } else { - buffer.append("
"); - } - } } diff --git a/lib/fontbox-1.1.0.License b/lib/fontbox-1.1.0.License deleted file mode 100644 index e5e25bbea..000000000 --- a/lib/fontbox-1.1.0.License +++ /dev/null @@ -1,236 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -CONTRIBUTIONS TO THE ORIGINAL CODEBASE - -Apache FontBox is based on contributions made to the original FontBox project: - - Copyright (c) 2006-2007, www.fontbox.org - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of fontbox; nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. diff --git a/lib/fontbox-1.1.0.jar b/lib/fontbox-1.2.1.jar similarity index 55% rename from lib/fontbox-1.1.0.jar rename to lib/fontbox-1.2.1.jar index 3bc5a7691..87cc6a768 100644 Binary files a/lib/fontbox-1.1.0.jar and b/lib/fontbox-1.2.1.jar differ diff --git a/lib/pdfbox-1.1.0.License b/lib/pdfbox-1.1.0.License deleted file mode 100644 index 329a3112a..000000000 --- a/lib/pdfbox-1.1.0.License +++ /dev/null @@ -1,587 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -EXTERNAL COMPONENTS - -Apache PDFBox includes a number of components with separate copyright notices -and license terms. Your use of these components is subject to the terms and -conditions of the following licenses. - -Contributions made to the original PDFBox, JempBox and FontBox projects: - - Copyright (c) 2002-2007, www.pdfbox.org - Copyright (c) 2006-2007, www.jempbox.org - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of pdfbox; nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - -Bouncy Castle encryption libraries - - Copyright (c) 2000-2006 The Legion Of The Bouncy Castle - (http://www.bouncycastle.org) - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files - (the "Software"), to deal in the Software without restriction, - including without limitation the rights to use, copy, modify, merge, - publish, distribute, sublicense, and/or sell copies of the Software, - and to permit persons to whom the Software is furnished to do so, - subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - -Adobe Font Metrics (AFM) for PDF Core 14 Fonts - - This file and the 14 PostScript(R) AFM files it accompanies may be used, - copied, and distributed for any purpose and without charge, with or without - modification, provided that all copyright notices are retained; that the - AFM files are not distributed without this file; that all modifications - to this file or any of the AFM files are prominently noted in the modified - file(s); and that this paragraph is not modified. Adobe Systems has no - responsibility or obligation to support the use of the AFM files. - -CMaps for PDF Fonts (http://www.adobe.com/devnet/font/#pcfi and -ftp://ftp.oreilly.com/pub/examples/nutshell/cjkv/adobe/) - - Copyright 1990-2001 Adobe Systems Incorporated. - All Rights Reserved. - - Patents Pending - - NOTICE: All information contained herein is the property - of Adobe Systems Incorporated. - - Permission is granted for redistribution of this file - provided this copyright notice is maintained intact and - that the contents of this file are not altered in any - way from its original form. - - PostScript and Display PostScript are trademarks of - Adobe Systems Incorporated which may be registered in - certain jurisdictions. - -Glyphlist (http://www.adobe.com/devnet/opentype/archives/glyph.html) - - Copyright (c) 1997,1998,2002,2007 Adobe Systems Incorporated - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this documentation file to use, copy, publish, distribute, - sublicense, and/or sell copies of the documentation, and to permit - others to do the same, provided that: - - No modification, editing or other alteration of this document is - allowed; and - - The above copyright notice and this permission notice shall be - included in all copies of the documentation. - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this documentation file, to create their own derivative works - from the content of this document to use, copy, publish, distribute, - sublicense, and/or sell the derivative works, and to permit others to do - the same, provided that the derived work is not represented as being a - copy or version of this document. - - Adobe shall not be liable to any party for any loss of revenue or profit - or for indirect, incidental, special, consequential, or other similar - damages, whether based on tort (including without limitation negligence - or strict liability), contract or other legal or equitable grounds even - if Adobe has been advised or had reason to know of the possibility of - such damages. The Adobe materials are provided on an "AS IS" basis. - Adobe specifically disclaims all express, statutory, or implied - warranties relating to the Adobe materials, including but not limited to - those concerning merchantability or fitness for a particular purpose or - non-infringement of any third party rights regarding the Adobe - materials. - -The JUnit test framework (http://junit.org/) - - Common Public License - v 1.0 - - THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC - LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM - CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - - 1. DEFINITIONS - - "Contribution" means: - - a) in the case of the initial Contributor, the initial code and - documentation distributed under this Agreement, and - - b) in the case of each subsequent Contributor: - - i) changes to the Program, and - - ii) additions to the Program; - - where such changes and/or additions to the Program originate from and - are distributed by that particular Contributor. A Contribution - 'originates' from a Contributor if it was added to the Program by such - Contributor itself or anyone acting on such Contributor's behalf. - Contributions do not include additions to the Program which: - (i) are separate modules of software distributed in conjunction with - the Program under their own license agreement, and (ii) are not - derivative works of the Program. - - "Contributor" means any person or entity that distributes the Program. - - "Licensed Patents " mean patent claims licensable by a Contributor which - are necessarily infringed by the use or sale of its Contribution alone or - when combined with the Program. - - "Program" means the Contributions distributed in accordance with this - Agreement. - - "Recipient" means anyone who receives the Program under this Agreement, - including all Contributors. - - 2. GRANT OF RIGHTS - - a) Subject to the terms of this Agreement, each Contributor hereby grants - Recipient a non-exclusive, worldwide, royalty-free copyright license to - reproduce, prepare derivative works of, publicly display, publicly - perform, distribute and sublicense the Contribution of such Contributor, - if any, and such derivative works, in source code and object code form. - - b) Subject to the terms of this Agreement, each Contributor hereby grants - Recipient a non-exclusive, worldwide, royalty-free patent license under - Licensed Patents to make, use, sell, offer to sell, import and otherwise - transfer the Contribution of such Contributor, if any, in source code - and object code form. This patent license shall apply to the combination - of the Contribution and the Program if, at the time the Contribution is - added by the Contributor, such addition of the Contribution causes such - combination to be covered by the Licensed Patents. The patent license - shall not apply to any other combinations which include the - Contribution. No hardware per se is licensed hereunder. - - c) Recipient understands that although each Contributor grants the licenses - to its Contributions set forth herein, no assurances are provided by any - Contributor that the Program does not infringe the patent or other - intellectual property rights of any other entity. Each Contributor - disclaims any liability to Recipient for claims brought by any other - entity based on infringement of intellectual property rights or - otherwise. As a condition to exercising the rights and licenses granted - hereunder, each Recipient hereby assumes sole responsibility to secure - any other intellectual property rights needed, if any. For example, if - a third party patent license is required to allow Recipient to - distribute the Program, it is Recipient's responsibility to acquire that - license before distributing the Program. - - d) Each Contributor represents that to its knowledge it has sufficient - copyright rights in its Contribution, if any, to grant the copyright - license set forth in this Agreement. - - 3. REQUIREMENTS - - A Contributor may choose to distribute the Program in object code form - under its own license agreement, provided that: - - a) it complies with the terms and conditions of this Agreement; and - - b) its license agreement: - - i) effectively disclaims on behalf of all Contributors all warranties - and conditions, express and implied, including warranties or - conditions of title and non-infringement, and implied warranties or - conditions of merchantability and fitness for a particular purpose; - - ii) effectively excludes on behalf of all Contributors all liability for - damages, including direct, indirect, special, incidental and - consequential damages, such as lost profits; - - iii) states that any provisions which differ from this Agreement are - offered by that Contributor alone and not by any other party; and - - iv) states that source code for the Program is available from such - Contributor, and informs licensees how to obtain it in a reasonable - manner on or through a medium customarily used for software - exchange. - - When the Program is made available in source code form: - - a) it must be made available under this Agreement; and - - b) a copy of this Agreement must be included with each copy of the Program. - - Contributors may not remove or alter any copyright notices contained within - the Program. - - Each Contributor must identify itself as the originator of its - Contribution, if any, in a manner that reasonably allows subsequent - Recipients to identify the originator of the Contribution. - - 4. COMMERCIAL DISTRIBUTION - - Commercial distributors of software may accept certain responsibilities - with respect to end users, business partners and the like. While this - license is intended to facilitate the commercial use of the Program, the - Contributor who includes the Program in a commercial product offering - should do so in a manner which does not create potential liability for - other Contributors. Therefore, if a Contributor includes the Program in - a commercial product offering, such Contributor ("Commercial Contributor") - hereby agrees to defend and indemnify every other Contributor ("Indemnified - Contributor") against any losses, damages and costs (collectively "Losses") - arising from claims, lawsuits and other legal actions brought by a third - party against the Indemnified Contributor to the extent caused by the acts - or omissions of such Commercial Contributor in connection with its - distribution of the Program in a commercial product offering. The - obligations in this section do not apply to any claims or Losses relating - to any actual or alleged intellectual property infringement. In order to - qualify, an Indemnified Contributor must: a) promptly notify the Commercial - Contributor in writing of such claim, and b) allow the Commercial - Contributor to control, and cooperate with the Commercial Contributor in, - the defense and any related settlement negotiations. The Indemnified - Contributor may participate in any such claim at its own expense. - - For example, a Contributor might include the Program in a commercial - product offering, Product X. That Contributor is then a Commercial - Contributor. If that Commercial Contributor then makes performance claims, - or offers warranties related to Product X, those performance claims and - warranties are such Commercial Contributor's responsibility alone. Under - this section, the Commercial Contributor would have to defend claims - against the other Contributors related to those performance claims and - warranties, and if a court requires any other Contributor to pay any - damages as a result, the Commercial Contributor must pay those damages. - - 5. NO WARRANTY - - EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED - ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER - EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR - CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A - PARTICULAR PURPOSE. Each Recipient is solely responsible for determining - the appropriateness of using and distributing the Program and assumes all - risks associated with its exercise of rights under this Agreement, - including but not limited to the risks and costs of program errors, - compliance with applicable laws, damage to or loss of data, programs or - equipment, and unavailability or interruption of operations. - - 6. DISCLAIMER OF LIABILITY - - EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY - CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION - LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE - EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGES. - - 7. GENERAL - - If any provision of this Agreement is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of the - remainder of the terms of this Agreement, and without further action by - the parties hereto, such provision shall be reformed to the minimum extent - necessary to make such provision valid and enforceable. - - If Recipient institutes patent litigation against a Contributor with - respect to a patent applicable to software (including a cross-claim or - counterclaim in a lawsuit), then any patent licenses granted by that - Contributor to such Recipient under this Agreement shall terminate as of - the date such litigation is filed. In addition, if Recipient institutes - patent litigation against any entity (including a cross-claim or - counterclaim in a lawsuit) alleging that the Program itself (excluding - combinations of the Program with other software or hardware) infringes - such Recipient's patent(s), then such Recipient's rights granted under - Section 2(b) shall terminate as of the date such litigation is filed. - - All Recipient's rights under this Agreement shall terminate if it fails - to comply with any of the material terms or conditions of this Agreement - and does not cure such failure in a reasonable period of time after - becoming aware of such noncompliance. If all Recipient's rights under this - Agreement terminate, Recipient agrees to cease use and distribution of the - Program as soon as reasonably practicable. However, Recipient's obligations - under this Agreement and any licenses granted by Recipient relating to the - Program shall continue and survive. - - Everyone is permitted to copy and distribute copies of this Agreement, but - in order to avoid inconsistency the Agreement is copyrighted and may only - be modified in the following manner. The Agreement Steward reserves the - right to publish new versions (including revisions) of this Agreement from - time to time. No one other than the Agreement Steward has the right to - modify this Agreement. IBM is the initial Agreement Steward. IBM may assign - the responsibility to serve as the Agreement Steward to a suitable separate - entity. Each new version of the Agreement will be given a distinguishing - version number. The Program (including Contributions) may always be - distributed subject to the version of the Agreement under which it was - received. In addition, after a new version of the Agreement is published, - Contributor may elect to distribute the Program (including its - Contributions) under the new version. Except as expressly stated in - Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to - the intellectual property of any Contributor under this Agreement, whether - expressly, by implication, estoppel or otherwise. All rights in the Program - not expressly granted under this Agreement are reserved. - - This Agreement is governed by the laws of the State of New York and the - intellectual property laws of the United States of America. No party to - this Agreement will bring a legal action under this Agreement more than - one year after the cause of action arose. Each party waives its rights to - a jury trial in any resulting litigation. - -The International Components for Unicode library (http://site.icu-project.org/) - - ICU License - ICU 1.8.1 and later - - COPYRIGHT AND PERMISSION NOTICE - - Copyright (c) 1995-2009 International Business Machines Corporation and others - - All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, provided that the above copyright notice(s) and this - permission notice appear in all copies of the Software and that both the - above copyright notice(s) and this permission notice appear in supporting - documentation. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE - BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, - OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, - ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - Except as contained in this notice, the name of a copyright holder shall - not be used in advertising or otherwise to promote the sale, use or other - dealings in this Software without prior written authorization of the - copyright holder. - diff --git a/lib/pdfbox-1.1.0.jar b/lib/pdfbox-1.2.1.jar similarity index 88% rename from lib/pdfbox-1.1.0.jar rename to lib/pdfbox-1.2.1.jar index ed19b4509..4d1731be8 100644 Binary files a/lib/pdfbox-1.1.0.jar and b/lib/pdfbox-1.2.1.jar differ diff --git a/source/de/anomic/crawler/RobotsEntry.java b/source/de/anomic/crawler/RobotsEntry.java index 87848bc0d..1dd887c2d 100644 --- a/source/de/anomic/crawler/RobotsEntry.java +++ b/source/de/anomic/crawler/RobotsEntry.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.kelondro.util.ByteArray; public class RobotsEntry { @@ -151,7 +152,7 @@ public class RobotsEntry { public Date getLoadedDate() { if (this.mem.containsKey(LOADED_DATE)) { - return new Date(Long.parseLong(new String(this.mem.get(LOADED_DATE)))); + return new Date(ByteArray.parseDecimal(this.mem.get(LOADED_DATE))); } return null; } @@ -164,7 +165,7 @@ public class RobotsEntry { public Date getModDate() { if (this.mem.containsKey(MOD_DATE)) { - return new Date(Long.parseLong(new String(this.mem.get(MOD_DATE)))); + return new Date(ByteArray.parseDecimal(this.mem.get(MOD_DATE))); } return null; } @@ -178,12 +179,12 @@ public class RobotsEntry { public long getCrawlDelayMillis() { if (this.mem.containsKey(CRAWL_DELAY_MILLIS)) try { - return Long.parseLong(new String(this.mem.get(CRAWL_DELAY_MILLIS))); + return ByteArray.parseDecimal(this.mem.get(CRAWL_DELAY_MILLIS)); } catch (final NumberFormatException e) { return 0; } if (this.mem.containsKey(CRAWL_DELAY)) try { - return 1000 * Integer.parseInt(new String(this.mem.get(CRAWL_DELAY))); + return 1000 * ByteArray.parseDecimal(this.mem.get(CRAWL_DELAY)); } catch (final NumberFormatException e) { return 0; } diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index 949fc375a..a7c0c7595 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -79,7 +79,7 @@ public final class Cache { // open the response header database final File dbfile = new File(cachePath, RESPONSE_HEADER_DB_NAME); try { - responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 1024, 500, '_'); + responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 1024, 100, '_'); } catch (final IOException e) { Log.logException(e); } diff --git a/source/de/anomic/search/TextSnippet.java b/source/de/anomic/search/TextSnippet.java index 8300356f0..f8a84b437 100644 --- a/source/de/anomic/search/TextSnippet.java +++ b/source/de/anomic/search/TextSnippet.java @@ -318,10 +318,11 @@ public class TextSnippet implements Comparable, Comparator, Comparator, Comparator, Comparator snippetMaxLength) line = line.substring(0, snippetMaxLength); // finally store this snippet in our own cache - storeToCache(wordhashes, new String(url.hash()), line); + storeToCache(wordhashes, urls, line); document.close(); return new TextSnippet(url, line, source, null, null, resFavicon); diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 2397931ec..14f223d84 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -55,6 +55,7 @@ import java.util.Random; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.HandleSet; @@ -238,9 +239,11 @@ public class yacySeed implements Cloneable { * @param name * @return a checked name without "<" and ">" */ + final static Pattern ltp = Pattern.compile("<"); + final static Pattern gtp = Pattern.compile(">"); private static String checkPeerName(String name) { - name = name.replaceAll("<", "_"); - name = name.replaceAll(">", "_"); + name = ltp.matcher(name).replaceAll("_"); + name = gtp.matcher(name).replaceAll("_"); return name; } diff --git a/source/net/yacy/cora/storage/ConcurrentARC.java b/source/net/yacy/cora/storage/ConcurrentARC.java index bb9b29631..455d53d0f 100644 --- a/source/net/yacy/cora/storage/ConcurrentARC.java +++ b/source/net/yacy/cora/storage/ConcurrentARC.java @@ -51,8 +51,10 @@ public final class ConcurrentARC extends AbstractMap implements Map< public ConcurrentARC(final int cacheSize, final int partitions) { int m = 1; while (m < partitions) m = m * 2; + int partitionSize = cacheSize / m; + if (partitionSize < 4) partitionSize = 4; this.arc = new HashARC[m]; - for (int i = 0; i < this.arc.length; i++) this.arc[i] = new HashARC(cacheSize / m); + for (int i = 0; i < this.arc.length; i++) this.arc[i] = new HashARC(partitionSize); m -= 1; this.mask = m; } @@ -67,8 +69,10 @@ public final class ConcurrentARC extends AbstractMap implements Map< public ConcurrentARC(final int cacheSize, final int partitions, Comparator comparator) { int m = 1; while (m < partitions) m = m * 2; + int partitionSize = cacheSize / m; + if (partitionSize < 4) partitionSize = 4; this.arc = new ComparableARC[m]; - for (int i = 0; i < this.arc.length; i++) this.arc[i] = new ComparableARC(cacheSize / m, comparator); + for (int i = 0; i < this.arc.length; i++) this.arc[i] = new ComparableARC(partitionSize, comparator); m -= 1; this.mask = m; } diff --git a/source/net/yacy/document/parser/pdfParser.java b/source/net/yacy/document/parser/pdfParser.java index 61f1383eb..703f632e4 100644 --- a/source/net/yacy/document/parser/pdfParser.java +++ b/source/net/yacy/document/parser/pdfParser.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.exceptions.CryptographyException; import org.apache.pdfbox.pdfparser.PDFParser; import org.apache.pdfbox.pdmodel.PDDocument; @@ -41,6 +42,7 @@ import org.apache.pdfbox.pdmodel.PDDocumentInformation; import org.apache.pdfbox.pdmodel.encryption.AccessPermission; import org.apache.pdfbox.pdmodel.encryption.BadSecurityHandlerException; import org.apache.pdfbox.pdmodel.encryption.StandardDecryptionMaterial; +import org.apache.pdfbox.pdmodel.font.PDFont; import org.apache.pdfbox.util.PDFTextStripper; import net.yacy.cora.document.MultiProtocolURI; @@ -112,13 +114,13 @@ public class pdfParser extends AbstractParser implements Parser { } CharBuffer writer = null; - PDFTextStripper stripper = null; try { // create a writer for output + PDFTextStripper stripper = null; writer = new CharBuffer(); stripper = new PDFTextStripper(); stripper.writeText(pdfDoc, writer); // may throw a NPE - pdfDoc.close(); + pdfDoc.close(); writer.close(); } catch (IOException e) { // close the writer @@ -143,6 +145,19 @@ public class pdfParser extends AbstractParser implements Parser { } finally { try {pdfDoc.close();} catch (IOException e) {} } + + // clear resources in pdfbox. they say that is resolved but it's not. see: + // https://issues.apache.org/jira/browse/PDFBOX-313 + // https://issues.apache.org/jira/browse/PDFBOX-351 + // https://issues.apache.org/jira/browse/PDFBOX-441 + // the pdfbox still generates enormeous number of object allocations and don't delete these + // the following Object are statically stored and never flushed: + // COSFloat, COSArray, COSInteger, COSObjectKey, COSObject, COSDictionary, + // COSStream, COSString, COSName, COSDocument, COSInteger[], COSNull + // the great number of these objects can easily be seen in Java Visual VM + // we try to get this shit out of the memory here by forced clear calls, hope the best the rubbish gets out. + COSName.clearResources(); + PDFont.clearResources(); return new Document[]{new Document( location, mimeType, diff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.java index 91a41278d..8817da6a1 100644 --- a/source/net/yacy/kelondro/blob/Compressor.java +++ b/source/net/yacy/kelondro/blob/Compressor.java @@ -238,7 +238,7 @@ public class Compressor implements BLOB { if (b != null) { // compress the entry now and put it to the backend try { - this.writeQueue.put(new Entity(new String(key), b)); + this.writeQueue.put(new Entity(keys, b)); this.bufferlength = this.bufferlength - b.length; return b; } catch (InterruptedException e) { diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index a724044bd..4b1dc36d7 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -43,6 +43,7 @@ import java.util.regex.Pattern; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.ByteArray; import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; @@ -146,7 +147,7 @@ public class Tables { if (pk == null) { pki = size(tablename); } else { - pki = Integer.parseInt(new String(pk)) + 1; + pki = (int) (ByteArray.parseDecimal(pk) + 1); } while (true) { pk = int2key(pki).getBytes(); @@ -426,7 +427,7 @@ public class Tables { byte[] r = this.get(colname); if (r == null) return dflt; try { - return Integer.parseInt(new String(r)); + return (int) ByteArray.parseDecimal(r); } catch (NumberFormatException e) { return dflt; } @@ -436,7 +437,7 @@ public class Tables { byte[] r = this.get(colname); if (r == null) return dflt; try { - return Long.parseLong(new String(r)); + return ByteArray.parseDecimal(r); } catch (NumberFormatException e) { return dflt; } diff --git a/source/net/yacy/kelondro/logging/ThreadDumpGenerator.java b/source/net/yacy/kelondro/logging/ThreadDumpGenerator.java new file mode 100644 index 000000000..1be7327bf --- /dev/null +++ b/source/net/yacy/kelondro/logging/ThreadDumpGenerator.java @@ -0,0 +1,192 @@ +// ThreadDumpGenerator +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004-2010 +// +// This File contains contributions from Alexander Fieger +// +// $LastChangedDate: 2010-09-02 21:24:22 +0200 (Do, 02 Sep 2010) $ +// $LastChangedRevision: 7092 $ +// $LastChangedBy: orbiter $ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.kelondro.logging; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.Map.Entry; + +import net.yacy.document.parser.html.CharacterCoding; +import net.yacy.kelondro.util.FileUtils; +import de.anomic.tools.nxTools; + +public class ThreadDumpGenerator { + + public static void appendStackTraces(final File rootPath, final StringBuilder buffer, final Map stackTraces, final boolean plain, final Thread.State stateIn) { + bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString()); + bufferappend(buffer, plain, ""); + // collect single dumps + HashMap> dumps = dumpCollection(rootPath, stackTraces, plain, stateIn); + + // write dumps + for (final Entry> entry: dumps.entrySet()) { + SortedSet threads = entry.getValue(); + for (String t: threads) bufferappend(buffer, plain, t); + bufferappend(buffer, plain, entry.getKey()); + bufferappend(buffer, plain, ""); + } + bufferappend(buffer, plain, ""); + } + + public static void appendStackTraceStats(final File rootPath, final StringBuilder buffer, final ArrayList> traces, final boolean plain, final Thread.State stateIn) { + if (stateIn != null) { + bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString()); + bufferappend(buffer, plain, ""); + } + // collect single dumps + HashMap dumps = dumpStatistic(rootPath, traces, plain, stateIn); + + // write dumps + while (!dumps.isEmpty()) { + Entry e = removeMax(dumps); + bufferappend(buffer, plain, "Occurrences: " + e.getValue()); + bufferappend(buffer, plain, e.getKey()); + //bufferappend(buffer, plain, ""); + } + bufferappend(buffer, plain, ""); + } + + private static Entry removeMax(HashMap result) { + Entry max = null; + for (final Entry e: result.entrySet()) { + if (max == null || e.getValue().intValue() > max.getValue().intValue()) { + max = e; + } + } + result.remove(max.getKey()); + return max; + } + + private static HashMap dumpStatistic(final File rootPath, final ArrayList> stackTraces, final boolean plain, final Thread.State stateIn) { + Map trace; + HashMap result = new HashMap(); + HashMap> x; + int count; + for (int i = 0; i < stackTraces.size(); i++) { + trace = stackTraces.get(i); + x = dumpCollection(rootPath, trace, plain, stateIn); + for (final Entry> e: x.entrySet()) { + Integer c = result.get(e.getKey()); + count = e.getValue().size(); + if (c == null) result.put(e.getKey(), Integer.valueOf(count)); + else { + c = Integer.valueOf(c.intValue() + count); + result.put(e.getKey(), c); + } + } + } + return result; + } + + private static HashMap> dumpCollection(final File appPath, final Map stackTraces, final boolean plain, final Thread.State stateIn) { + final File classPath = new File(appPath, "source"); + + Thread thread; + // collect single dumps + HashMap> dumps = new HashMap>(); + for (final Entry entry: stackTraces.entrySet()) { + thread = entry.getKey(); + final StackTraceElement[] stackTraceElements = entry.getValue(); + StackTraceElement ste; + String line; + String tracename = ""; + File classFile; + if ((stateIn == null || stateIn.equals(thread.getState())) && stackTraceElements.length > 0) { + StringBuilder sb = new StringBuilder(3000); + if (plain) { + classFile = getClassFile(classPath, stackTraceElements[stackTraceElements.length - 1].getClassName()); + tracename = classFile.getName(); + if (tracename.endsWith(".java")) tracename = tracename.substring(0, tracename.length() - 5); + if (tracename.length() > 20) tracename = tracename.substring(0, 20); + while (tracename.length() < 20) tracename = tracename + "_"; + tracename = "[" + tracename + "] "; + } + String threadtitle = tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString(); + String className; + boolean cutcore = true; + for (int i = 0; i < stackTraceElements.length; i++) { + ste = stackTraceElements[i]; + className = ste.getClassName(); + if (cutcore && (className.startsWith("java.") || className.startsWith("sun."))) { + sb.setLength(0); + bufferappend(sb, plain, tracename + "at " + CharacterCoding.unicode2html(ste.toString(), true)); + } else { + cutcore = false; + if (i == 0) { + line = getLine(getClassFile(classPath, className), ste.getLineNumber()); + } else { + line = null; + } + if ((line != null) && (line.length() > 0)) { + bufferappend(sb, plain, tracename + "at " + CharacterCoding.unicode2html(ste.toString(), true) + " [" + line.trim() + "]"); + } else { + bufferappend(sb, plain, tracename + "at " + CharacterCoding.unicode2html(ste.toString(), true)); + } + } + } + String threaddump = sb.toString(); + SortedSet threads = dumps.get(threaddump); + if (threads == null) threads = new TreeSet(); + threads.add(threadtitle); + dumps.put(threaddump, threads); + } + } + return dumps; + } + + private static File getClassFile(final File sourcePath, final String classname) { + final String classPath = classname.replace('.', '/') + ".java"; + final File file = new File(sourcePath, classPath); + return file; + } + + private static String getLine(final File file, final int line) { + // find class + if (!file.exists()) return ""; + try { + final String lineString = nxTools.line(FileUtils.read(file), line); + if (lineString == null) return "@ERROR"; + return lineString; + } catch (final IOException e) { + return "@EXCEPTION: " + e.getMessage(); + } + } + + public static void bufferappend(final StringBuilder buffer, final boolean plain, final String a) { + buffer.append(a); + if (plain) { + buffer.append("\n"); + } else { + buffer.append("
"); + } + } + +} diff --git a/source/net/yacy/kelondro/util/ByteArray.java b/source/net/yacy/kelondro/util/ByteArray.java index a02557d1e..2c54c78eb 100644 --- a/source/net/yacy/kelondro/util/ByteArray.java +++ b/source/net/yacy/kelondro/util/ByteArray.java @@ -106,6 +106,46 @@ public class ByteArray { while (--l >= 0) if (this.buffer[l] != b.buffer[l]) return false; return true; } + + public static long parseDecimal(final byte[] s) throws NumberFormatException { + if (s == null) throw new NumberFormatException("null"); + + long result = 0; + boolean negative = false; + int i = 0, max = s.length; + long limit; + long multmin; + long digit; + + if (max <= 0) throw new NumberFormatException(new String(s)); + if (s[0] == '-') { + negative = true; + limit = Long.MIN_VALUE; + i++; + } else { + limit = -Long.MAX_VALUE; + } + multmin = limit / 10; + if (i < max) { + digit = s[i++] - 48; + if (digit < 0) throw new NumberFormatException(new String(s)); + result = -digit; + } + while (i < max) { + // Accumulating negatively avoids surprises near MAX_VALUE + digit = s[i++] - 48; + if (digit < 0) throw new NumberFormatException(new String(s)); + if (result < multmin) throw new NumberFormatException(new String(s)); + result *= 10; + if (result < limit + digit) throw new NumberFormatException(new String(s)); + result -= digit; + } + if (negative) { + if (i > 1) return result; else throw new NumberFormatException(new String(s)); + } else { + return -result; + } + } public static void main(String[] args) { ByteArray a0 = new ByteArray("abc".getBytes()); @@ -126,5 +166,6 @@ public class ByteArray { System.out.println("a0 " + ((map.containsKey(a0)) ? "in" : "not in") + " map"); System.out.println("a1 " + ((map.containsKey(a1)) ? "in" : "not in") + " map"); System.out.println("b " + ((map.containsKey(b)) ? "in" : "not in") + " map"); + System.out.println("parseIntDecimal " + parseDecimal("6543".getBytes())); } } diff --git a/startYACY.bat b/startYACY.bat index 794d5e8d1..b2e0b1ce8 100644 --- a/startYACY.bat +++ b/startYACY.bat @@ -21,7 +21,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA -set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 +set javacmd=%javacmd% -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 Rem Starting YaCy Echo Generated classpath:%CLASSPATH% Echo JRE Parameters:%javacmd% diff --git a/startYACY.sh b/startYACY.sh index ba42c5ae4..0cd5e1fd8 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -6,7 +6,7 @@ PIDFILE="yacy.pid" OS="`uname`" #get javastart args -JAVA_ARGS="-server -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true"; +JAVA_ARGS="-server -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary @@ -177,7 +177,7 @@ CLASSPATH="" for N in lib/*.jar; do CLASSPATH="$CLASSPATH$N:"; done CLASSPATH=".:htroot:$CLASSPATH" -cmdline="$JAVA $JAVA_ARGS -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 -classpath $CLASSPATH net.yacy.yacy"; +cmdline="$JAVA $JAVA_ARGS -classpath $CLASSPATH net.yacy.yacy"; if [ $GUI -eq 1 ] #gui then cmdline="$cmdline -gui $parameter"