boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

upgraded pdfbox to 3.0.0

Browse Source
pull/607/head
Michael Peter Christen 1 year ago
parent c10944bd4a
commit 5ba5fb5d23
3 changed files with 46 additions and 43 deletions
Show Stats Download Patch File Download Diff File

2
ivy.xml
Unescape View File

@ -55,7 +55,7 @@
<dependency org="org.apache.lucene" name="lucene-queryparser" rev="8.11.2" conf="compile->master"/> <dependency org="org.apache.lucene" name="lucene-queryparser" rev="8.11.2" conf="compile->master"/>
<dependency org="org.apache.lucene" name="lucene-spatial-extras" rev="8.11.2" conf="compile->master"/> <dependency org="org.apache.lucene" name="lucene-spatial-extras" rev="8.11.2" conf="compile->master"/>
<dependency org="org.apache.lucene" name="lucene-suggest" rev="8.11.2"/> <dependency org="org.apache.lucene" name="lucene-suggest" rev="8.11.2"/>
<dependency org="org.apache.pdfbox" name="pdfbox" rev="2.0.29" /> <dependency org="org.apache.pdfbox" name="pdfbox" rev="3.0.0" />
<dependency org="org.apache.poi" name="poi" rev="3.17" /> <dependency org="org.apache.poi" name="poi" rev="3.17" />
<dependency org="org.apache.poi" name="poi-scratchpad" rev="3.17" /> <dependency org="org.apache.poi" name="poi-scratchpad" rev="3.17" />
<dependency org="org.apache.solr" name="solr-core" rev="8.11.2" conf="compile->master"/> <dependency org="org.apache.solr" name="solr-core" rev="8.11.2" conf="compile->master"/>

13
source/net/yacy/cora/util/Html2Image.java
Unescape View File

@ -43,6 +43,7 @@ import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit; import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.ImageView; import javax.swing.text.html.ImageView;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType; import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer; import org.apache.pdfbox.rendering.PDFRenderer;
@ -211,7 +212,7 @@ public class Html2Image {
* call termination. Beyond this limit the process is killed. * call termination. Beyond this limit the process is killed.
* @return true when the destination file was successfully written * @return true when the destination file was successfully written
*/ */
public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) { public static boolean writeWkhtmltopdf(final String url, final String proxy, final String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {
boolean success = false; boolean success = false;
for (final boolean ignoreErrors: new boolean[]{false, true}) { for (final boolean ignoreErrors: new boolean[]{false, true}) {
success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds); success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
@ -352,7 +353,7 @@ public class Html2Image {
// convert pdf to jpg using internal pdfbox capability // convert pdf to jpg using internal pdfbox capability
if (convertCmd == null) { if (convertCmd == null) {
try (final PDDocument pdoc = PDDocument.load(pdf);) { try (final PDDocument pdoc = Loader.loadPDF(pdf);) {
final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB); final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
@ -432,7 +433,7 @@ public class Html2Image {
* @param size * @param size
* @throws IOException * @throws IOException
*/ */
public static void writeSwingImage(String url, Dimension size, File destination) throws IOException { public static void writeSwingImage(final String url, final Dimension size, final File destination) throws IOException {
// set up a pane for rendering // set up a pane for rendering
final JEditorPane htmlPane = new JEditorPane(); final JEditorPane htmlPane = new JEditorPane();
@ -453,7 +454,7 @@ public class Html2Image {
public ViewFactory getViewFactory() { public ViewFactory getViewFactory() {
return new HTMLFactory() { return new HTMLFactory() {
@Override @Override
public View create(Element elem) { public View create(final Element elem) {
final View view = super.create(elem); final View view = super.create(elem);
if (view instanceof ImageView) { if (view instanceof ImageView) {
((ImageView) view).setLoadsSynchronously(true); ((ImageView) view).setLoadsSynchronously(true);
@ -467,7 +468,7 @@ public class Html2Image {
htmlPane.setContentType("text/html"); htmlPane.setContentType("text/html");
htmlPane.addPropertyChangeListener(new PropertyChangeListener() { htmlPane.addPropertyChangeListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(final PropertyChangeEvent evt) {
} }
}); });
@ -501,7 +502,7 @@ public class Html2Image {
* </li> * </li>
* </ol> * </ol>
*/ */
public static void main(String[] args) { public static void main(final String[] args) {
final String usageMessage = "Usage : java " + Html2Image.class.getName() final String usageMessage = "Usage : java " + Html2Image.class.getName()
+ " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]"; + " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";
int exitStatus = 0; int exitStatus = 0;

44
source/net/yacy/document/parser/pdfParser.java
Unescape View File

@ -39,8 +39,10 @@ import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.io.MemoryUsageSetting; import org.apache.pdfbox.io.RandomAccessRead;
import org.apache.pdfbox.io.RandomAccessReadBuffer;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentInformation; import org.apache.pdfbox.pdmodel.PDDocumentInformation;
import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPage;
@ -98,8 +100,8 @@ public class pdfParser extends AbstractParser implements Parser {
PDDocument pdfDoc; PDDocument pdfDoc;
try { try {
Thread.currentThread().setPriority(Thread.MIN_PRIORITY); // the pdfparser is a big pain Thread.currentThread().setPriority(Thread.MIN_PRIORITY); // the pdfparser is a big pain
MemoryUsageSetting mus = MemoryUsageSetting.setupMixed(200*1024*1024); final RandomAccessRead readBuffer = new RandomAccessReadBuffer(source);
pdfDoc = PDDocument.load(source, mus); pdfDoc = Loader.loadPDF(readBuffer);
} catch (final IOException e) { } catch (final IOException e) {
throw new Parser.Failure(e.getMessage(), location); throw new Parser.Failure(e.getMessage(), location);
} finally { } finally {
@ -155,8 +157,8 @@ public class pdfParser extends AbstractParser implements Parser {
// the new documents will get a virtual link with a post argument page=X appended to the original url // the new documents will get a virtual link with a post argument page=X appended to the original url
// collect text // collect text
int pagecount = pdfDoc.getNumberOfPages(); final int pagecount = pdfDoc.getNumberOfPages();
String[] pages = new String[pagecount]; final String[] pages = new String[pagecount];
for (int page = 1; page <= pagecount; page++) { for (int page = 1; page <= pagecount; page++) {
stripper.setStartPage(page); stripper.setStartPage(page);
stripper.setEndPage(page); stripper.setEndPage(page);
@ -167,7 +169,7 @@ public class pdfParser extends AbstractParser implements Parser {
// create individual documents for each page // create individual documents for each page
assert pages.length == pdflinks.size() : "pages.length = " + pages.length + ", pdflinks.length = " + pdflinks.size(); assert pages.length == pdflinks.size() : "pages.length = " + pages.length + ", pdflinks.length = " + pdflinks.size();
result = new Document[Math.min(pages.length, pdflinks.size())]; result = new Document[Math.min(pages.length, pdflinks.size())];
String loc = location.toNormalform(true); final String loc = location.toNormalform(true);
for (int page = 0; page < result.length; page++) { for (int page = 0; page < result.length; page++) {
result[page] = new Document( result[page] = new Document(
new AnchorURL(loc + (loc.indexOf('?') > 0 ? '&' : '?') + individualPagePropertyname + '=' + (page + 1)), // these are virtual new pages; we cannot combine them with '#' as that would be removed when computing the urlhash new AnchorURL(loc + (loc.indexOf('?') > 0 ? '&' : '?') + individualPagePropertyname + '=' + (page + 1)), // these are virtual new pages; we cannot combine them with '#' as that would be removed when computing the urlhash
@ -217,8 +219,8 @@ public class pdfParser extends AbstractParser implements Parser {
writer.close(); // free writer resources writer.close(); // free writer resources
} }
Collection<AnchorURL> pdflinksCombined = new HashSet<AnchorURL>(); final Collection<AnchorURL> pdflinksCombined = new HashSet<>();
for (Collection<AnchorURL> pdflinksx: pdflinks) if (pdflinksx != null) pdflinksCombined.addAll(pdflinksx); for (final Collection<AnchorURL> pdflinksx: pdflinks) if (pdflinksx != null) pdflinksCombined.addAll(pdflinksx);
result = new Document[]{new Document( result = new Document[]{new Document(
location, location,
mimeType, mimeType,
@ -258,25 +260,25 @@ public class pdfParser extends AbstractParser implements Parser {
* @return all detected links * @return all detected links
*/ */
private List<Collection<AnchorURL>> extractPdfLinks(final PDDocument pdf) { private List<Collection<AnchorURL>> extractPdfLinks(final PDDocument pdf) {
List<Collection<AnchorURL>> linkCollections = new ArrayList<>(pdf.getNumberOfPages()); final List<Collection<AnchorURL>> linkCollections = new ArrayList<>(pdf.getNumberOfPages());
for (PDPage page : pdf.getPages()) { for (final PDPage page : pdf.getPages()) {
final Collection<AnchorURL> pdflinks = new ArrayList<AnchorURL>(); final Collection<AnchorURL> pdflinks = new ArrayList<>();
try { try {
List<PDAnnotation> annotations = page.getAnnotations(); final List<PDAnnotation> annotations = page.getAnnotations();
if (annotations != null) { if (annotations != null) {
for (PDAnnotation pdfannotation : annotations) { for (final PDAnnotation pdfannotation : annotations) {
if (pdfannotation instanceof PDAnnotationLink) { if (pdfannotation instanceof PDAnnotationLink) {
PDAction link = ((PDAnnotationLink)pdfannotation).getAction(); final PDAction link = ((PDAnnotationLink)pdfannotation).getAction();
if (link != null && link instanceof PDActionURI) { if (link != null && link instanceof PDActionURI) {
PDActionURI pdflinkuri = (PDActionURI) link; final PDActionURI pdflinkuri = (PDActionURI) link;
String uristr = pdflinkuri.getURI(); final String uristr = pdflinkuri.getURI();
AnchorURL url = new AnchorURL(uristr); final AnchorURL url = new AnchorURL(uristr);
pdflinks.add(url); pdflinks.add(url);
} }
} }
} }
} }
} catch (IOException ex) {} } catch (final IOException ex) {}
linkCollections.add(pdflinks); linkCollections.add(pdflinks);
} }
return linkCollections; return linkCollections;
@ -345,7 +347,7 @@ public class pdfParser extends AbstractParser implements Parser {
if(inStream != null) { if(inStream != null) {
try { try {
inStream.close(); inStream.close();
} catch(IOException e) { } catch(final IOException e) {
System.err.println("Could not close input stream on file " + pdfFile); System.err.println("Could not close input stream on file " + pdfFile);
} }
} }
@ -359,7 +361,7 @@ public class pdfParser extends AbstractParser implements Parser {
System.out.println("\t!!!Parsing without result!!!"); System.out.println("\t!!!Parsing without result!!!");
} else { } else {
System.out.println("\tParsed text with " + document.getTextLength() + " chars of text and " + document.getAnchors().size() + " anchors"); System.out.println("\tParsed text with " + document.getTextLength() + " chars of text and " + document.getAnchors().size() + " anchors");
InputStream textStream = document.getTextStream(); final InputStream textStream = document.getTextStream();
try { try {
// write file // write file
FileUtils.copy(textStream, new File("parsedPdf.txt")); FileUtils.copy(textStream, new File("parsedPdf.txt"));
@ -372,7 +374,7 @@ public class pdfParser extends AbstractParser implements Parser {
/* textStream can be a FileInputStream : we must close it to ensure releasing system resource */ /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */
textStream.close(); textStream.close();
} }
} catch (IOException e) { } catch (final IOException e) {
ConcurrentLog.warn("PDFPARSER", "Could not close text input stream"); ConcurrentLog.warn("PDFPARSER", "Could not close text input stream");
} }
} }

Write Preview
Loading…
Cancel
Save