From d2cc11ea8f679a47343cc7d723c02e2b73d07360 Mon Sep 17 00:00:00 2001
From: reger <reger18@arcor.de>
Date: Sat, 19 Sep 2015 05:30:55 +0200
Subject: [PATCH] fix html parser taking <style> content as text. Noticed some
 result description contain css content from style tag. Added <style> to tag
 list to scrape it's content not as text + test case included

---
 .../document/parser/html/ContentScraper.java  |  5 ++++-
 .../yacy/document/parser/htmlParserTest.java  | 22 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java
index 17f9362c7..77b59ad79 100644
--- a/source/net/yacy/document/parser/html/ContentScraper.java
+++ b/source/net/yacy/document/parser/html/ContentScraper.java
@@ -115,7 +115,10 @@ public class ContentScraper extends AbstractScraper implements Scraper {
         script(TagType.pair),
         span(TagType.pair),
         div(TagType.pair),
-        article(TagType.pair);
+        article(TagType.pair),
+        // tags used to capture tag content
+        // TODO: considere to use </head> or <body> as trigger to scape for text content
+        style(TagType.pair); // embedded css (if not declared as tag content is parsed as text)
 
         public TagType type;
         private TagName(final TagType type) {
diff --git a/test/net/yacy/document/parser/htmlParserTest.java b/test/net/yacy/document/parser/htmlParserTest.java
index 1ab31f9bf..2d7ad5b73 100644
--- a/test/net/yacy/document/parser/htmlParserTest.java
+++ b/test/net/yacy/document/parser/htmlParserTest.java
@@ -118,4 +118,26 @@ public class htmlParserTest extends TestCase {
         ImageEntry img = scraper.getImages().get(1);
         assertEquals(550,img.width());
     }
+
+    /**
+     * Test of parseToScraper method, of class htmlParser
+     * for scraping tag content from text (special test to verify <style> not counted as text
+     */
+    @Test
+    public void testParseToScraper_TagTest() throws Exception {
+        final AnchorURL url = new AnchorURL("http://localhost/");
+        final String mimetype = "text/html";
+        final String textSource = "test text";
+        final String testhtml = "<html>"
+                + "<head><style type=\"text/css\"> h1 { color: #ffffff; }</style></head>"
+                + "<body>"
+                + "<p>" + textSource + "</p>"
+                + "</body></html>";
+
+        ContentScraper scraper = parseToScraper(url, mimetype, new VocabularyScraper(), 0, testhtml, 10);
+
+        String txt = scraper.getText();
+        System.out.println("ScraperTagTest: [" + textSource + "] = [" + txt + "]");
+        assertEquals(txt, textSource);
+    }
 }
