|
|
|
@ -366,7 +366,7 @@ public class Crawler_p {
|
|
|
|
|
boolean obeyHtmlRobotsNofollow = "on".equals(post.get("obeyHtmlRobotsNofollow", "false"));
|
|
|
|
|
env.setConfig("obeyHtmlRobotsNofollow", obeyHtmlRobotsNofollow);
|
|
|
|
|
|
|
|
|
|
final boolean indexText = "on".equals(post.get("indexText", "false"));
|
|
|
|
|
final boolean indexText = "on".equals(post.get("indexText", "on"));
|
|
|
|
|
env.setConfig("indexText", indexText);
|
|
|
|
|
|
|
|
|
|
final boolean indexMedia = "on".equals(post.get("indexMedia", "false"));
|
|
|
|
|