robots.ensureExist(entry.url(),profile.getAgent(),true);// concurrently load all robots.txt
// profile might be null when continue crawls after YaCy restart
robots.ensureExist(entry.url(),profile==null?ClientIdentification.yacyInternetCrawlerAgent:profile.getAgent(),true);// concurrently load all robots.txt