diff --git a/htroot/ConfigPortal.html b/htroot/ConfigPortal.html index 6e5dda7d1..760d9ad17 100644 --- a/htroot/ConfigPortal.html +++ b/htroot/ConfigPortal.html @@ -53,7 +53,7 @@ ideaSpeed up search results with this option! (use CACHEONLY or FALSE to switch off verification)
NOCACHE: no use of web cache, load all snippets online
IFFRESH: use the cache if the cache exists and is fresh otherwise load online
- IFEXIST: use the cache if the cache exist or load online
+ IFEXIST: use the cache if the cache exist or load online
If verification fails, delete index reference

CACHEONLY: never go online, use all content from cache. If no cache entry exist, consider content nevertheless as available and show result without snippet
FALSE: no link verification and not snippet generation: all search results are valid without verification diff --git a/htroot/Steering.html b/htroot/Steering.html index c66fcbc65..dec27e8a6 100644 --- a/htroot/Steering.html +++ b/htroot/Steering.html @@ -174,7 +174,7 @@ XDtoU7vQ/wIAAP//AwBb7ktEXQ4nqQAAAABJRU5ErkJggg==" width="128" height="64" alt="K Therefore we like to ask you: do you like YaCy? Will you use it again... if not, why? Is is possible that we change a bit to suit your needs?

Please send us feed-back about your experience with an
anonymous message
- or a
+ or a
posting to our web forums
or a
bug report!

diff --git a/htroot/yacysearchitem.html b/htroot/yacysearchitem.html index afa3f3abf..21cc9b4a2 100644 --- a/htroot/yacysearchitem.html +++ b/htroot/yacysearchitem.html @@ -10,7 +10,7 @@ #(/heuristic)# #(authorized)#:: #(bookmark)#:: - bookmark + bookmark #(/bookmark)# #(recommend)# recommend diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index c8a7521c8..7e2aaa8c5 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -317,7 +317,7 @@ public class yacysearchitem { final String license = URLLicense.aquireLicense(image.imageUrl); // this is just the license key to get the image forwarded through the YaCy thumbnail viewer, not an actual lawful license //sb.loader.loadIfNotExistBackground(image.imageUrl, 1024 * 1024 * 10, null, ClientIdentification.yacyIntranetCrawlerAgent); - prop.putHTML("content_item_hrefCache", "/ViewImage." + ("gif".equals(imageUrlExt) ? "gif" : "png") + "?maxwidth=128&maxheight=128&isStatic=true&quadratic=&url=" + imageUrlstring); + prop.putHTML("content_item_hrefCache", "/ViewImage." + ("gif".equals(imageUrlExt) ? "gif" : "png") + "?maxwidth=128&maxheight=128&code="+license+"&isStatic=true&quadratic=&url=" + imageUrlstring); prop.putHTML("content_item_href", imageUrlstring); prop.putHTML("content_item_target", target); prop.put("content_item_code", license); diff --git a/locales/fr.lng b/locales/fr.lng index 1922d1f14..af36087e2 100644 --- a/locales/fr.lng +++ b/locales/fr.lng @@ -463,7 +463,7 @@ RWIs at Start:==RWIs au départ: RWIs now:==RWIs maintenant: wordHash in Progress:==Mots-Hash en cours: last wordHash with deleted URLs:==dernier mot-hash avec URLs supprimées: -Number of deleted URLs in on this Hash:==Nombre d'URLs supprimés sur ce hâchage: +Number of deleted URLs in on this Hash:==Nombre d'URLs supprimés sur ce hachage: UrldbCleaner - Clean up the database by deletion of blacklisted urls:==UrlDBCleaner - Nettoie la base de données en supprimant les URLs en liste noire: Start/Resume==Départ/Reprise Stop==Arrêt @@ -509,7 +509,7 @@ Word-Hash:==Mot-Hash: #File: CrawlStartExpert.html #--------------------------- Index Creation==Créer un index -Start Crawling Job:==Tâche de démarrage du crawl: +Start Crawling Job:==Tâche de démarrage du crawl: You can define URLs as start points for Web page crawling and start crawling here. \"Crawling\" means that YaCy will download the given website, extract all links in it and then download the content behind these links. This is repeated as long as specified under \"Crawling Depth\".==Vous pouvez définir les URLs de départ et démarrer le crawl ici. Crawler signifie que YaCy téléchargera les sites de départ et extraira tous leurs liens ainsi que leur contenu. Cela sera répété jusqu'a ce que la \"profondeur de crawl\" soit atteinte. #Attribut==Attribut Value==Valeur @@ -764,8 +764,8 @@ Import Path:==Import-Pfad: "Start Import"=="Demarrer l'importation" Attention:==Attention: Always do a backup of your source and destination database before starting to use this import function.==Toujours faire une sauvegarde de vos bases source et destination avant de lancer cette fonction d'importation. -Currently running jobs==Tâches actuellement en cours -Job Type==Type de tâche +Currently running jobs==Tâches actuellement en cours +Job Type==Type de tâche >Path==>Chemin Status==Etat Elapsed
Time==Temps
écoulé @@ -852,7 +852,7 @@ and automatically excluded from indexing.==et exclus automatiquement de l'indexa Set the proxy settings of your browser to the same port as given==Configurez les paramètres de proxy de votre brower avec le même port que celui on the 'Settings'-page in the 'Proxy and Administration Port' field.==du champ 'Port de proxy et d'administration' de la page de 'Configuration'. \(5\) Index Monitor for Local Crawling.==(5) Moniteur d'index pour le crawl local -These web pages had been crawled by your own crawl task.==Cette page web a été crawlée par votre propre tâche de crawl. +These web pages had been crawled by your own crawl task.==Cette page web a été crawlée par votre propre tâche de crawl. Use Case: start a crawl by setting a crawl start point on the 'Index Create' page.==Cas d'utilisation: Lancer un crawl en paramètrant un point de départ sur la page 'Création d'index'. \(6\) Index Monitor for Global Crawling==(6) Moniteur d'index pour le crawl global These pages had been indexed by your peer, but the crawl was initiated by a remote peer.==Ces pages ont été indexées par votre noeud, mais le crawl était initié par un noeud distant. @@ -1095,7 +1095,7 @@ Attributes==Attributs #File: PerformanceQueues_p.html #--------------------------- Performance Settings of Queues and Processes==Paramètres de performance des files et des processus -Scheduled tasks overview and waiting time settings:==Aperçu des tâches programmées et paramètres d'attente: +Scheduled tasks overview and waiting time settings:==Aperçu des tâches programmées et paramètres d'attente: Queue Size==Taille de file >Total==>Totale Block Time==Temps de blocage @@ -1133,8 +1133,8 @@ Maximum number of Word Caches, high limit:==Nombre maximal de mots dans le cache This is is the number of word indexes that shall be held in the==C'est le nombre d'index de mots qui doivent être gardés dans le ram cache during indexing. When YaCy is shut down, this cache must be==cache mémoire pendant l'indexation. Quand YaCy est stoppé, ce cache doit flushed to disc; this may last some minutes.==être transféré sur disque; cela peut durer quelques minutes. -The low limit is valid for crawling tasks, the high limit is valid==La limite basse est valide pour les tâches de crawl, la limite haute est valide -for search and DHT transmission tasks.==pour les tâches de recherche et de transmission de DHT. +The low limit is valid for crawling tasks, the high limit is valid==La limite basse est valide pour les tâches de crawl, la limite haute est valide +for search and DHT transmission tasks.==pour les tâches de recherche et de transmission de DHT. Enter New Cache Size==Entrer une nouvelle taille de cache Thread pool settings:==Paramètres du pool de threads: maximum Active==max. actif @@ -1201,7 +1201,7 @@ The Response Header database stores the HTTP heades that other servers send when during proxy mode, when performing crawls or if it fetches pages for snippet generation.==durant le mode proxy en exécutant des crawl ou en parcourant les pages pour la génération d'extraits. Increasing this cache will be most important for a fast proxy mode.==Augmenter ce cache est important pour avoir un mode proxy rapide. 'loaded' URLs==URLs chargées -This is the database that holds the hash/url - relation and properties regarding the url like load date and server date.==C'est la base de données qui maintient la relation hâchage/url et les propriétés concernant l'URL comme la date de chargement et la date du serveur. +This is the database that holds the hash/url - relation and properties regarding the url like load date and server date.==C'est la base de données qui maintient la relation hachage/url et les propriétés concernant l'URL comme la date de chargement et la date du serveur. This cache is very important for a fast search process.==Ce cache est très important pour un processus de recherche rapide. Increasing the cache size will result in more search results and less IO during DHT transfer.==Augmenter la taille du cache augmente le nombre de résultats de recherche et diminue le nombre d'E/S pendant le transfert de DHT. 'noticed' URLs==URLs 'remarquées' @@ -1672,7 +1672,7 @@ For community support, please visit our Settings page if you want to make more changes.==Vous pouvez maintenant retourner sur la page des paramètres si vous voulez effectuer d'autres modifications. See you soon!==À bientôt! -Application will terminate after working off all scheduled tasks.=L'application se terminera après avoir achevé toutes les taches programmées. +Application will terminate after working off all scheduled tasks\.==L'application se terminera après avoir achevé toutes les tâches planifiées. Please send us feed-back!==Envoyez-nous vos commentaires! We don't track YaCy users, YaCy does not send 'home-pings', we do not even know how many people use YaCy as their private search engine.==Nous ne traçons pas les utilisateurs de YaCy, YaCy n'envoie pas de "home-pings", nous ne savons même pas combien de gens utilisent YaCy comme moteur de recherche. -Therefore we like to ask you: do you like YaCy? Will you use it again... if not, why? Is is possible that we change a bit to suit your needs?==C'est pourquoi nous apprécierions beaucoup que vous nous disiez ce que vous pensez de YaCy. Appréciez-vous YaCy? Allez-vous l'utiliser à nouveau... si non, pourquoi? Nous pouvons peut-être modifier YaCy pour qu'il réponde à vos besoins. +Therefore we like to ask you: do you like YaCy\? Will you use it again\.\.\. if not, why\? Is is possible that we change a bit to suit your needs\?==C'est pourquoi nous aimerions beaucoup que vous nous disiez ce que vous pensez de YaCy. Appréciez-vous YaCy? Allez-vous l'utiliser à nouveau... si non, pourquoi? Nous pouvons peut-être modifier YaCy pour qu'il réponde à vos besoins. Please send us feed-back about your experience with an==Vous pouvez nous faire parvenir vos commentaires avec un -anonymous message==message anonynme -or a
==ou un
+anonymous message==message anonyme +or a
==ou un
posting to our web forums
==message sur nos forums
-bug report!==rapport de bogue! +bug report==rapport de bogue

Professional Support

==

Support professionnel

If you are a professional user and you would like to use YaCy in your company in combination with consulting services by YaCy specialists, please see==Si vous souhaitez utiliser YaCy dans votre entreprise en association avec un service de consultations d'expertise par des spécialistes de YaCy, veuillez consulter Just a moment, please!==Juste un instant, s'il vous plaît! @@ -2078,7 +2078,7 @@ YaCy Interactive Search==YaCy Recherche interactive >Name==>Nom >Size==>Taille >Date==>Date -kiosk mode==Kiosque Mode +kiosk mode==Mode Kiosque >Link==>Lien #----------------------------- @@ -2110,6 +2110,38 @@ Map \(c\) by==Carte (c) fournie par and contributors, CC-BY-SA==et ses contributeurs, CC-BY-SA #----------------------------- +#File: yacysearchitem.html +#--------------------------- +"recommend"=="recommander" +"bookmark"=="ajouter aux favoris" +"delete"=="supprimer" +#----------------------------- + +#File: yacysearchtrailer.html +#--------------------------- +Your search is done using peers in the YaCy P2P network\.==Recherche effectuée grâce aux noeuds du réseau P2P YaCy. +You can switch to 'Stealth Mode' which will switch off P2P\, giving you full privacy\. Expect less results then, because then only your own search index is used\.==Vous pouvez passer en 'Mode furtif' ce qui désactivera le réseau pair à pair et vous offrira une privatisation complète. Cependant vous obtiendrez probablement moins de résultats car la recherche sera alors basée uniquement sur votre propre index. +"Privacy"=="Privé" + Peer-to-Peer == Pair à pair  +Stealth Mode==Mode furtif +Your search is done using only your own peer, locally\.==La recherche est réalisée uniquement sur les données locales de votre noeud. +You can switch to 'Peer-to-Peer Mode' which will cause that your search is done using the other peers in the YaCy network\.==Vous pouvez passer en mode 'Pair à pair' dans lequel votre recherche sera basée sur les autres noeuds du réseau YaCy. +"Peer-to-Peer"==Pair à pair +Context Ranking==Classement contextuel +Sort by Date==Trier par date +Video==Vidéo +>Apps==>Applications +

Location

==

Lieu

+show search results for "#\[query\]#" on map==Montrer sur la carte les résultats pour "#[query]#" +click to expand facet==cliquer pour révéler cet aspect +>Provider==>Site +>Filetype==>Type de fichier +>Language==>Langue +>Name Space==>Espace de noms +>Author==>Auteur +>Collection==>Collection +#----------------------------- + ### Subdirectory env/templates ### #File: env/templates/header.template #--------------------------- diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.java index 454ba15db..e7bcd5e3b 100644 --- a/source/net/yacy/crawler/data/CrawlQueues.java +++ b/source/net/yacy/crawler/data/CrawlQueues.java @@ -674,7 +674,6 @@ public class CrawlQueues { if (CrawlQueues.log.isFine()) { CrawlQueues.log.fine("problem loading " + request.url().toString() + ": " + e.getMessage()); } - e.printStackTrace(); error = "load error - " + e.getMessage(); } @@ -693,7 +692,6 @@ public class CrawlQueues { } } catch (final Exception e) { CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, e.getMessage() + " - in worker", -1); - ConcurrentLog.logException(e); request.setStatus("worker-exception", WorkflowJob.STATUS_FINISHED); } finally { request = null; diff --git a/source/net/yacy/data/Translator.java b/source/net/yacy/data/Translator.java index 884029bdf..11d34351d 100644 --- a/source/net/yacy/data/Translator.java +++ b/source/net/yacy/data/Translator.java @@ -52,6 +52,7 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.Formatter; import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverSwitch; +import net.yacy.utils.translation.SourceFileFilter; import java.util.*; @@ -126,6 +127,13 @@ public class Translator { return translateFile(sourceFile, destFile, loadTranslationsLists(translationFile).get(sourceFile.getName())); } + /** + * Translate sourceFile to destFile using translationList. + * @param sourceFile file to translate + * @param destFile file to write + * @param translationList map of translations + * @return true when destFile was sucessfully written, false otherwise + */ public static boolean translateFile(final File sourceFile, final File destFile, final Map translationList){ StringBuilder content = new StringBuilder(); @@ -147,11 +155,11 @@ public class Translator { } } - content = new StringBuilder(translate(content.toString(), translationList)); + String processedContent = translate(content.toString(), translationList); BufferedWriter bw = null; try{ bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile),"UTF-8")); - bw.write(content.toString()); + bw.write(processedContent); bw.close(); }catch(final IOException e){ return false; @@ -172,38 +180,28 @@ public class Translator { public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final Map> translationLists, final String extensions){ destDir.mkdirs(); - final File[] sourceFiles = sourceDir.listFiles(); final List exts = ListManager.string2vector(extensions); - boolean rightExtension; + final File[] sourceFiles = sourceDir.listFiles(new SourceFileFilter(exts)); String relativePath; for (final File sourceFile : sourceFiles) { - rightExtension=false; - for (final String ext : exts) { - if (sourceFile.getName().endsWith(ext)) { - rightExtension=true; - break; - } - } - if (rightExtension) { - try { - relativePath=sourceFile.getAbsolutePath().substring(baseDir.getAbsolutePath().length()+1); //+1 to get the "/" - relativePath = relativePath.replace(File.separatorChar, '/'); - } catch (final IndexOutOfBoundsException e) { - ConcurrentLog.severe("TRANSLATOR", "Error creating relative Path for "+sourceFile.getAbsolutePath()); - relativePath = "wrong path"; //not in translationLists - } - if (translationLists.containsKey(relativePath)) { - ConcurrentLog.info("TRANSLATOR", "Translating file: "+ relativePath); - if(!translateFile( - sourceFile, - new File(destDir, sourceFile.getName().replace('/', File.separatorChar)), - translationLists.get(relativePath))) - { - ConcurrentLog.severe("TRANSLATOR", "File error while translating file "+relativePath); - } - //}else{ - //serverLog.logInfo("TRANSLATOR", "No translation for file: "+relativePath); + try { + relativePath=sourceFile.getAbsolutePath().substring(baseDir.getAbsolutePath().length()+1); //+1 to get the "/" + relativePath = relativePath.replace(File.separatorChar, '/'); + } catch (final IndexOutOfBoundsException e) { + ConcurrentLog.severe("TRANSLATOR", "Error creating relative Path for "+sourceFile.getAbsolutePath()); + relativePath = "wrong path"; //not in translationLists + } + if (translationLists.containsKey(relativePath)) { + ConcurrentLog.info("TRANSLATOR", "Translating file: "+ relativePath); + if(!translateFile( + sourceFile, + new File(destDir, sourceFile.getName().replace('/', File.separatorChar)), + translationLists.get(relativePath))) + { + ConcurrentLog.severe("TRANSLATOR", "File error while translating file "+relativePath); } + //}else{ + //serverLog.logInfo("TRANSLATOR", "No translation for file: "+relativePath); } } return true; diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java index 824d5dfa4..af07f20df 100644 --- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java +++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java @@ -460,6 +460,7 @@ public class YaCyDefaultServlet extends HttpServlet { } catch (IllegalArgumentException iae) { if (!response.isCommitted()) { response.sendError(HttpServletResponse.SC_BAD_REQUEST, iae.getMessage()); + return false; } throw iae; } @@ -1075,9 +1076,9 @@ public class YaCyDefaultServlet extends HttpServlet { filecontent = item.getInputStream(); files.put(new AbstractMap.SimpleEntry(item.getFieldName(), FileUtils.read(filecontent))); } catch (IOException e) { - ConcurrentLog.logException(e); + ConcurrentLog.info("FILEHANDLER", e.getMessage()); } finally { - if (filecontent != null) try {filecontent.close();} catch (IOException e) {ConcurrentLog.logException(e);} + if (filecontent != null) try {filecontent.close();} catch (IOException e) {ConcurrentLog.info("FILEHANDLER", e.getMessage());} } } } @@ -1108,7 +1109,7 @@ public class YaCyDefaultServlet extends HttpServlet { for (int j = 0; j < t; j++) p[j].join(); } } catch (Exception ex) { - ConcurrentLog.logException(ex); + ConcurrentLog.info("FILEHANDLER", ex.getMessage()); } } diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 34b05b815..4c0b44523 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -34,6 +34,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -44,6 +45,7 @@ import java.io.UnsupportedEncodingException; import java.io.Writer; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -55,13 +57,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import org.mozilla.intl.chardet.nsDetector; -import org.mozilla.intl.chardet.nsPSMDetector; - import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.storage.Files; import net.yacy.cora.util.ConcurrentLog; +import org.mozilla.intl.chardet.nsDetector; +import org.mozilla.intl.chardet.nsPSMDetector; + public final class FileUtils { private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size @@ -668,6 +670,23 @@ public final class FileUtils { public static ArrayList getDirsRecursive(final File dir, final String notdir) { return getDirsRecursive(dir, notdir, true); } + + /** + * @param sourceDir source directory. Must be not null. + * @param notdir name of dir to exlcude. Can be null + * @param fileNameFilter filter to apply on file names. Can be null. + * @return list of all files passing fileFilter under sourceDir including sub directories + */ + public static List getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) { + List dirList = getDirsRecursive(sourceDir, + notdir); + dirList.add(sourceDir); + List files = new ArrayList<>(); + for (final File dir : dirList) { + Collections.addAll(files, dir.listFiles(fileNameFilter)); + } + return files; + } /** * Returns a List of all dirs and subdirs as File Objects Warning: untested diff --git a/source/net/yacy/utils/translation/ListNonTranslatedFiles.java b/source/net/yacy/utils/translation/ListNonTranslatedFiles.java new file mode 100755 index 000000000..fe50749e9 --- /dev/null +++ b/source/net/yacy/utils/translation/ListNonTranslatedFiles.java @@ -0,0 +1,111 @@ +// ListNonTranslatedFiles.java +// ------------------------------------- +// part of YACY +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.utils.translation; + +import java.io.File; +import java.io.FilenameFilter; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.ListManager; +import net.yacy.data.Translator; +import net.yacy.kelondro.util.FileUtils; + +/** + * Util to help identifying non translated files. + * + * @author luc + * + */ +public class ListNonTranslatedFiles extends TranslatorUtil { + + /** + * Print on standard output result of search + * @param nonTranslatedFiles list of non translated files + */ + private static void printResults(List nonTranslatedFiles) { + System.out.println(nonTranslatedFiles.size() + " files are not translated."); + for(File file : nonTranslatedFiles) { + System.out.println(file); + } + } + + /** + * List all files from srcDir directory which are not translated using + * specified locale file with specified extensions. If no argument is set, + * default values are used. + * + * @param args + * runtime arguments
+ *
    + *
  • args[0] : source dir path
    • + *
  • args[1] : translation file path
    • + *
  • args[2] : extensions (separated by commas)
    • + *
+ */ + public static void main(String args[]) { + File sourceDir = getSourceDir(args); + Path sourcePath = sourceDir.toPath(); + + File translationFile = getTranslationFile(args); + + List extensions = ListManager + .string2vector(getExtensions(args)); + + FilenameFilter fileFilter = new SourceFileFilter(extensions); + + String excludedDir = "locale"; + + ConcurrentLog.info("ListNonTranslatedFiles", "Listing non translated " + + extensions + " files from " + sourceDir + " using " + + translationFile); + + try { + Set translatedRelativePaths = Translator.loadTranslationsLists(translationFile).keySet(); + + List srcFiles = FileUtils.getFilesRecursive(sourceDir, excludedDir, fileFilter); + + List nonTranslatedFiles = new ArrayList<>(); + for(File srcFile : srcFiles) { + Path relativeSrcFile = sourcePath.relativize(srcFile.toPath()); + if(!translatedRelativePaths.contains(relativeSrcFile.toString())) { + nonTranslatedFiles.add(srcFile); + } + } + + printResults(nonTranslatedFiles); + + + } finally { + ConcurrentLog.shutdown(); + } + + } + +} diff --git a/source/net/yacy/utils/translation/SourceFileFilter.java b/source/net/yacy/utils/translation/SourceFileFilter.java new file mode 100755 index 000000000..4c82459aa --- /dev/null +++ b/source/net/yacy/utils/translation/SourceFileFilter.java @@ -0,0 +1,83 @@ +// SourceFileFilter.java +// ------------------------------------- +// part of YACY +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.utils.translation; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.ArrayList; +import java.util.List; + +/** + * File filter for source files to translate + * + * @author luc + * + */ +public class SourceFileFilter implements FilenameFilter { + + /** Extensions required to pass filter */ + private List extensions; + + /** + * Contructor with extensions + * + * @param extensions + * extensions required. When this list is null or empty, filter + * let pass all files. + */ + public SourceFileFilter(List extensions) { + if (extensions == null) { + this.extensions = new ArrayList<>(); + } else { + this.extensions = new ArrayList<>(extensions); + } + } + + /** + * @param file + * file to check + * @return true when file name ends with one of the extensions list or + * extensions list is empty + */ + @Override + public boolean accept(File dir, String name) { + boolean accepted = false; + if (name != null) { + if (extensions.size() == 0) { + accepted = true; + } else { + for (String ext : extensions) { + if (name.endsWith(ext)) { + accepted = true; + break; + } + } + } + } + return accepted; + } + +} diff --git a/source/net/yacy/utils/translation/TranslateAll.java b/source/net/yacy/utils/translation/TranslateAll.java new file mode 100755 index 000000000..de1f699c2 --- /dev/null +++ b/source/net/yacy/utils/translation/TranslateAll.java @@ -0,0 +1,100 @@ +// TranslateAll.java +// ------------------------------------- +// part of YACY +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.utils.translation; + +import java.io.File; + +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.Translator; + +/** + * Util to translate all files without launching full YaCy application. + * + * @author luc + * + */ +public class TranslateAll extends TranslatorUtil { + + /** + * Translate all files from srcDir directory into dstDir directory using + * specified locale file with specified extensions. If no argument is set, + * default values are used. + * + * @param args + * runtime arguments
+ *
    + *
  • args[0] : source dir path
    • + *
  • args[1] : destination dir path
    • + *
  • args[2] : translation file path
    • + *
  • args[3] : extensions (separated by commas)
    • + *
+ */ + public static void main(String args[]) { + File sourceDir = getSourceDir(args); + + File destDir = getDestDir(args); + + File translationFile = getTranslationFile(args); + + String extensions = getExtensions(args); + + ConcurrentLog.info("TranslateAll", "Translating " + extensions + + " files from " + sourceDir + " to " + destDir + " using " + + translationFile); + + try { + Translator.translateFilesRecursive(sourceDir, destDir, + translationFile, extensions, "locale"); + } finally { + ConcurrentLog.shutdown(); + } + + } + + /** + * @param args + * main parameters + * @return translation source dir from parameters or default + * @throws IllegalArgumentException + * when no parameters is set and default is not found + */ + protected static File getDestDir(String[] args) { + File destDir; + if (args.length > 1) { + destDir = new File(args[1]); + } else { + String tmpDir = System.getProperty("java.io.tmpdir"); + if (tmpDir == null) { + throw new IllegalArgumentException( + "No destination dir specified, and default not found"); + } + destDir = new File(tmpDir + File.separator + + TranslateAll.class.getCanonicalName()); + } + return destDir; + } + +} diff --git a/source/net/yacy/utils/translation/TranslatorUtil.java b/source/net/yacy/utils/translation/TranslatorUtil.java new file mode 100755 index 000000000..a88cd3f28 --- /dev/null +++ b/source/net/yacy/utils/translation/TranslatorUtil.java @@ -0,0 +1,116 @@ +// TranslateAll.java +// ------------------------------------- +// part of YACY +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.utils.translation; + +import java.io.File; +import java.util.Locale; + +import net.yacy.yacy; +import net.yacy.data.Translator; + +/** + * Base class for launching a {@link Translator} method without full YaCy + * application started. + * + * @author luc + * + */ +public abstract class TranslatorUtil { + + /** + * @param args + * main parameters + * @return translation source dir from parameters or default + * (workingDir/htroot) + * @throws IllegalArgumentException + * when no parameters is set and default is not found + */ + protected static File getSourceDir(String[] args) { + File sourceDir; + if (args.length > 0) { + sourceDir = new File(args[0]); + } else { + String workingDir = System.getProperty("user.dir"); + if (workingDir == null) { + throw new IllegalArgumentException( + "No translation file specified, and default not found"); + } + sourceDir = new File(workingDir, "htroot"); + if (!sourceDir.exists() && !sourceDir.isDirectory()) { + throw new IllegalArgumentException( + "No translation file specified, and default not found : " + + sourceDir.getPath()); + } + } + return sourceDir; + } + + /** + * @param args + * main parameters + * @return translation file from parameters or default (base on current + * Locale) + * @throws IllegalArgumentException + * when no parameters is set and default is not found + */ + protected static File getTranslationFile(String[] args) { + File translationFile; + if (args.length > 2) { + translationFile = new File(args[2]); + } else { + String workingDir = System.getProperty("user.dir"); + if (workingDir == null) { + throw new IllegalArgumentException( + "No translation file specified, and default not found"); + } + translationFile = new File(workingDir, "locales" + File.separator + + Locale.getDefault().getLanguage() + ".lng"); + if (!translationFile.exists()) { + throw new IllegalArgumentException( + "No translation file specified, and default not found : " + + translationFile.getPath()); + } + } + return translationFile; + } + + /** + * @param args + * main parameters + * @return extensions list from parameters or default (same as used in + * {@link yacy}) + */ + protected static String getExtensions(String[] args) { + String extensions; + if (args.length > 3) { + extensions = args[3]; + } else { + extensions = "html,template,inc"; + } + return extensions; + } + +}