1: <?php
2: /*****************************************************************************************
3: * X2Engine Open Source Edition is a customer relationship management program developed by
4: * X2Engine, Inc. Copyright (C) 2011-2016 X2Engine Inc.
5: *
6: * This program is free software; you can redistribute it and/or modify it under
7: * the terms of the GNU Affero General Public License version 3 as published by the
8: * Free Software Foundation with the addition of the following permission added
9: * to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED WORK
10: * IN WHICH THE COPYRIGHT IS OWNED BY X2ENGINE, X2ENGINE DISCLAIMS THE WARRANTY
11: * OF NON INFRINGEMENT OF THIRD PARTY RIGHTS.
12: *
13: * This program is distributed in the hope that it will be useful, but WITHOUT
14: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15: * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
16: * details.
17: *
18: * You should have received a copy of the GNU Affero General Public License along with
19: * this program; if not, see http://www.gnu.org/licenses or write to the Free
20: * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21: * 02110-1301 USA.
22: *
23: * You can contact X2Engine, Inc. P.O. Box 66752, Scotts Valley,
24: * California 95067, USA. or at email address [email protected].
25: *
26: * The interactive user interfaces in modified source and object code versions
27: * of this program must display Appropriate Legal Notices, as required under
28: * Section 5 of the GNU Affero General Public License version 3.
29: *
30: * In accordance with Section 7(b) of the GNU Affero General Public License version 3,
31: * these Appropriate Legal Notices must retain the display of the "Powered by
32: * X2Engine" logo. If the display of the logo is not reasonably feasible for
33: * technical reasons, the Appropriate Legal Notices must display the words
34: * "Powered by X2Engine".
35: *****************************************************************************************/
36:
37: /**
38: * A behavior to automatically parse the software for translation calls on text,
39: * add that text to our translation files, consolidate duplicate entries into
40: * common.php, and then translate all missing entries via Google Translate API.
41: * To run the translation automation, navigate to "admin/automateTranslation" in
42: * the software. End users should never need to run this code (and in fact without
43: * the Google API Key file and Google Translation Billing API configured it
44: * will not work). This class is primarily designed for developer use to update
45: * translations for new releases.
46: * @package application.components
47: * @author "Jake Houser" <[email protected]>, "Demitri Morgan" <[email protected]>
48: */
49: class X2TranslationBehavior extends CBehavior {
50:
51:
52: /*
53: * This behemoth of a regex is now generated by getRegex with configurable
54: * special characters.
55: *
56: * const REGEX = '/(?:(?<installer>installer_tr?)\s*|Yii::\s*t\s*)\(\s*(?(installer)|(?:(?<openquote1>")|\')(?<module>\w+)(?(openquote1)"|\')\s*,)\s*(?<message>(?:((?<openquote2>")|\')(?:(?(openquote2)\\\\"|\\\\\')|(?(openquote2)\'|")|\w|\s|[\(\)\{\}_\.\-\,\*\#\|\&\!\?\/\<\>;:])+(?(openquote2)"|\')((\.\n\s*)|(\n\s*\.\s*))?)+)/';
57: */
58:
59: private $_regex;
60: private $_allowedChars;
61:
62: public $verbose = false;
63:
64: public $newMessages = 0;
65: public $addedToCommon = 0;
66: public $messagesRemoved = 0;
67: public $untranslated = 0;
68: public $characterCount = 0;
69: public $customMessageCount = 0;
70: public $languageStats = array();
71: public $errors = array();
72: public $limitReached = false;
73:
74: /**
75: * The regular expression for matching calls to Yii::t
76: *
77: * See protected/tests/data/messageparser/modules1.php for examples of what
78: * will be matched by this pattern.
79: * @param string $allowedChars valid special characters to match inside of translation calls
80: * @return string the constructed regex pattern
81: */
82: public function getRegex($allowedChars = "(){}_.-,+^%@*#|&!?/<>;:"){
83: if(!isset($this->_regex) || $this->_allowedChars !== $allowedChars){
84: // Forward slash for delimeter
85: $regex = '/';
86:
87: /*
88: * Non-capturing match installer_tr or Yii::t
89: * installer_tr is the translation function for requirements and installation
90: * Yii::t can optionally have spaces on either side of the 't'
91: */
92: $regex .= '(?:(?<installer>installer_tr?)\s*|Yii::\s*t\s*)';
93:
94: /*
95: * If installer has been captured, match nothing followed by optional whitepsace and a comma
96: * Otherwise, match an opening quote, a word, and a closing quote followed by optional
97: * white space and a comma. This block corresponds to the translation file in a Yii::t
98: * call i.e. the pattern will now match "Yii::t('app',"
99: */
100: $regex .= '\(\s*(?(installer)|(?:(?<openquote1>")|\')(?<module>\w+)(?(openquote1)"|\')\s*,)';
101:
102: //Match optional whitespace. This separation exists to clearly distinguish the next block
103: $regex .= '\s*';
104:
105: /*
106: * This piece defines the start of the message. Begin the message
107: * named subpattern and match the initial quote as either a single or double
108: * quote, and based on the openquote2 subpattern we will know which type it
109: * was.
110: */
111: $regex .= '(?<message>(?:((?<openquote2>")|\')';
112:
113: // Everything that follows is considered the text of the message
114:
115: /*
116: * The first thing we have the ability to match in a message is an escaped
117: * quote. If we matched a doublequote at first, we can match a \" by adding
118: * the \\\\" pattern. Otherwise, we match escaped singlequotes with \\\\\\'
119: * which matches \ followed by \'
120: */
121: $regex .= '(?:(?(openquote2)\\\\"|\\\\\')|';
122:
123: /*
124: * The next valid match is an unescaped quote. If we matched a double quote
125: * first, that equates to \' and if we matched a single quote first, that
126: * equates to "
127: */
128: $regex .= '(?(openquote2)\'|")|';
129:
130: /*
131: * The next things we're allowed to have in a translation message are
132: * word characters and whitespace. Fairly self-explanatory.
133: */
134: $regex .= '\w|\s|';
135:
136: /*
137: * We can also match non-word characters that might be found inside of
138: * translation calls. Certain special characters are not allowed (like $)
139: * for various reasons. The $allowedChars parameter for this function
140: * builds this list.
141: */
142: $chars = str_split($allowedChars);
143: $regex .= '[\\'.implode('\\',$chars).']';
144:
145: /*
146: * Close our current capturing segment and expect to see one or more
147: * of the previous pattern (the actual letters of the message)
148: */
149: $regex .= ')+';
150:
151: /*
152: * Next, we match the closing quote for the translation message. If we
153: * matched a double quote, it'll be a ", otherwise '
154: */
155: $regex .= '(?(openquote2)"|\')';
156:
157: /*
158: * Next, we can optionally match either a . followed by a newline and optional
159: * whitespace or a newline followed by optional whitespace and a .
160: * This pattern matches multiline translation calls with concatenated strings
161: */
162: $regex .= '((\h*\.\h*\n\h*)|(\h*\n\h*\.\h*))?';
163:
164: /*
165: * Finally, expect to see one or more lines of messages and close the
166: * message named subpattern
167: */
168: $regex .= ')+)';
169:
170: //Closing delimiter
171: $regex .= '/';
172:
173: $this->_allowedChars = $allowedChars;
174: $this->_regex = $regex;
175: }
176: return $this->_regex;
177: }
178:
179: /**
180: * Add missing translations to files, first step of automation.
181: *
182: * Function to find all untralsated text in the software, and then take that
183: * array of messages and add them to translation files for all languages.
184: * Called in {@link X2TranslationAction::run} function as part of the full
185: * translation suite.
186: */
187: public function addMissingTranslations(){
188: $this->verbose && print("Searching for missing translations...\n");
189: $messages = $this->getAttributeLabels();
190: $files = $this->fileList();
191: $this->verbose && print("Searching filesystem for translation calls...\n");
192: foreach($files as $file){
193: $messages = array_merge_recursive($messages, $this->getMessageList($file));
194: }
195: $languages = $this->getValidLanguagePacks();
196: $this->verbose && print("Adding new messages to language packs...\n");
197: foreach ($languages as $lang) {
198: if ($lang != '.' && $lang != '..') { // Don't include the current or parent directory.
199: foreach ($messages as $fileName => $messageList) {
200: $file = Yii::app()->basePath."/messages/$lang/$fileName.php";
201: $common = Yii::app()->basePath."/messages/$lang/common.php";
202: $this->addMessages($file, $messageList, $common); // Add each message to the end of the relevant file.
203: }
204: }
205: }
206: $this->verbose && print("Adding missing translations complete!\n");
207: }
208:
209: public function getAttributeLabels() {
210: $this->verbose && print("Checking for untranslated attribute labels...\n");
211: $fields = Yii::app()->db->createCommand()
212: ->select('attributeLabel, modelName')
213: ->from('x2_fields')
214: ->where('custom=0')
215: ->queryAll(); // Grab all the attribute labels for fields for all non-custom modules that might need to be translated.
216: foreach ($fields as $field) {
217: if ($translationFile = $this->getTranslationFileName($field['modelName'])) { // Get the name of the translation file each model is associated with.
218: $messages[$translationFile][] = $field['attributeLabel']; // Add the attribute labels to our list of text to be translated.
219: }
220: }
221: return $messages;
222: }
223:
224: /**
225: * Converts model name to translation file name.
226: *
227: * Helper method called in {@link getMessageList} to
228: * find the correct translation file for a model. This is necessary because some
229: * models have class names like Quote or Opportunity but their file names are
230: * quotes and opportunities.
231: *
232: * @param string $modelName The name of the model to look up the related translation file for.
233: * @return string|boolean Returns the name of the translation file to use, or false if a correct file cannot be found.
234: */
235: public function getTranslationFileName($modelName){
236: $excludeList = array(
237: 'BugReports', // Don't translate bug reports... not really used as a module
238: );
239: $modelToTranslation = array(
240: 'Accounts' => 'accounts',
241: 'Actions' => 'actions',
242: 'Calendar' => 'calendar',
243: 'AnonContact' => 'marketing',
244: 'Campaign' => 'marketing',
245: 'Fingerprint' => 'marketing',
246: 'Charts' => 'charts',
247: 'Contacts' => 'contacts',
248: 'Docs' => 'docs',
249: 'EmailInboxes' => 'app',
250: 'Groups' => 'groups',
251: 'Media' => 'media',
252: 'Opportunity' => 'opportunities',
253: 'Product' => 'products',
254: 'Quote' => 'quotes',
255: 'Reports' => 'reports',
256: 'Services' => 'services',
257: 'Topics' => 'topics',
258: 'X2Leads' => 'x2Leads',
259: 'X2List' => 'contacts',
260: );
261: if(isset($modelToTranslation[$modelName])){
262: return $modelToTranslation[$modelName];
263: }else{
264: if(!in_array($modelName, $excludeList)){
265: if(!isset($this->errors['missingAttributes'])){
266: $this->errors['missingAttributes'] = array();
267: }
268: if(!in_array($modelName, $this->errors['missingAttributes'])){
269: $this->errors['missingAttributes'][] = $modelName;
270: }
271: }
272: return false; // Translation file not found for the specified model.
273: }
274: }
275:
276: /**
277: * Parse file structure for valid files.
278: *
279: * Returns a list of all files in the codebase that are eligible for searching
280: * for Yii::t calls within.
281: *
282: * @param string $revision Unused, may implement comparison between Git revisions rather than searching all files.
283: * @return array List of files to be parsed for Yii::t calls
284: */
285: public function fileList($revision = null){
286: $this->verbose && print("Generating list of files to search for translation calls...\n");
287: $cwd = Yii::app()->basePath;
288: $fileList = array();
289: $basePath = realpath($cwd.'/../');
290: $objects = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($basePath), RecursiveIteratorIterator::SELF_FIRST); // Build PHP File Iterator to loop through valid directories
291: foreach($objects as $name => $object){
292: if(!$object->isDir()){ // Make sure it's actually a file if we're going to try to parse it.
293: $relPath = str_replace("$basePath/", '', $name); // Get the relative path to it.
294: if(!$this->excludePath($relPath)){ // Make sure the file is not in one of the excluded diectories.
295: $fileList[] = $name;
296: }
297: }
298: }
299: return $fileList;
300: }
301:
302: /**
303: * Returns true or false based on whether a path should be parsed for
304: * messages.
305: *
306: * Some files in the software don't need to be translated. Yii provides all
307: * of its own translations for the framework directory, and there are other
308: * files which simply have no possibility of having Yii::t calls in them.
309: * Ignoring these files speeds up the process, especially since framework is
310: * a very large directory.
311: *
312: * @param string $relPath Paths to folders which should not be included in the Yii::t search
313: * @return boolean True if file should be excluded from the search, false if the file is OK.
314: */
315: public function excludePath($relPath){
316: $paths = array(
317: 'framework', // Yii handles its own translations
318: 'protected/data', //Data files do not have Yii::t calls
319: 'protected/messages', // These are the translation files...
320: 'protected/extensions', // Extensions are rarely translated and generally don't display text.
321: 'protected/integration', // Integrations are rarely translated and generally don't display text.
322: 'protected/migrations', // Migrations are all back-end and have no text
323: 'protected/tests', // Unit tests have no translation calls
324: 'backup', // Backup of older files that may no longer be relevant
325: );
326: foreach($paths as $path)
327: if(strpos($relPath, $path) === 0) // We found the excluded directory in the relative path.
328: return true;
329: return !preg_match('/\.php$/', $relPath); // Only look in PHP files.
330: }
331:
332: /**
333: * Gets a list of Yii::t calls.
334: *
335: * Helper function called by {@link addMissingTranslations}
336: * to get a list of messages found in Yii::t calls found in the software in
337: * an easily parsed array format. Also checks attribute labels of non-custom
338: * modules in the x2_fields table.
339: *
340: * @return array An array of messages found in the software that need to be added to the translation files.
341: */
342: public function getMessageList($file) {
343: $messages = array();
344: $newMessages = $this->parseFile($file); // Parse the file for all messages within Yii::t calls.
345: foreach ($newMessages as $fileName => $messageList) { // Loop through the found messages.
346: if (array_key_exists($fileName, $messages)) { // We've already got this file in our return array
347: $messages[$fileName] = array_unique(array_merge($messages[$fileName],
348: array_keys($messageList))); // Merge the new messages with the old messages for the given file
349: } else {
350: $messages[$fileName] = array_unique(array_keys($messageList)); // Otherwise, define the messages we found as the initial data set for this file.
351: }
352: }
353: return $messages;
354: }
355:
356: /**
357: * Return Yii::t calls in a specific file
358: *
359: * Helper method called in {@link getMessageList}
360: * Parses a file and returns an associative array of module names to messages
361: * for that file.
362: *
363: * @param string $path Filepath to the file to be checked by the REGEX
364: * @return array An array of messages in Yii::t calls in the provided file.
365: */
366: public function parseFile($path){
367: if(!file_exists($path))
368: return array();
369: preg_match_all($this->getRegex(), file_get_contents($path), $matches);
370: // Modify the match array to incorporate the special installer_t case
371: foreach($matches['installer'] as $index => $groupText)
372: if($groupText != '')
373: $matches['module'][$index] = 'install';
374:
375: $messages = array_fill_keys(array_unique($matches['module']), array());
376: foreach($matches['message'] as $index => $message){
377: $message = $this->parseRegexMatch($message);
378: $message = str_replace("\\'", "'", $message);
379: //$message = str_replace("'", "\\'", $message);
380: $messages[$matches['module'][$index]][$message] = '';
381: }
382: if(isset($messages['yii'])){
383: unset($messages['yii']);
384: }
385: return $messages;
386: }
387:
388: public function parseRegexMatch($message) {
389: $ret = preg_replace("/(\'|\")((\h*\.\h*(\r\n?|\n)\h*)|(\h*(\r\n?|\n)\h*\.\h*))(\'|\")/", '', $message);
390: if (strpos($ret, '"') === 0 || strpos($ret, "'") === 0) {
391: $ret = substr($ret, 1);
392: }
393: if (strpos(strrev($ret), '"') === 0 || strpos(strrev($ret), "'") === 0) {
394: $ret = substr($ret, 0, -1);
395: }
396: return $ret;
397: }
398:
399: /**
400: * Commented out until unit test is built.
401: * @param type $file
402: * @param type $messageList
403: */
404: public function addMessages($file, $messageList, $common = null) {
405: if (file_exists($file)) {
406: $fileMessages = require $file;
407: if (isset($common) && file_exists($common)) {
408: $messages = array_merge(array_keys(require $file),
409: array_keys(require $common)); // Get all of the messages already in the appropriate language as well as common.php
410: } else {
411: $messages = array_keys(require $file);
412: }
413: $diff = array_diff($messageList, $messages); // Create a diff array of messages not already in the provided language file or common.php
414: if (!empty($diff)) {
415: $contents = file_get_contents($file); // Grab the array of messages from the translation file.
416: foreach ($diff as $message) {
417: if (strpos($file, 'template') !== false) {
418: //Only count new messages once.
419: $this->newMessages++;
420: $this->verbose && print (' Adding: '.$message."\n");
421: }
422: $fileMessages[$message] = '';
423: }
424: $this->writeMessagesToFile($file, $fileMessages);
425: }
426: } else {
427: if (!isset($this->errors['missingFiles']))
428: $this->errors['missingFiles'] = array();
429: $this->errors['missingFiles'][] = $file;
430: }
431: }
432:
433: /**
434: * Move commonly used phrases to common.php, second step of automation.
435: *
436: * Function that parses translation files for all languages and consolidates
437: * them. First it builds a list of redundancies between files, then loops
438: * through that array, adding redundant phrases to common.php and removing
439: * them from their original files. This means any given word/phrase in the
440: * software only needs to be translated once. Called in {@link X2TranslationAction::run}
441: * function as part of the full translation suite.
442: */
443: public function consolidateMessages(){
444: $this->verbose && print("Consolidating duplicate messages into common...\n");
445: $redundancies = $this->buildRedundancyList(); // Get a list of all redundancies between translation files and store it in $this->intersect.
446: $this->verbose && print(count($redundancies)." redundancies found.\n");
447: for($i = 0; $i < 5 && !empty($redundancies); $i++){ // Keep going until we run out of attempts or there are no more redundant translations.
448: foreach($redundancies as $data){
449: $first = $data['first']; // Get the name of the first file that has the redundancy
450: $second = $data['second']; // Get the name of the second file that has the redundancy
451: $messages = $data['messages']; // Get the text of the redundant message.
452: foreach($messages as $message){
453: if($first != 'common.php' && $second != 'common.php'){ // If neither of the matched files are common.php
454: $this->verbose && print(' Moving '.$message.' from '.$first.' and '.$second." to common.php\n");
455: $this->addedToCommon++;
456: $this->addToCommon($message); // Add the message to common.php
457: }
458: if($first != 'common.php'){ // Only remove messages from the original files if the file isn't common.php
459: $this->verbose && print(' Removing '.$message.' from '.$first."\n");
460: $this->messagesRemoved++;
461: $this->removeMessage($first, $message);
462: }
463: if($second != 'common.php'){
464: $this->verbose && print(' Removing '.$message.' from '.$second."\n");
465: $this->messagesRemoved++;
466: $this->removeMessage($second, $message);
467: }
468: }
469: }
470: $redundancies = $this->buildRedundancyList(); // Rebuild the redundancy list to be sure there aren't any new redundancies created by the process
471: }
472: $this->verbose && print("Consolidating duplicate messages complete!\n");
473: }
474:
475: /**
476: * Get redundant translations to be merged into common.php
477: *
478: * Helper function called by {@link consolidateMessages)
479: * to build a list of files that have redundant messages in them, as well as a
480: * list of what those messages are. Loads this data into the property
481: * $this->intersect;
482: */
483: public function buildRedundancyList(){
484: $redundancies = array();
485: $files = scandir(Yii::app()->basePath.'/messages/template'); // Only need to check template, not all languages. All languages should mirror template.
486: $languageList = array();
487: foreach($files as $file){
488: if($file != '.' && $file != '..'){
489: $languageList[$file] = array_keys(include(Yii::app()->basePath."/messages/template/$file")); // Get the messages from each file in the template folder.
490: }
491: }
492: $keys = array_keys($languageList);
493: for($i = 0; $i < count($languageList); $i++){ // Outer loop to check all files in the language list.
494: for($j = $i + 1; $j < count($languageList); $j++){ // Inner loop to compare each file against each other file.
495: $messages = array_intersect($languageList[$keys[$i]], $languageList[$keys[$j]]); // Calculate the intersection of the messages between each pair of files.
496: if(!empty($messages)){ // If we found messages that exist in both, add them to the intersect array to be consolidated.
497: $redundancies[] = array('first' => $keys[$i], 'second' => $keys[$j], 'messages' => $messages);
498: }
499: }
500: }
501: return $redundancies;
502: }
503:
504: /**
505: * Add a message to common.php for all languages
506: *
507: * Helper function called by {@link consolidateMessages}
508: * to add a redundant message into common.php. The message will nto be added
509: * if it already exists in common.
510: *
511: * @param string $message The message to be added to common.php
512: */
513: public function addToCommon($message){
514: $languages = $this->getValidLanguagePacks();
515: foreach($languages as $lang){
516: if($lang != '.' && $lang != '..'){
517: $fileName = Yii::app()->basePath.'/messages/'.$lang.'/'.'common.php';
518: if(!file_exists($fileName)){ // For some reason common.php doesn't exist for this language.
519: $this->writeMessagesToFile($fileName, array());
520: }
521: $messages = require $fileName;
522: if(!array_key_exists($message, $messages)){
523: $messages[$message] = '';
524: $this->writeMessagesToFile($fileName, $messages);
525: }
526: }
527: }
528: }
529:
530: /**
531: * Deletes a message from a language file in all languages.
532: *
533: * Called as a part of the consolidation process to remove redundant messages
534: * from the files they were found in. This keeps the amount of messages lower
535: * and reduced the burden on anyone who is translating the software.
536: *
537: * @param string $file The name of the file to look for the message in
538: * @param string $message The message to be removed
539: */
540: public function removeMessage($file, $message){
541: $languages = $this->getValidLanguagePacks(); // Load all languages.
542: foreach($languages as $lang){
543: if($lang != '.' && $lang != '..'){
544: if(file_exists(Yii::app()->basePath.'/messages/'.$lang.'/'.$file)){
545: $messages = require Yii::app()->basePath.'/messages/'.$lang.'/'.$file;
546: if(isset($messages[$message])){
547: unset($messages[$message]);
548: }
549: $this->writeMessagesToFile(Yii::app()->basePath.'/messages/'.$lang.'/'.$file, $messages);
550: }
551: }
552: }
553: }
554:
555: /**
556: * Call Google Translate API for mising translations, third step of automation.
557: *
558: * This method will get a list of all messages which do not have translations
559: * into the appropriate language from all of our language files. Then, it will
560: * call Google Translate's API to get a base translation of the message and
561: * insert the translated versions into our translation files. Called in
562: * {@link X2TranslationAction::run} function as part of the full translation suite.
563: */
564: public function updateTranslations(){
565: $this->verbose && print("Translating messages via Google Translate API...\n");
566: $untranslated = $this->getUntranslatedText(); // Get a list of all messages with missing translations.
567: $limit = $this->untranslated; // Set limit to number of expected translations to prevent infinite loops
568: $this->verbose && print($this->untranslated." messages need to be translated. Setting limit to ".$this->untranslated.".\n");
569: foreach($untranslated as $lang => $langData){
570: $this->languageStats[$lang] = 0; // Start tracking stats for this langage.
571: foreach($langData as $fileName => $file){
572: $translations = array(); // Store translated messages to only do 1 file write per file.
573: foreach($file as $index){
574: if($limit >= 0){
575: $limit--;
576: $message = $this->translateMessage($index, $lang); // Translate message for the specified language
577: $translations[$index] = $message; // Store the translation (and original message) to be written to the file later.
578: $this->languageStats[$lang]++;
579: }else{ // We hit our limit
580: $this->replaceTranslations($lang, $fileName, $translations); // Replace translations for what we have now, we'll manually refresh to get more.
581: $this->limitReached = true;
582: break 3; // Break out of all the loops to save time
583: }
584: }
585: $this->replaceTranslations($lang, $fileName, $translations); // Replace the translated messages into the right file.
586: }
587: }
588: $this->verbose && print("Translating via Google API complete!\n");
589: }
590:
591: /**
592: * Get all untranslated messages
593: *
594: * Helper function called by {@link updateTranslations}
595: * to get an array of all messages which have indeces in the translation files
596: * but no translated version.
597: *
598: * @return array A list of all messages which have missing translations.
599: */
600: public function getUntranslatedText() {
601: $untranslated = array();
602: $languages = $this->getValidLanguagePacks();
603: foreach ($languages as $lang) {
604: if (!in_array($lang, array('template', '.', '..'))) { // Ignore current, parent, and template (all template translations are blank) directories.
605: $untranslated[$lang] = array();
606: $files = scandir(Yii::app()->basePath . '/messages/' . $lang); // Get all the files for the current language.
607: foreach ($files as $file) {
608: if ($file != '.' && $file != '..') {
609: $untranslated[$lang][$file] = array();
610: $translations = (include(Yii::app()->basePath . '/messages/' . $lang . '/' . $file)); // Include the translations.
611: foreach ($translations as $index => $message) {
612: if (!empty($index) && empty($message)) {
613: $untranslated[$lang][$file][] = $index; // If the translated version is empty, add the message index to our unranslated array.
614: $this->untranslated++;
615: }
616: }
617: if (empty($untranslated[$lang][$file])) {
618: unset($untranslated[$lang][$file]); // If we don't find any untranslated messages, don't both returning that file.
619: }
620: }
621: }
622: if (empty($untranslated[$lang])) {
623: unset($untranslated[$lang]); // The whole language is translated, no need to return it either.
624: }
625: }
626: }
627: return $untranslated;
628: }
629:
630: /**
631: * Translate a message via Google Translate API.
632: *
633: * Helper function called by {@link updateTranslations}
634: * to translate individual messages via the Google Translate API. Any text
635: * between braces {} is preserved as is for variable replacement.
636: *
637: * @param string $message The untranslated message
638: * @param string $lang The language to translate to
639: * @return string The translated message
640: */
641: public function translateMessage($message, $lang) {
642: $this->verbose && print(" Translating $message to $lang\n");
643: $key = require Yii::app()->basePath . '/config/googleApiKey.php'; // Git Ignored file containing the Google API key to store. Ours is not included with public release for security reasons...
644: $message = $this->addNoTranslateTags($message);
645: $this->characterCount+=mb_strlen($message, 'UTF-8');
646: $params = array(
647: 'key' => $key,
648: 'source' => 'en',
649: 'target' => $lang,
650: 'q' => $message,
651: );
652: $url = 'https://www.googleapis.com/language/translate/v2?' . http_build_query($params);
653: $data = RequestUtil::request(array(
654: 'url' => $url,
655: 'method' => 'GET',
656: ));
657: $data = json_decode($data, true); // Response is JSON, need to decode it to an array.
658: if (isset($data['data'], $data['data']['translations'],
659: $data['data']['translations'][0],
660: $data['data']['translations'][0]['translatedText'])) {
661: $message = $data['data']['translations'][0]['translatedText']; // Make sure the data structure returned is correct, then store the message as the translated version.
662: } else {
663: $message = ''; // Otherwise, leave the message blank.
664: }
665: $message = $this->removeNoTranslateTags($message);
666: $message = trim($message, '\\/'); // Trim any harmful characters Google Translate may have moved around, like leaving a "\" at the end of the string...
667: return $message;
668: }
669:
670: public function addNoTranslateTags($message){
671: return preg_replace_callback('/(\{(.*?)\}|<(.*?)>)/', function($matches){
672: return '<span class="notranslate">'.$matches[0].'</span>'; // Replace every instance of text between braces like {text} with <span class="notranslate">{text}</span>. This will make Google Translate ignore that text.
673: }, $message);
674: }
675:
676: public function removeNoTranslateTags($message){
677: return preg_replace_callback('/'.preg_quote('<span class="notranslate">', '/').'(.*?)'.preg_quote('</span>', '/').'/', function($matches){
678: return $matches[1];
679: }, $message);
680: }
681:
682: /**
683: * Add translated messages to translation files.
684: *
685: * Helper function called by {@link updateTranslations}
686: * to replace the untranslated messages in a translation file with the response
687: * we got from Google.
688: *
689: * @param string $lang The language we translated our messages to
690: * @param string $file The file we need to put the translations in
691: * @param array $translations An array of translations with the English message as the index and the translated version as the value.
692: */
693: public function replaceTranslations($lang, $file, $translations){
694: $this->verbose && print(" Writing translations to $lang/$file\n");
695: $fileName = Yii::app()->basePath.'/messages/'.$lang.'/'.$file;
696: if(file_exists($fileName)){
697: $messages = require $fileName;
698: $messages = array_merge($messages,$translations);
699: $this->writeMessagesToFile($fileName, $messages);
700: }
701: }
702:
703: public function mergeCustomTranslations() {
704: $customDir = str_replace('/protected','/custom/protected',Yii::app()->basePath);
705: if (is_dir($customDir . '/messages/')) {
706: $customMessages = $customDir . '/messages';
707: $customLanguagePacks = array_diff(scandir($customMessages),
708: array('.', '..'));
709: foreach ($customLanguagePacks as $dirName) {
710: if (is_dir($customMessages . '/' . $dirName)) {
711: $this->mergeCustomLanguagePack($customMessages . '/' . $dirName);
712: }
713: }
714: }
715: }
716:
717: public function mergeCustomLanguagePack($dir){
718: $languageFiles = array_diff(scandir($dir),array('.','..'));
719: foreach($languageFiles as $file){
720: if(is_file($dir.'/'.$file)){
721: $this->mergeCustomTranslationFile($dir.'/'.$file);
722: }
723: }
724: }
725:
726: public function mergeCustomTranslationFile($file){
727: $customMessages = require $file;
728: $this->customMessageCount += count($customMessages);
729: if(is_array($customMessages) && !empty($customMessages)){
730: $baseFile = str_replace('/custom','',$file);
731: if(file_exists($baseFile)){
732: $defaultMessages = require $baseFile;
733: $messages = array_merge($defaultMessages, $customMessages);
734: $this->writeMessagesToFile($baseFile, $messages);
735: }
736: }
737: }
738:
739: public function assimilateLanguageFiles(){
740: $languagePackPath = Yii::app()->basePath."/messages";
741: $languagePacks = array_diff(scandir($languagePackPath),array('.','..','template'));
742: foreach($languagePacks as $languagePack){
743: if (is_dir($languagePackPath . '/' . $languagePack)) {
744: $this->assimilateLanguagePack($languagePack);
745: }
746: }
747: }
748:
749: public function assimilateLanguagePack($lang){
750: $languagePackPath = Yii::app()->basePath."/messages";
751: $languageFiles = array_diff(scandir($languagePackPath . '/' . $lang),array('.','..'));
752: foreach($languageFiles as $file){
753: if(is_file($languagePackPath . '/' . $lang . '/' . $file)){
754: $this->assimilateLanguageFile($lang, $file);
755: }
756: }
757: }
758:
759: public function assimilateLanguageFile($lang, $file){
760: $templateMessages = require Yii::app()->basePath."/messages/template/$file";
761: $langMessages = require Yii::app()->basePath."/messages/$lang/$file";
762:
763: $intersection = array_intersect_key($langMessages,array_flip(array_keys($templateMessages)));
764:
765: $this->writeMessagesToFile(Yii::app()->basePath."/messages/$lang/$file", $intersection);
766: }
767:
768: /**
769: * Helper function exists in case we change how we write to files again.
770: */
771: private function writeMessagesToFile($file, $messages){
772: file_put_contents($file, '<?php return '.var_export( $messages, true ).";\n");
773: }
774:
775: private function getValidLanguagePacks() {
776: $languageDirs = scandir(Yii::app()->basePath . '/messages/'); // scan for installed language folders
777: $languages = array();
778: foreach ($languageDirs as $code) {
779: if ($this->isValidLanguagePack($code)) {
780: $languages[] = $code;
781: }
782: }
783: return $languages;
784: }
785:
786: private function isValidLanguagePack($code) { // lookup language name for the language code provided
787: $appMessageFile = Yii::app()->basePath . "/messages/$code/app.php";
788: if (file_exists($appMessageFile)) { // attempt to load 'app' messages in
789: $appMessages = include($appMessageFile); // the chosen language
790: return is_array($appMessages) && isset($appMessages['languageName']);
791: }
792: return false;
793: }
794:
795: }
796: