Skip to content

Commit 20fb276

Browse files
crazyservercrazyserver
committed
Create a language parser
1 parent 574c3a6 commit 20fb276

File tree

4 files changed

+508
-0
lines changed

4 files changed

+508
-0
lines changed

‎.github/workflows/parse.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Parse languages
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- 'main'
8+
paths:
9+
- 'langindex.json'
10+
schedule:
11+
- cron: '0 5 * * *'
12+
13+
jobs:
14+
parse:
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
version: ['3.9', '4.0', '4.1', '4.2', '4.3']
19+
steps:
20+
- uses: actions/checkout@v2
21+
- name: Setup PHP with PECL extension
22+
uses: shivammathur/setup-php@v2
23+
with:
24+
php-version: '8.2'
25+
- name: Parse
26+
env:
27+
GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
28+
run: ./parser/parse.sh ${{ matrix.version }}

‎parser/functions.php‎

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
<?php
2+
// This file is part of Moodle - http://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
/**
18+
* Helper functions converting moodle strings to json.
19+
*/
20+
21+
/**
22+
* Builds all languages specified in the list.
23+
* @param $time_first Time of the first execution.
24+
*/
25+
function build_languages($time_first) {
26+
global $LANGUAGES;
27+
global $LAST_RUN_LANGUAGES;
28+
global $STATS;
29+
30+
load_languages();
31+
32+
$now = time();
33+
$maxlangs = MAX_LANGS;
34+
35+
// Process the languages.
36+
foreach ($LANGUAGES as $language) {
37+
if (!get_langpack_folder($language->lang)) {
38+
echo "Cannot translate $language->lang, folder not found\n";
39+
continue;
40+
}
41+
42+
if (isset($LAST_RUN_LANGUAGES[$language->lang])) {
43+
$lastrun = $LAST_RUN_LANGUAGES[$language->lang];
44+
# Check already parsed in this script.
45+
if (isset($lastrun->lastupdate) && $lastrun->lastupdate > $time_first) {
46+
continue;
47+
}
48+
}
49+
50+
build_lang($language);
51+
$LAST_RUN_LANGUAGES[$language->lang] = $language;
52+
53+
$maxlangs--;
54+
if ($maxlangs <= 0) {
55+
break;
56+
}
57+
}
58+
59+
// Include stats and save.
60+
$LAST_RUN_LANGUAGES['languages'] = new StdClass();
61+
$LAST_RUN_LANGUAGES['languages']->local = $STATS->local;
62+
$LAST_RUN_LANGUAGES['languages']->total = $STATS->total;
63+
64+
save_json('languages.json', $LAST_RUN_LANGUAGES);
65+
}
66+
67+
/**
68+
* Loads lang index keys.
69+
*/
70+
function load_languages() {
71+
global $LANGUAGES;
72+
global $LAST_RUN_LANGUAGES;
73+
global $LANGINDEX;
74+
global $STATS;
75+
76+
$LANGUAGES = load_csv(LANGPACKSFOLDER . '/languages.md5', ['lang', 'md5', 'name']);
77+
$LAST_RUN_LANGUAGES = load_json('languages.json');
78+
79+
// Loads lang index keys.
80+
$local = 0;
81+
$total = 0;
82+
83+
// Process the index file, just once.
84+
$langindexjson = load_json('langindex.json');
85+
86+
$LANGINDEX = [];
87+
foreach ($langindexjson as $appkey => $value) {
88+
if ($value == APPMODULENAME) {
89+
$file = $value;
90+
$lmskey = $appkey;
91+
$local++;
92+
} else {
93+
$exp = explode('/', $value, 2);
94+
$file = $exp[0];
95+
if (count($exp) == 2) {
96+
$lmskey = $exp[1];
97+
} else {
98+
$exp = explode('.', $appkey, 3);
99+
100+
if (count($exp) == 3) {
101+
$lmskey = $exp[2];
102+
} else {
103+
$lmskey = $exp[1];
104+
}
105+
}
106+
}
107+
108+
if (!isset($LANGINDEX[$file])) {
109+
$LANGINDEX[$file] = [];
110+
}
111+
112+
$LANGINDEX[$file][$appkey] = $lmskey;
113+
$total++;
114+
}
115+
116+
if ($total == 0) {
117+
die("Langindex error. exiting...");
118+
}
119+
120+
$STATS = new StdClass();
121+
$STATS->local = $local;
122+
$STATS->total = $total;
123+
124+
echo "Strings to translate $total\n";
125+
echo "Local strings $local\n";
126+
echo "Languages ".count($LANGUAGES)."\n";
127+
}
128+
129+
/**
130+
* Build translations files from langpack.
131+
*
132+
* @param $language Language object including name and code.
133+
*/
134+
function build_lang(&$language) {
135+
global $STATS;
136+
global $LANGINDEX;
137+
138+
$lang = $language->lang;
139+
140+
$langfoldername = get_langpack_folder($lang);
141+
142+
$language->local = 0;
143+
144+
$langparts = explode('-', $lang, 2);
145+
$parentname = $langparts[0] ? $langparts[0] : "";
146+
$parent = "";
147+
148+
echo "Processing $language->name ($lang)";
149+
// Check parent language exists.
150+
if ($parentname != $lang && get_langpack_folder($parentname)) {
151+
echo " Parent: $parentname";
152+
$parent = $parentname;
153+
}
154+
155+
$langFile = false;
156+
if (file_exists($lang.'.json')) {
157+
// Load lang files just once.
158+
$langFile = load_json($lang.'.json');
159+
}
160+
161+
$translations = [];
162+
// Add the translation to the array.
163+
foreach ($LANGINDEX as $file => $keys) {
164+
$lmsstring = get_translation_strings($langfoldername, $file);
165+
if (empty($lmsstring)) {
166+
continue;
167+
}
168+
169+
foreach ($keys as $appkey => $lmskey) {
170+
if (!isset($lmsstring[$lmskey])) {
171+
continue;
172+
}
173+
174+
$text = $lmsstring[$lmskey];
175+
176+
if ($file != APPMODULENAME) {
177+
$text = str_replace('$a->@', '$a.', $text);
178+
$text = str_replace('$a->', '$a.', $text);
179+
$text = str_replace('{$a', '{{$a', $text);
180+
$text = str_replace('}', '}}', $text);
181+
$text = preg_replace('/@@.+?@@(<br>)?\\s*/', '', $text);
182+
// Prevent double.
183+
$text = str_replace(['{{{', '}}}'], ['{{', '}}'], $text);
184+
} else {
185+
// @TODO: Remove that line when core.cannotconnect and core.login.invalidmoodleversion are completelly changed to use $a
186+
if (($appkey == 'core.cannotconnect' || $appkey == 'core.login.invalidmoodleversion') && strpos($text, '2.4')) {
187+
if (DEBUG) {
188+
echo "****** Found 2.4 \n";
189+
}
190+
$text = str_replace('2.4', '{{$a}}', $text);
191+
}
192+
$language->local++;
193+
}
194+
195+
$translations[$appkey] = html_entity_decode($text);
196+
}
197+
}
198+
199+
if (!empty($parent)) {
200+
$translations['core.parentlanguage'] = $parent;
201+
} else if (isset($translations['core.parentlanguage'])) {
202+
unset($translations['core.parentlanguage']);
203+
}
204+
205+
// Sort and save.
206+
ksort($translations);
207+
save_json($lang.'.json', $translations);
208+
$language->lastupdate = time();
209+
210+
$language->translated = count($translations);
211+
$percentage = floor($language->translated/$STATS->total * 100);
212+
$bar = progressbar($percentage);
213+
if (strlen($lang) <= 2 && !$parent) {
214+
echo "\t";
215+
}
216+
echo "\t\t$language->translated of $STATS->total -> $percentage% $bar ($language->local local)\n";
217+
}
218+
219+
/**
220+
* Save json data.
221+
*
222+
* @param $path Path of the file to load.
223+
* @param $content Content string to save.
224+
*/
225+
function save_json($path, $content) {
226+
file_put_contents($path, str_replace('\/', '/', json_encode($content, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT))."\n");
227+
}
228+
229+
/**
230+
* Load json data.
231+
*
232+
* @param $path Path of the file to load.
233+
* @returns Associative array obtained from json.
234+
*/
235+
function load_json($path) {
236+
$file = file_get_contents($path);
237+
return (array) json_decode($file);
238+
}
239+
240+
/**
241+
* Load CSV data.
242+
*
243+
* @param $path Path of the file to load.
244+
* @param $fields Sorted fields. First param will be the array key.
245+
* @returns Associative array obtained from csv.
246+
*/
247+
function load_csv($path, $fields) {
248+
$csv = array_map('str_getcsv', file($path));
249+
$csv_sorted = [];
250+
foreach ($csv as $line) {
251+
$csv_sorted[$line[0]] = (object) array_combine($fields, $line);
252+
}
253+
254+
return $csv_sorted;
255+
}
256+
257+
/**
258+
* Get's lang folder from lang code.
259+
*
260+
* @param $lang Lang code.
261+
* @returns Folder path.
262+
*/
263+
function get_langpack_folder($lang) {
264+
$folder = LANGPACKSFOLDER.'/'.$lang;
265+
if (!is_dir($folder) || !is_file($folder.'/langconfig.php')) {
266+
return false;
267+
}
268+
269+
return $folder;
270+
}
271+
272+
/**
273+
* Import translation file from langpack and returns it.
274+
*
275+
* @param $langfoldername Lang folder path.
276+
* @param $file File name (excluding extension).
277+
* @returns String array.
278+
*/
279+
function get_translation_strings($langfoldername, $file) {
280+
$path = $langfoldername.'/'.$file.'.php';
281+
// Apply translations.
282+
if (!file_exists($path)) {
283+
return [];
284+
}
285+
286+
$string = [];
287+
288+
include($path);
289+
290+
return $string;
291+
}
292+
293+
/**
294+
* Generates an ASCII progress bar.
295+
*
296+
* @param $percentage Done part.
297+
* @param $length Length of the text.
298+
* @returns Text generated.
299+
*/
300+
function progressbar($percentage, $length = 10) {
301+
$done = floor($percentage / $length);
302+
return "\t".str_repeat('=', $done) . str_repeat('-', $length - $done);
303+
}

‎parser/moodle_to_json.php‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
// This file is part of Moodle - http://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
/**
18+
* Script for converting moodle strings to json.
19+
*/
20+
21+
// Check we are in CLI.
22+
if (isset($_SERVER['REMOTE_ADDR'])) {
23+
exit(1);
24+
}
25+
define('MOODLE_INTERNAL', 1);
26+
define('LANGPACKSFOLDER', '/tmp/moodle-langpacks');
27+
define('APPMODULENAME','local_moodlemobileapp');
28+
define('DEBUG', false);
29+
define('MAX_LANGS', 50);
30+
31+
global $strings;
32+
33+
require_once('functions.php');
34+
35+
$time = isset($argv[1]) ? $argv[1] : time();
36+
build_languages($time);

0 commit comments

Comments
 (0)