Update scripts to handle tranlsations

Author: goanpeca

.gitignore

@@ -145,7 +145,6 @@ doc/source/savefig/
 
 # Web & Translations #
 ##############################
-web/pandas-translations.tar.gz
 web/translations/
 web/pandas/pt/
 web/pandas/es/

web/pandas_translations.py

@@ -5,85 +5,52 @@
     Popen,
 )
 import tarfile
-
 import requests
-import yaml
+import io
+import shutil
 
 
-def download_translations(url, fname):
+def download_and_extract_translations(url, dir_name):
     """
     Download the translations from the GitHub repository.
     """
     response = requests.get(url)
     if response.status_code == 200:
-        with open(fname, "wb") as f:
-            f.write(response.content)
+        doc = io.BytesIO(response.content)
+        with tarfile.open(None, "r:gz", doc) as tar:
+            tar.extractall(dir_name)
     else:
         raise Exception(f"Failed to download translations: {response.status_code}")
 
 
-def extract_translations(fpath, dir_name):
-    """
-    Extract the translations from the tar file.
-    """
-    with tarfile.open(fpath, "r:gz") as tar:
-        tar.extractall(dir_name)
-    print(f"Translations extracted to '{dir_name}' directory.")
-
-
-def load_translations_config(path):
-    """
-    Load the translations configuration from the YAML file.
-    """
-    with open(path) as f:
-        config = yaml.safe_load(f)
-    return config
-
-
-def load_status_config(path):
+def copy_translations(src_path, dest_path):
     """
-    Load the translations configuration from the YAML file.
+    Copy the translations to the appropriate directory.
     """
-    with open(path) as f:
-        config = yaml.safe_load(f)
-    return config
+    languages_path = f"{src_path}/pandas-translations-main/web/pandas/"
+    en_path = f"{languages_path}/en/"
+    if os.path.exists(en_path):
+        shutil.rmtree(en_path)
 
+    languages = []
+    
+    paths = os.listdir(languages_path)
+    for path in paths:
+        if os.path.isdir(f"{languages_path}/{path}"):
+            languages.append(path)
 
-def copy_translations(data, translation_percentage, dir_name, dest_dir):
-    """
-    Copy the translations to the appropriate directory.
-    """
-    for lang, value in data.items():
-        if value["progress"] >= translation_percentage:
-            language_code = lang[:2]
-            src_path = (
-                f"{dir_name}/pandas-translations-main/web/pandas/{language_code}/"
-            )
-            dest_path = f"{dest_dir}/{language_code}/"
-            cmds = ["rsync", "-av", "--delete", src_path, dest_path]
-            p = Popen(cmds, stdout=PIPE, stderr=PIPE)
-            stdout, stderr = p.communicate()
-            print(stdout.decode())
-            print(stderr.decode())
+    print(languages)
+    cmds = ["rsync", "-av", "--delete", languages_path, dest_path]
+    p = Popen(cmds, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = p.communicate()
+    print(stdout.decode())
+    print(stderr.decode())
 
 
 if __name__ == "__main__":
     os.chdir(Path(__file__).parent.parent)
     url = "https://github.com/Scientific-Python-Translations/pandas-translations/archive/refs/heads/main.tar.gz"
-    fpath = "web/pandas-translations.tar.gz"
-    dir_name = "web/translations"
-    dest_dir = "web/pandas"
-    config_path = (
-        f"{dir_name}/pandas-translations-main/.github/workflows/sync_translations.yml"
-    )
-    status_path = f"{dir_name}/pandas-translations-main/status.yml"
-
-    download_translations(url, fpath)
-    extract_translations(fpath, dir_name)
-    config = load_translations_config(config_path)
-    variables = config["jobs"]["sync_translations"]["steps"][0]["with"]
-    translation_percentage = int(variables["translation-percentage"])
-    status = load_status_config(status_path)
-    copy_translations(
-        status, translation_percentage, dir_name=dir_name, dest_dir=dest_dir
-    )
+    src_path = "web/translations"
+    dest_path = "web/pandas"
+    download_and_extract_translations(url, src_path)
+    copy_translations(src_path, dest_path)

web/pandas_web.py

@@ -81,11 +81,14 @@ def navbar_add_info(context):
         ``has_subitems`` that tells which one of them every element is. It
         also adds a ``slug`` field to be used as a CSS id.
         """
+        lang = context['language']
+        prefix = f"{lang}/" if lang != "en" else "" 
         for i, item in enumerate(context["navbar"]):
+            # print("lang", lang, prefix + (item["name"].replace(" ", "-").lower()))
             context["navbar"][i] = dict(
                 item,
                 has_subitems=isinstance(item["target"], list),
-                slug=(item["name"].replace(" ", "-").lower()),
+                slug=prefix + (item["name"].replace(" ", "-").lower()),
             )
         return context
 
@@ -202,12 +205,15 @@ def maintainers_add_info(context):
 
         # save the data fetched from github to use it in case we exceed
         # git github api quota in the future
-        with open(
-            pathlib.Path(context["target_path"]) / "maintainers.json",
-            "w",
-            encoding="utf-8",
-        ) as f:
-            json.dump(maintainers_info, f)
+        try:
+            with open(
+                pathlib.Path(context["target_path"]) / "maintainers.json",
+                "w",
+                encoding="utf-8",
+            ) as f:
+                json.dump(maintainers_info, f)
+        except Exception:
+            pass
 
         return context
 
@@ -396,6 +402,8 @@ def get_context(config_fname: str, **kwargs):
 
     context["source_path"] = os.path.dirname(config_fname)
     context.update(kwargs)
+    context["selected_language"] = context.get("language", "en")
+    context["languages"] = context.get("languages", ["en"])
 
     preprocessors = (
         get_callable(context_prep)
@@ -409,14 +417,25 @@ def get_context(config_fname: str, **kwargs):
     return context
 
 
-def get_source_files(source_path: str) -> typing.Generator[str, None, None]:
+def get_source_files(source_path: str, language, languages) -> typing.Generator[str, None, None]:
     """
     Generate the list of files present in the source directory.
     """
+    paths = []
+    all_languages = languages[:]
+    all_languages.remove(language)
     for root, dirs, fnames in os.walk(source_path):
         root_rel_path = os.path.relpath(root, source_path)
         for fname in fnames:
-            yield os.path.join(root_rel_path, fname)
+            path = os.path.join(root_rel_path, fname) 
+            for language in all_languages:
+                if path.startswith(language + "/"):
+                   break
+            else:
+                paths.append(path)
+
+    for path in paths:
+        yield path 
 
 
 def extend_base_template(content: str, base_template: str) -> str:
@@ -434,65 +453,85 @@ def extend_base_template(content: str, base_template: str) -> str:
 def main(
     source_path: str,
     target_path: str,
+    languages: typing.List[str],
 ) -> int:
     """
     Copy every file in the source directory to the target directory.
 
     For ``.md`` and ``.html`` files, render them with the context
     before copying them. ``.md`` files are transformed to HTML.
     """
-    config_fname = os.path.join(source_path, "config.yml")
+    base_source_path = source_path
+    base_target_path = target_path
+    for language in languages:
+        print(f"\n\nProcessing language: {language}\n\n")
+        if language != "en":
+            target_path = os.path.join(base_target_path, language)
+            source_path = os.path.join(base_source_path, language)
 
-    shutil.rmtree(target_path, ignore_errors=True)
-    os.makedirs(target_path, exist_ok=True)
+        shutil.rmtree(target_path, ignore_errors=True)
+        os.makedirs(target_path, exist_ok=True)
 
-    sys.stderr.write("Generating context...\n")
-    context = get_context(config_fname, target_path=target_path)
-    sys.stderr.write("Context generated\n")
+        config_fname = os.path.join(source_path, "config.yml")
+        sys.stderr.write("Generating context...\n")
 
-    templates_path = os.path.join(source_path, context["main"]["templates_path"])
-    jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))
+        context = get_context(config_fname, target_path=target_path, languages=languages, language=language)
+        sys.stderr.write("Context generated\n")
 
-    for fname in get_source_files(source_path):
-        if os.path.normpath(fname) in context["main"]["ignore"]:
-            continue
+        templates_path = os.path.join(source_path, context["main"]["templates_path"])
+        jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))
 
-        sys.stderr.write(f"Processing {fname}\n")
-        dirname = os.path.dirname(fname)
-        os.makedirs(os.path.join(target_path, dirname), exist_ok=True)
+        for fname in get_source_files(source_path, language, languages):
+            if os.path.normpath(fname) in context["main"]["ignore"]:
+                continue
 
-        extension = os.path.splitext(fname)[-1]
-        if extension in (".html", ".md"):
-            with open(os.path.join(source_path, fname), encoding="utf-8") as f:
-                content = f.read()
-            if extension == ".md":
-                body = markdown.markdown(
-                    content, extensions=context["main"]["markdown_extensions"]
+            sys.stderr.write(f"Processing {fname}\n")
+            dirname = os.path.dirname(fname)
+            os.makedirs(os.path.join(target_path, dirname), exist_ok=True)
+
+            extension = os.path.splitext(fname)[-1]
+            if extension in (".html", ".md"):
+                with open(os.path.join(source_path, fname), encoding="utf-8") as f:
+                    content = f.read()
+                if extension == ".md":
+                    body = markdown.markdown(
+                        content, extensions=context["main"]["markdown_extensions"]
+                    )
+                    # Apply Bootstrap's table formatting manually
+                    # Python-Markdown doesn't let us config table attributes by hand
+                    body = body.replace("<table>", '<table class="table table-bordered">')
+                    content = extend_base_template(body, context["main"]["base_template"])
+                context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/"))
+                content = jinja_env.from_string(content).render(**context)
+                fname_html = os.path.splitext(fname)[0] + ".html"
+                with open(
+                    os.path.join(target_path, fname_html), "w", encoding="utf-8"
+                ) as f:
+                    f.write(content)
+            else:
+                shutil.copy(
+                    os.path.join(source_path, fname), os.path.join(target_path, dirname)
                 )
-                # Apply Bootstrap's table formatting manually
-                # Python-Markdown doesn't let us config table attributes by hand
-                body = body.replace("<table>", '<table class="table table-bordered">')
-                content = extend_base_template(body, context["main"]["base_template"])
-            context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/"))
-            content = jinja_env.from_string(content).render(**context)
-            fname_html = os.path.splitext(fname)[0] + ".html"
-            with open(
-                os.path.join(target_path, fname_html), "w", encoding="utf-8"
-            ) as f:
-                f.write(content)
-        else:
-            shutil.copy(
-                os.path.join(source_path, fname), os.path.join(target_path, dirname)
-            )
+    return 0
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Documentation builder.")
+
+    # For each language, the output will be written in a subdirectory named
+    # after the language (default: build/, build/es, etc...)
+    parser.add_argument(
+        "--target-path", default="build", help="directory where to write the output."
+    )
+    # e.g. python pandas_web.py --source_path pandas/ --languages en pt
     parser.add_argument(
-        "source_path", help="path to the source directory (must contain config.yml)"
+        "source_path",
+        help="path to the source directory (must contain each language folder)",
     )
     parser.add_argument(
-        "--target-path", default="build", help="directory where to write the output"
+        "--languages",
+        nargs="*",
+        help="language codes to build",
     )
     args = parser.parse_args()
-    sys.exit(main(args.source_path, args.target_path))
+    sys.exit(main(args.source_path, args.target_path, args.languages))