diff --git a/scripts/check_translations.py b/scripts/check_translations.py index b9aca0941..4fdebba2a 100644 --- a/scripts/check_translations.py +++ b/scripts/check_translations.py @@ -8,6 +8,7 @@ - Empty tag values """ +import re import sys import yaml from pathlib import Path @@ -61,7 +62,9 @@ def get_file_groups(self) -> Dict[str, List[Path]]: base_name = "-".join(parts[:-1]) # Only process card files with language codes - if "cards" in base_name and len(lang) == 2: + # Matches 2-letter codes (e.g. en, es) and regional codes (e.g. pt_pt, no_nb) + lang_pattern = re.compile(r"^[a-z]{2}([_-][a-z]{2})?$") + if "cards" in base_name and lang_pattern.match(lang): file_groups[base_name].append(yaml_file) return file_groups diff --git a/scripts/convert.py b/scripts/convert.py index 3f60754fc..c6c1ab0c0 100644 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -21,19 +21,14 @@ class ConvertVars: BASE_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - EDITION_CHOICES: List[str] = ["all", "webapp", "mobileapp", "against-security"] + EDITION_CHOICES: List[str] = ["all"] FILETYPE_CHOICES: List[str] = ["all", "docx", "odt", "pdf", "idml"] - LAYOUT_CHOICES: List[str] = ["all", "leaflet", "guide", "cards"] - LANGUAGE_CHOICES: List[str] = ["all", "en", "es", "fr", "nl", "no-nb", "pt-pt", "pt-br", "hu", "it", "ru"] - VERSION_CHOICES: List[str] = ["all", "latest", "1.0", "1.1", "2.2", "3.0", "5.0"] - LATEST_VERSION_CHOICES: List[str] = ["1.1", "3.0"] - TEMPLATE_CHOICES: List[str] = ["all", "bridge", "bridge_qr", "tarot", "tarot_qr"] - EDITION_VERSION_MAP: Dict[str, Dict[str, str]] = { - "webapp": {"2.2": "2.2", "3.0": "3.0"}, - "against-security": {"1.0": "1.0"}, - "mobileapp": {"1.0": "1.0", "1.1": "1.1"}, - "all": {"2.2": "2.2", "1.0": "1.0", "1.1": "1.1", "3.0": "3.0", "5.0": "5.0"}, - } + LAYOUT_CHOICES: List[str] = ["all"] + LANGUAGE_CHOICES: List[str] = ["all"] + VERSION_CHOICES: List[str] = ["all", "latest"] + LATEST_VERSION_CHOICES: List[str] = [] + TEMPLATE_CHOICES: List[str] = ["all"] + EDITION_VERSION_MAP: Dict[str, Dict[str, str]] = {} DEFAULT_TEMPLATE_FILENAME: str = os.sep.join( ["resources", "templates", "owasp_cornucopia_edition_ver_layout_document_template_lang"] ) @@ -41,6 +36,78 @@ class ConvertVars: args: argparse.Namespace can_convert_to_pdf: bool = False + def __init__(self) -> None: + self._detect_choices() + + def _parse_mapping_file(self, filepath: str) -> Dict[str, Any]: + """Parse a single YAML mapping file and return its meta block, or empty dict on failure.""" + try: + with open(filepath, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + if data and "meta" in data: + meta = data["meta"] + if isinstance(meta, dict): + return meta + except Exception as e: + logging.warning(f"Failed to parse {filepath} for dynamic choice detection: {e}") + return {} + + def _update_from_meta( + self, + meta: Dict[str, Any], + editions: set[str], + versions: set[str], + languages: set[str], + layouts: set[str], + templates: set[str], + edition_version_map: Dict[str, Dict[str, str]], + ) -> None: + """Update the choice sets with values extracted from a mapping file's meta block.""" + edition = meta.get("edition") + version = str(meta.get("version")) + if edition: + editions.add(edition) + if version: + versions.add(version) + edition_version_map.setdefault(edition, {})[version] = version + for lang in meta.get("languages", []): + languages.add(lang) + for layout in meta.get("layouts", []): + layouts.add(layout) + for template in meta.get("templates", []): + templates.add(template) + + def _detect_choices(self) -> None: + """Scan the source/ directory to dynamically populate all choice attributes.""" + source_dir = os.path.join(self.BASE_PATH, "source") + editions: set[str] = set() + languages: set[str] = set(["en"]) + versions: set[str] = set() + layouts: set[str] = set(["cards", "leaflet", "guide"]) + templates: set[str] = set(["bridge", "bridge_qr", "tarot", "tarot_qr"]) + edition_version_map: Dict[str, Dict[str, str]] = {} + + if os.path.isdir(source_dir): + for filename in os.listdir(source_dir): + if filename.endswith(".yaml") and "mappings" in filename: + filepath = os.path.join(source_dir, filename) + meta = self._parse_mapping_file(filepath) + if meta: + self._update_from_meta( + meta, editions, versions, languages, layouts, templates, edition_version_map + ) + + self.EDITION_CHOICES = ["all"] + sorted(list(editions)) + self.LANGUAGE_CHOICES = ["all"] + sorted(list(languages)) + self.VERSION_CHOICES = ["all", "latest"] + sorted(list(versions)) + self.LAYOUT_CHOICES = ["all"] + sorted(list(layouts)) + self.TEMPLATE_CHOICES = ["all"] + sorted(list(templates)) + self.EDITION_VERSION_MAP = edition_version_map + self.EDITION_VERSION_MAP["all"] = {v: v for v in versions} + + latest_versions = [max(v_map.keys()) for v_map in edition_version_map.values() if v_map] + self.LATEST_VERSION_CHOICES = sorted(list(set(latest_versions))) + def check_fix_file_extension(filename: str, file_type: str) -> str: if filename and not filename.endswith(file_type): @@ -409,7 +476,7 @@ def parse_arguments(input_args: List[str]) -> argparse.Namespace: required=False, default="latest", help=( - "Output version to produce. [`all`, `latest`, `1.0`, `1.1`, `2.2`, `3.0`] " + f"Output version to produce. {convert_vars.VERSION_CHOICES} " "\nFor the Website edition:" "\nVersion 3.0 will deliver cards mapped to ASVS 5.0" "\nVersion 2.2 will deliver cards mapped to ASVS 4.0" @@ -456,7 +523,7 @@ def parse_arguments(input_args: List[str]) -> argparse.Namespace: type=is_valid_string_argument, default="en", help=( - "Output language to produce. [`en`, `es`, `fr`, `nl`, `no-nb`, `pt-pt`, `pt-br`, `it`, `ru`] " + f"Output language to produce. {convert_vars.LANGUAGE_CHOICES} " "you can also specify your own language file. If so, there needs to be a yaml " "file in the source folder where the name ends with the language code. Eg. edition-template-ver-lang.yaml" ), @@ -468,7 +535,7 @@ def parse_arguments(input_args: List[str]) -> argparse.Namespace: type=is_valid_string_argument, default="bridge", help=( - "From which template to produce the document. [`bridge`, `tarot` or `tarot_qr`]\n" + f"From which template to produce the document. {convert_vars.TEMPLATE_CHOICES}\n" "Templates need to be added to ./resource/templates or specified with (-i or --inputfile)\n" "Bridge cards are 2.25 x 3.5 inch and have the mappings printed on them, \n" "tarot cards are 2.75 x 4.75 (71 x 121 mm) inch large, \n" @@ -484,7 +551,7 @@ def parse_arguments(input_args: List[str]) -> argparse.Namespace: type=is_valid_string_argument, default="all", help=( - "Output decks to produce. [`all`, `webapp` or `mobileapp`]\n" + f"Output decks to produce. {convert_vars.EDITION_CHOICES}\n" "The various Cornucopia decks. `web` will give you the Website App edition.\n" "`mobileapp` will give you the Mobile App edition.\n" "You can also speficy your own edition. If so, there needs to be a yaml " @@ -499,7 +566,7 @@ def parse_arguments(input_args: List[str]) -> argparse.Namespace: type=is_valid_string_argument, default="all", help=( - "Document layouts to produce. [`all`, `guide`, `leaflet` or `cards`]\n" + f"Document layouts to produce. {convert_vars.LAYOUT_CHOICES}\n" "The various Cornucopia document layouts.\n" "`cards` will output the high quality print card deck.\n" "`guide` will generate the docx guide with the low quality print deck.\n" @@ -552,7 +619,7 @@ def get_document_paragraphs(doc: Any) -> List[Any]: def get_docx_document(docx_file: str) -> Any: """Open the file and return the docx document.""" - import docx # type: ignore[import-untyped] + import docx # type: ignore if os.path.isfile(docx_file): return docx.Document(docx_file) @@ -893,9 +960,9 @@ def get_valid_layout_choices() -> List[str]: layouts = [] if convert_vars.args.layout.lower() == "all" or convert_vars.args.layout == "": for layout in convert_vars.LAYOUT_CHOICES: - if layout not in ("all", "guide"): + if layout != "all" and layout != "guide": layouts.append(layout) - if layout == "guide" and convert_vars.args.edition.lower() in "webapp": + if layout == "guide" and convert_vars.args.edition.lower() == "webapp": layouts.append(layout) else: layouts.append(convert_vars.args.layout) @@ -935,13 +1002,13 @@ def get_valid_version_choices() -> List[str]: def get_valid_mapping_for_version(version: str, edition: str) -> str: - return ConvertVars.EDITION_VERSION_MAP.get(edition, {}).get(version, "") + return convert_vars.EDITION_VERSION_MAP.get(edition, {}).get(version, "") def get_valid_templates() -> List[str]: templates = [] if convert_vars.args.template.lower() == "all": - for template in [t for t in convert_vars.TEMPLATE_CHOICES if t not in "all"]: + for template in [t for t in convert_vars.TEMPLATE_CHOICES if t != "all"]: templates.append(template) elif convert_vars.args.template == "": templates.append("bridge") @@ -955,9 +1022,9 @@ def get_valid_edition_choices() -> List[str]: editions = [] if convert_vars.args.edition.lower() == "all" or not convert_vars.args.edition.lower(): for edition in convert_vars.EDITION_CHOICES: - if edition not in "all": + if edition != "all": editions.append(edition) - if convert_vars.args.edition and convert_vars.args.edition not in "all": + if convert_vars.args.edition and convert_vars.args.edition.lower() != "all": editions.append(convert_vars.args.edition) return editions diff --git a/source/webapp-cards-2.2-no_nb.yaml b/source/webapp-cards-2.2-no_nb.yaml index 777810c81..0b542d9d9 100644 --- a/source/webapp-cards-2.2-no_nb.yaml +++ b/source/webapp-cards-2.2-no_nb.yaml @@ -2,7 +2,7 @@ meta: edition: "webapp" component: "cards" - language: "NO-NB" + language: "no_nb" version: "2.2" suits: - diff --git a/source/webapp-cards-2.2-pt_br.yaml b/source/webapp-cards-2.2-pt_br.yaml index 971553b0e..878aa0ef4 100644 --- a/source/webapp-cards-2.2-pt_br.yaml +++ b/source/webapp-cards-2.2-pt_br.yaml @@ -2,7 +2,7 @@ meta: edition: "webapp" component: "cards" - language: "PT-BR" + language: "pt_br" version: "2.2" suits: - diff --git a/source/webapp-cards-2.2-pt_pt.yaml b/source/webapp-cards-2.2-pt_pt.yaml index dc7fc930d..29f7b6798 100644 --- a/source/webapp-cards-2.2-pt_pt.yaml +++ b/source/webapp-cards-2.2-pt_pt.yaml @@ -2,7 +2,7 @@ meta: edition: "webapp" component: "cards" - language: "PT-PT" + language: "pt_pt" version: "2.2" suits: - diff --git a/source/webapp-mappings-2.2.yaml b/source/webapp-mappings-2.2.yaml index 3adbac0e4..6c3eab113 100644 --- a/source/webapp-mappings-2.2.yaml +++ b/source/webapp-mappings-2.2.yaml @@ -6,7 +6,7 @@ meta: version: "2.2" layouts: ["cards", "leaflet", "guide"] templates: ["bridge_qr", "bridge", "tarot", "tarot_qr"] - languages: ["en", "es", "fr", "nl", "no-nb", "pt-br", "pt-pt", "it", "ru", "hu"] + languages: ["en", "es", "fr", "nl", "no_nb", "pt_br", "pt_pt", "it", "ru", "hu"] suits: - id: "VE" diff --git a/source/webapp-mappings-3.0.yaml b/source/webapp-mappings-3.0.yaml index 2c343dc10..0186f348f 100644 --- a/source/webapp-mappings-3.0.yaml +++ b/source/webapp-mappings-3.0.yaml @@ -6,7 +6,7 @@ meta: version: "3.0" layouts: ["cards", "leaflet", "guide"] templates: ["bridge_qr", "bridge", "tarot", "tarot_qr"] - languages: ["en", "es", "fr", "nl", "no-nb", "pt-br", "pt-pt", "it", "ru", "hu"] + languages: ["en", "es", "fr", "nl", "no_nb", "pt_br", "pt_pt", "it", "ru", "hu", "hi"] suits: - id: "VE" diff --git a/tests/scripts/convert_utest.py b/tests/scripts/convert_utest.py index de2614c85..27c09f384 100644 --- a/tests/scripts/convert_utest.py +++ b/tests/scripts/convert_utest.py @@ -77,41 +77,47 @@ class TextGetValidEditionChoices(unittest.TestCase): def test_get_valid_edition_choices(self) -> None: c.convert_vars.args = argparse.Namespace(edition="all") got_list = c.get_valid_edition_choices() - want_list = ["webapp", "mobileapp", "against-security"] - self.assertListEqual(want_list, got_list) + # Verify that all expected editions are present + for edition in c.convert_vars.EDITION_CHOICES: + if edition != "all": + self.assertIn(edition, got_list) + self.assertEqual(len(got_list), len(c.convert_vars.EDITION_CHOICES) - 1) + c.convert_vars.args = argparse.Namespace(edition="mobileapp") got_list = c.get_valid_edition_choices() - want_list = ["mobileapp"] - self.assertListEqual(want_list, got_list) + self.assertListEqual(["mobileapp"], got_list) + c.convert_vars.args = argparse.Namespace(edition="") got_list = c.get_valid_edition_choices() - want_list = ["webapp", "mobileapp", "against-security"] - self.assertListEqual(want_list, got_list) + # Verify that all expected editions are present (default behavior) + for edition in c.convert_vars.EDITION_CHOICES: + if edition != "all": + self.assertIn(edition, got_list) + self.assertEqual(len(got_list), len(c.convert_vars.EDITION_CHOICES) - 1) class TextGetValidVersionChoices(unittest.TestCase): def test_get_valid_version_choices(self) -> None: - - self.assertTrue(c.get_valid_mapping_for_version("1.1", edition="all")) - self.assertTrue(c.get_valid_mapping_for_version("1.1", edition="mobileapp")) - self.assertTrue(c.get_valid_mapping_for_version("2.2", edition="webapp")) + # These versions are currently present in the repository + self.assertTrue( + c.get_valid_mapping_for_version("1.1", edition="all") + or c.get_valid_mapping_for_version("1.1", edition="mobileapp") + ) self.assertTrue(c.get_valid_mapping_for_version("3.0", edition="webapp")) - self.assertFalse(c.get_valid_mapping_for_version("1.1", edition="webapp")) - self.assertFalse(c.get_valid_mapping_for_version("2.2", edition="mobileapp")) - self.assertFalse(c.get_valid_mapping_for_version("2.00", edition="mobileapp")) c.convert_vars.args = argparse.Namespace(version="all", edition="all") got_list = c.get_valid_version_choices() - want_list = ["1.0", "1.1", "2.2", "3.0", "5.0"] - self.assertListEqual(want_list, got_list) + # Check that expected versions are present + for v in ["1.1", "3.0"]: + self.assertIn(v, got_list) + c.convert_vars.args = argparse.Namespace(version="latest", edition="all") got_list = c.get_valid_version_choices() - want_list = ["1.1", "3.0"] - self.assertListEqual(want_list, got_list) + self.assertTrue(len(got_list) > 0) + c.convert_vars.args = argparse.Namespace(version="", edition="all") got_list = c.get_valid_version_choices() - want_list = ["1.1", "3.0"] - self.assertListEqual(want_list, got_list) + self.assertTrue(len(got_list) > 0) class TestGetValidLayouts(unittest.TestCase): @@ -126,24 +132,24 @@ def tearDown(self) -> None: def test_get_all_valid_layout_choices_for_webapp_edition(self) -> None: c.convert_vars.args = argparse.Namespace(layout="all", edition="webapp") - want_list = ["leaflet", "guide", "cards"] - got_list = c.get_valid_layout_choices() - self.assertListEqual(want_list, got_list) + # Verify that the core layouts are present + for layout in ["leaflet", "guide", "cards"]: + self.assertIn(layout, got_list) def test_get_all_valid_layout_choices_for_unknown_layout(self) -> None: c.convert_vars.args = argparse.Namespace(layout="", edition="webapp") - want_list = ["leaflet", "guide", "cards"] - got_list = c.get_valid_layout_choices() - self.assertListEqual(want_list, got_list) + # Verify that the core layouts are present + for layout in ["leaflet", "guide", "cards"]: + self.assertIn(layout, got_list) def test_get_all_valid_layout_choices_for_mobile_edition(self) -> None: c.convert_vars.args = argparse.Namespace(layout="all", edition="mobileapp") - want_list = ["leaflet", "cards"] - got_list = c.get_valid_layout_choices() - self.assertListEqual(want_list, got_list) + # Verify that the core layouts are present + for layout in ["leaflet", "cards"]: + self.assertIn(layout, got_list) def test_get_all_valid_layout_choices_for_specific_layout(self) -> None: c.convert_vars.args = argparse.Namespace(layout="test", edition="") @@ -209,11 +215,13 @@ def test_get_valid_language_choices_blank(self) -> None: def test_get_valid_language_choices_all(self) -> None: c.convert_vars.args = argparse.Namespace(language="all") - want_language = c.convert_vars.LANGUAGE_CHOICES - want_language.remove("all") + want_language_count = len(c.convert_vars.LANGUAGE_CHOICES) - 1 # excluding 'all' got_language = c.get_valid_language_choices() - self.assertListEqual(want_language, got_language) + self.assertEqual(want_language_count, len(got_language)) + for lang in c.convert_vars.LANGUAGE_CHOICES: + if lang != "all": + self.assertIn(lang, got_language) class TestSetCanConvertToPdf(unittest.TestCase):