fix: fix issue in map_to_canonical_id function (#332)

Solves #331

Author: raphael0202

openfoodfacts/taxonomy.py

@@ -3,7 +3,7 @@
 
 import requests
 
-from openfoodfacts.utils.text import get_tag
+from openfoodfacts.utils.text import get_tag, replace_lang_prefix
 
 from .types import Environment, Flavor, JSONType, TaxonomyType
 from .utils import (
@@ -495,5 +495,18 @@ def map_to_canonical_id(
             raise ValueError(
                 f"Invalid value: '{value}', expected value to be in 'lang:tag' format"
             )
-    tags = [get_tag(value) for value in values]
-    return {value: taxonomy_mapping.get(tag, tag) for tag, value in zip(tags, values)}
+
+    output = {}
+    for value in values:
+        tag = get_tag(value)
+        output[value] = (
+            # Look for a direct match first
+            taxonomy_mapping.get(tag)
+            # Then look for a match with the xx prefix (language-independent
+            # entry)
+            or taxonomy_mapping.get(replace_lang_prefix(tag, "xx"))
+            # If no match is found, return the original taggified value
+            or tag
+        )
+
+    return output

openfoodfacts/utils/text.py

@@ -90,3 +90,17 @@ def get_tag(text: str) -> str:
     if lang_prefix:
         text = f"{lang_prefix}:{text}"
     return text
+
+
+def replace_lang_prefix(tag: str, new_lang_prefix: str) -> str:
+    """Replace the language prefix of a tag with a new one."""
+
+    if len(new_lang_prefix) != 2:
+        raise ValueError(
+            f"new_lang_prefix '{new_lang_prefix}' must be a 2-letter code."
+        )
+
+    if len(tag) < 3 or tag[2] != ":":
+        raise ValueError(f"tag '{tag}' has an invalid language prefix")
+
+    return f"{new_lang_prefix}:{tag[3:]}"

tests/unit/test_taxonomy.py

@@ -21,20 +21,28 @@ def test_map_to_canonical_id():
         "en:apples": "en:apples",
         "fr:pomme": "en:apples",
         "fr:noix-d-isere": "en:nuts-from-isere",
+        "xx:provence-alpes-cote-d-azur": "en:provence-alpes-cote-d-azur",
+        "xx:sashimi": "xx:sashimi",
     }
     values = [
         "en: Apple",
         "en: apples",
         "fr: Pomme",
         "fr: Bananes d'Isère",
         "fr: Noix d'Isère",
+        "fr: Provence-Alpes-Côte d'Azur",
+        "pt: Provence-Alpes-Côte d'Azur",
+        "it: sashimi",
     ]
     expected = {
         "en: Apple": "en:apples",
         "en: apples": "en:apples",
         "fr: Pomme": "en:apples",
         "fr: Bananes d'Isère": "fr:bananes-d-isere",
         "fr: Noix d'Isère": "en:nuts-from-isere",
+        "fr: Provence-Alpes-Côte d'Azur": "en:provence-alpes-cote-d-azur",
+        "pt: Provence-Alpes-Côte d'Azur": "en:provence-alpes-cote-d-azur",
+        "it: sashimi": "xx:sashimi",
     }
     assert map_to_canonical_id(taxonomy_mapping, values) == expected
 
@@ -70,15 +78,22 @@ def test_basic(self):
             names={"fr": "Noix d'Isère"},
             synonyms={"fr": ["Noix d'Isère"]},
         )
+        node3 = TaxonomyNode(
+            identifier="xx:sashimi",
+            names={"xx": "Sashimi"},
+            synonyms={"xx": ["Sashimi"]},
+        )
         taxonomy.add(node1.id, node1)
         taxonomy.add(node2.id, node2)
+        taxonomy.add(node3.id, node3)
 
         expected_mapping = {
             "en:apple": "en:apples",
             "fr:pomme": "en:apples",
             "en:apples": "en:apples",
             "fr:pommes": "en:apples",
             "fr:noix-d-isere": "en:nuts-from-isere",
+            "xx:sashimi": "xx:sashimi",
         }
 
         assert create_taxonomy_mapping(taxonomy) == expected_mapping

tests/unit/utils/test_text.py

@@ -1,6 +1,6 @@
 import pytest
 
-from openfoodfacts.utils.text import get_tag
+from openfoodfacts.utils.text import get_tag, replace_lang_prefix
 
 
 @pytest.mark.parametrize(
@@ -22,3 +22,30 @@
 )
 def test_get_tag(value: str, output: str):
     assert get_tag(value) == output
+
+
+@pytest.mark.parametrize(
+    "tag,new_lang_prefix,output",
+    [
+        ("fr:gesiers", "en", "en:gesiers"),
+        ("fr:gesiers", "fr", "fr:gesiers"),
+        ("fr:gesiers", "ar", "ar:gesiers"),
+        ("en:apple", "fr", "fr:apple"),
+        ("xx:sashimi", "it", "it:sashimi"),
+        ("xx:sashimi", "xx", "xx:sashimi"),
+    ],
+)
+def test_replace_lang_prefix(tag, new_lang_prefix, output):
+    assert replace_lang_prefix(tag, new_lang_prefix) == output
+
+
+def test_replace_lang_prefix_invalid_new_lang_prefix():
+    with pytest.raises(ValueError, match="new_lang_prefix 'a' must be a 2-letter code"):
+        replace_lang_prefix("en:apples", "a")
+
+
+def test_replace_lang_prefix_invalid_tag():
+    with pytest.raises(
+        ValueError, match="tag 'e:apples' has an invalid language prefix"
+    ):
+        replace_lang_prefix("e:apples", "fr")