Drop unused functionality for function based converters

Author: ivorbosloper

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 
 - Improve Converter template.py usability
+- Remove functionality for function based converters
 
 ## [v0.10.0] - 2025-03-11
 

README.md

@@ -189,8 +189,8 @@ Install the pre-commit hook with `pre-commit install`, so you never commit incor
 
 The following high-level description gives an idea how to implement a converter in fiboa CLI:
 
-1. Create a new file in `fiboa_cli/datasets` based on the `template.py`
-2. Implement the `convert()` function / test it / run it
+1. Create a new file in `fiboa_cli/datasets/your_file.py` based on the `template.py`
+2. Fill in the required variables / test it / run it
 3. Add missing dependencies into a separate dependency group in `setup.py`
 4. Add the converter to the list above
 5. Create a PR to submit your converter for review

fiboa_cli/__init__.py

@@ -525,11 +525,11 @@ def converters(providers, sources, verbose):
     """
     log(f"fiboa CLI {__version__} - List of Converters\n", "success")
 
-    columns = {"SHORT_NAME": "Short Title", "LICENSE": "License"}
+    columns = {"short_name": "Short Title", "license": "License"}
     if providers:
-        columns["PROVIDERS"] = "Provider(s)"
+        columns["providers"] = "Provider(s)"
     if sources:
-        columns["SOURCES"] = "Source(s)"
+        columns["sources"] = "Source(s)"
 
     keys = list(columns.keys())
     converters = list_all_converters(keys)

fiboa_cli/convert.py

@@ -62,24 +62,15 @@ def list_all_converters(keys):
         obj = {}
         try:
             converter = read_converter(id)
-            # todo remove this if once class-based converters have been fully implemented
-            if isinstance(converter, BaseConverter):
-                assert converter.__class__.__name__ != "TemplateConverter", (
-                    f"Please change TemplateConverter for {id}"
-                )
 
             for key in keys:
-                # todo remove this if once class-based converters have been fully implemented
-                if isinstance(converter, BaseConverter):
-                    value = getattr(converter, key.lower())
-                else:
-                    value = getattr(converter, key, None)
+                value = getattr(converter, key.lower())
 
-                if key == "SOURCES" and isinstance(value, dict):
+                if key == "sources" and isinstance(value, dict):
                     value = ", ".join(list(value.keys()))
-                elif key == "LICENSE" and isinstance(value, dict):
+                elif key == "license" and isinstance(value, dict):
                     value = value["href"]
-                elif key == "PROVIDERS" and isinstance(value, list):
+                elif key == "providers" and isinstance(value, list):
                     value = ", ".join(list(map(lambda x: x["name"], value)))
 
                 obj[key] = value
@@ -93,17 +84,15 @@ def list_all_converters(keys):
 def read_converter(_id):
     module_name = f".datasets.{_id}"
     module = importlib.import_module(module_name, package="fiboa_cli")
-    # todo: remove conditional once class-based converters have been fully implemented
-    if not hasattr(module, "convert"):
-        try:
-            clazz = next(
-                v
-                for v in module.__dict__.values()
-                if type(v) is type
-                and issubclass(v, BaseConverter)
-                and "BaseConverter" not in v.__name__
-            )
-            return clazz()
-        except StopIteration:
-            log("Missing convert function or Converter class for module {_id}", "warning")
-    return module
+    try:
+        clazz = next(
+            v
+            for v in module.__dict__.values()
+            if type(v) is type
+            and issubclass(v, BaseConverter)
+            and "BaseConverter" not in v.__name__
+        )
+        return clazz()
+    except StopIteration:
+        log(f"Missing Converter class for module {_id}", "error")
+        raise ImportError(f"Missing Converter class for module {_id}")

fiboa_cli/convert_utils.py

@@ -28,73 +28,6 @@
 from .version import fiboa_version
 
 
-def convert(
-    output_file,
-    cache,
-    urls,
-    column_map,
-    id,
-    title,
-    description,
-    input_files=None,
-    bbox=None,
-    providers=[],
-    source_coop_url=None,
-    extensions=set(),
-    missing_schemas={},
-    column_additions={},
-    column_filters={},
-    column_migrations={},
-    migration=None,
-    file_migration=None,
-    layer_filter=None,
-    attribution=None,
-    store_collection=False,
-    license=None,
-    compression=None,
-    geoparquet1=False,
-    original_geometries=False,
-    index_as_id=False,
-    year=None,  # noqa unused
-    **kwargs,
-):
-    # this function is a (temporary) bridge from function-based converters to class-based converters
-    # todo: this convert-function should be removed once class-based converters have been fully implemented
-
-    converter = BaseConverter(
-        sources=urls,
-        columns=column_map,
-        id=id,
-        title=title,
-        description=description,
-        bbox=bbox,
-        providers=providers,
-        short_name=id,
-        extensions=extensions,
-        missing_schemas=missing_schemas,
-        column_additions=column_additions,
-        column_filters=column_filters,
-        column_migrations=column_migrations,
-        migrate=migration,
-        file_migration=file_migration,
-        layer_filter=layer_filter,
-        attribution=attribution,
-        license=license,
-        index_as_id=index_as_id,
-    )
-    converter.convert(
-        output_file,
-        cache,
-        input_files=input_files,
-        source_coop_url=source_coop_url,
-        store_collection=store_collection,
-        compression=compression,
-        geoparquet1=geoparquet1,
-        original_geometries=original_geometries,
-        **kwargs,
-    )
-
-
 def add_asset_to_collection(collection, output_file, rows=None, columns=None):
     c = collection.copy()
     if "assets" not in c or not isinstance(c["assets"], dict):
@@ -184,11 +117,7 @@ class BaseConverter:
 
     index_as_id = False
 
-    def __init__(self, **kwargs):
-        # This init method allows you to override all properties & methods
-        # It's a bit hacky but allows a gradual conversion from function-based to class-based converters
-        # todo remove this once class-based converters have been fully implemented
-        self.__dict__.update({k: v for k, v in kwargs.items() if v is not None})
+    def __init__(self):
         for key in ("id", "short_name", "title", "license", "columns"):
             assert getattr(self, key) is not None, (
                 f"{inspect.getfile(self.__class__)}:{self.__class__.__name__} misses required attribute {key}"
@@ -200,10 +129,6 @@ def __init__(self, **kwargs):
             if not key.startswith("_") and isinstance(item, (list, dict, set)):
                 setattr(self, key, copy(item))
 
-    @property
-    def ID(self):  # noqa backwards compatibility for function-based converters
-        return self.id
-
     def migrate(self, gdf) -> gpd.GeoDataFrame:
         return gdf
 

fiboa_cli/datasets/commons/ec.py

@@ -5,73 +5,57 @@
 
 from fiboa_cli.util import load_file
 
-from .dictobject import DictObject
 
+class EuroCropsConverterMixin:
+    license = "CC-BY-SA-4.0"
 
-def add_eurocrops(base, year=None):
-    if isinstance(base, dict):
-        base = DictObject(base)
-
-    ID = base.ID
-    if not base.ID.startswith("ec_"):
-        ID = "ec_" + ID
-
-    SUFFIX = " - Eurocrops"
-    if year is not None:
-        SUFFIX += " " + str(year)
-
-    TITLE = base.TITLE + SUFFIX
-
-    SHORT_NAME = base.SHORT_NAME + SUFFIX
-
-    DESCRIPTION = (
-        base.DESCRIPTION.strip()
-        + """
-
-This dataset is an extended version of the original dataset, with additional columns and attributes added by the EuroCrops project.
-
-The project developed a new **Hierarchical Crop and Agriculture Taxonomy (HCAT)** that harmonises all declared crops across the European Union.
-In the data you'll find this as additional attributes:
-
-- `EC_trans_n`: The original crop name translated into English
-- `EC_hcat_n`: The machine-readable HCAT name of the crop
-- `EC_hcat_c`: The 10-digit HCAT code indicating the hierarchy of the crop
-    """
-    )
+    ec_mapping_csv = None
+    mapping_file = None
+    ec_mapping = None
 
-    PROVIDERS = base.PROVIDERS + [
-        {"name": "EuroCrops", "url": "https://github.com/maja601/EuroCrops", "roles": ["processor"]}
-    ]
+    def __init__(self, *args, year=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not self.id.startswith("ec_"):
+            self.id = "ec_" + self.id
 
-    EXTENSIONS = getattr(base, "EXTENSIONS", None) or set()
-    EXTENSIONS.add("https://fiboa.github.io/hcat-extension/v0.1.0/schema.yaml")
-    COLUMNS = base.COLUMNS | {
-        "EC_trans_n": "ec:translated_name",
-        "EC_hcat_n": "ec:hcat_name",
-        "EC_hcat_c": "ec:hcat_code",
-    }
+        suffix = " - Eurocrops"
+        if year is not None:
+            suffix = f"{suffix} {year}"
 
-    LICENSE = "CC-BY-SA-4.0"
+        self.title += suffix
+        self.short_name += suffix
 
-    return ID, SHORT_NAME, TITLE, DESCRIPTION, PROVIDERS, EXTENSIONS, COLUMNS, LICENSE
+        self.description = (
+            self.description.strip()
+            + """
 
+    This dataset is an extended version of the original dataset, with additional columns and attributes added by the EuroCrops project.
 
-class EuroCropsConverterMixin:
-    ec_mapping_csv = None
-    mapping_file = None
-    ec_mapping = None
+    The project developed a new **Hierarchical Crop and Agriculture Taxonomy (HCAT)** that harmonises all declared crops across the European Union.
+    In the data you'll find this as additional attributes:
 
-    def __init__(self, *args, year=None, **kwargs):
-        super().__init__(*args, **kwargs)
-        # Some meta-magic to reuse existing add_eurocrops routine
-        attributes = (
-            "ID, SHORT_NAME, TITLE, DESCRIPTION, PROVIDERS, EXTENSIONS, COLUMNS, LICENSE".split(
-                ", "
-            )
+    - `EC_trans_n`: The original crop name translated into English
+    - `EC_hcat_n`: The machine-readable HCAT name of the crop
+    - `EC_hcat_c`: The 10-digit HCAT code indicating the hierarchy of the crop
+        """
         )
-        base = {k: getattr(self, k.lower()) for k in attributes}
-        for k, v in zip(attributes, add_eurocrops(base, year=year)):
-            setattr(self, k.lower(), v)
+
+        self.providers += [
+            {
+                "name": "EuroCrops",
+                "url": "https://github.com/maja601/EuroCrops",
+                "roles": ["processor"],
+            }
+        ]
+
+        self.extensions = getattr(self, "extensions", set())
+        self.extensions.add("https://fiboa.github.io/hcat-extension/v0.1.0/schema.yaml")
+        self.columns |= {
+            "EC_trans_n": "ec:translated_name",
+            "EC_hcat_n": "ec:hcat_name",
+            "EC_hcat_c": "ec:hcat_code",
+        }
+        self.license = "CC-BY-SA-4.0"
 
     def convert(self, *args, **kwargs):
         self.mapping_file = kwargs.get("mapping_file")

fiboa_cli/datasets/ec_lv.py

@@ -42,15 +42,15 @@ class Converter(EuroCropsConverterMixin, BaseConverter):
         "required": [
             "year",
             "parcel_id",
-            "crop_id",
+            "crop:code",
             "subsidy_type",
             "EC_NUTS3",
             #       'PRODUCT_DE',
         ],
         "properties": {
             "year": {"type": "uint16", "minLength": 4, "maxLength": 4},
             "parcel_id": {"type": "uint64", "minLength": 8, "maxLength": 8},
-            "crop_id": {"type": "uint16", "minLength": 3, "maxLength": 3},
+            "crop:code": {"type": "uint16", "minLength": 3, "maxLength": 3},
             "subsidy_type": {"type": "string"},
             "EC_NUTS3": {
                 "type": "string",
@@ -63,3 +63,7 @@ class Converter(EuroCropsConverterMixin, BaseConverter):
             # },
         },
     }
+
+    def add_hcat(self, gdf):
+        # skip adding hcat
+        return gdf

fiboa_cli/datasets/ec_si.py

@@ -3,6 +3,7 @@
 
 
 class Converter(EuroCropsConverterMixin, BaseConverter):
+    ec_mapping_csv = "si_2021.csv"
     sources = {
         "https://zenodo.org/records/10118572/files/SI_2021.zip?download=1": ["SI_2021_EC21.shp"]
     }
@@ -46,3 +47,7 @@ class Converter(EuroCropsConverterMixin, BaseConverter):
             "EC_NUTS3": {"type": "string"},
         },
     }
+
+    def add_hcat(self, gdf):
+        # skip adding hcat
+        return gdf