Skip to content

Commit 951a0c1

Browse files
ivorbosloperivorbosloper
committed
Refactor all function based converters
1 parent ed7db2c commit 951a0c1

File tree

9 files changed

+381
-503
lines changed

9 files changed

+381
-503
lines changed

‎CHANGELOG.md

+4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
77

88
## [Unreleased]
99

10+
### Added
11+
12+
### Changed
13+
1014
- Improve Converter template.py usability
1115

1216
## [v0.10.0] - 2025-03-11

‎fiboa_cli/datasets/ai4sf.py

+115-137
Large diffs are not rendered by default.

‎fiboa_cli/datasets/ec_be_vlg.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
from .be_vlg import Converter as base
1+
from .be_vlg import Converter as BEVLGBaseConverter
22
from .commons.ec import EuroCropsConverterMixin
33

44

5-
class ECConverter(EuroCropsConverterMixin, base):
5+
class ECConverter(EuroCropsConverterMixin, BEVLGBaseConverter):
66
sources = {
77
"https://zenodo.org/records/10118572/files/BE_VLG_2021.zip?download=1": [
88
"BE_VLG_2021/BE_VLG_2021_EC21.shp"

‎fiboa_cli/datasets/ec_ee.py

+54-75
Original file line numberDiff line numberDiff line change
@@ -1,83 +1,62 @@
11
import pandas as pd
22

3-
from ..convert_utils import convert as convert_
4-
from .commons.ec import add_eurocrops
3+
from ..convert_utils import BaseConverter
4+
from .commons.ec import EuroCropsConverterMixin
55

66
# todo: The dataset doesn't validate due to a self intersecting polygon
77
# How do we want to handle this?
88

9-
SOURCES = "https://zenodo.org/records/8229128/files/EE_2021.zip?download=1"
109

11-
ID = "ec_ee"
12-
SHORT_NAME = "Estonia"
13-
TITLE = "Field boundaries for Estonia"
14-
DESCRIPTION = """
15-
Geospatial Aid Application Estonia Agricultural parcels.
16-
The original dataset is provided by ARIB and obtained from the INSPIRE theme GSAA (specifically Geospaial Aid Application Estonia Agricultural parcels) through which the data layer Fields and Eco Areas (GSAA) is made available.
17-
The data comes from ARIB's database of agricultural parcels.
18-
"""
19-
PROVIDERS = [
20-
{
21-
"name": "Põllumajanduse Registrite ja Informatsiooni Amet",
22-
"url": "http://data.europa.eu/88u/dataset/pria-pollud",
23-
"roles": ["producer", "licensor"],
10+
class Convert(EuroCropsConverterMixin, BaseConverter):
11+
ec_mapping_csv = "ee_2021.csv"
12+
sources = "https://zenodo.org/records/8229128/files/EE_2021.zip?download=1"
13+
id = "ec_ee"
14+
short_name = "Estonia"
15+
title = "Field boundaries for Estonia"
16+
description = """
17+
Geospatial Aid Application Estonia Agricultural parcels.
18+
The original dataset is provided by ARIB and obtained from the INSPIRE theme GSAA (specifically Geospaial Aid Application Estonia Agricultural parcels) through which the data layer Fields and Eco Areas (GSAA) is made available.
19+
The data comes from ARIB's database of agricultural parcels.
20+
"""
21+
providers = [
22+
{
23+
"name": "Põllumajanduse Registrite ja Informatsiooni Amet",
24+
"url": "http://data.europa.eu/88u/dataset/pria-pollud",
25+
"roles": ["producer", "licensor"],
26+
}
27+
]
28+
attribution = "© Põllumajanduse Registrite ja Informatsiooni Amet"
29+
30+
columns = {
31+
"geometry": "geometry",
32+
"pollu_id": "id",
33+
"taotlusaas": "determination_datetime", # year
34+
"pindala_ha": "area", # area (in ha)
35+
"taotletud_": "crop:code", # requested crop culture
36+
"taotletu_1": "taotletud_maakasutus", # requested land use
37+
"taotletu_2": "taotletud_toetus", # requested support
38+
"niitmise_t": "niitmise_tuvastamise_staatus", # mowing detection status
39+
"niitmise_1": "niitmise_tuvast_ajavahemik", # mowing detection period
40+
"viimase_mu": "viimase_muutmise_aeg", # Last edit time (date-date)
41+
"taotleja_n": "taotleja_nimi", # name of applicant
42+
"taotleja_r": "taotleja_registrikood", # applicant's registration code
43+
}
44+
column_migrations = {"JAHR": lambda col: pd.to_datetime(col, format="%Y")}
45+
missing_schemas = {
46+
"required": [
47+
"taotletud_kultuur",
48+
"taotletud_maakasutus",
49+
"viimase_muutmise_aeg",
50+
"taotleja_nimi",
51+
],
52+
"properties": {
53+
"taotletud_kultuur": {"type": "string"},
54+
"taotletud_maakasutus": {"type": "string"},
55+
"niitmise_tuvastamise_staatus": {"type": "string"},
56+
"niitmise_tuvast_ajavahemik": {"type": "string"},
57+
"viimase_muutmise_aeg": {"type": "string"},
58+
"taotletud_toetus": {"type": "string"},
59+
"taotleja_nimi": {"type": "string"},
60+
"taotleja_registrikood": {"type": "string"},
61+
},
2462
}
25-
]
26-
ATTRIBUTION = "© Põllumajanduse Registrite ja Informatsiooni Amet"
27-
28-
COLUMNS = {
29-
"geometry": "geometry",
30-
"pollu_id": "id",
31-
"taotlusaas": "determination_datetime", # year
32-
"pindala_ha": "area", # area (in ha)
33-
"taotletud_": "taotletud_kultuur", # requested crop culture
34-
"taotletu_1": "taotletud_maakasutus", # requested land use
35-
"taotletu_2": "taotletud_toetus", # requested support
36-
"niitmise_t": "niitmise_tuvastamise_staatus", # mowing detection status
37-
"niitmise_1": "niitmise_tuvast_ajavahemik", # mowing detection period
38-
"viimase_mu": "viimase_muutmise_aeg", # Last edit time (date-date)
39-
"taotleja_n": "taotleja_nimi", # name of applicant
40-
"taotleja_r": "taotleja_registrikood", # applicant's registration code
41-
}
42-
COLUMN_MIGRATIONS = {"JAHR": lambda col: pd.to_datetime(col, format="%Y")}
43-
MISSING_SCHEMAS = {
44-
"required": [
45-
"taotletud_kultuur",
46-
"taotletud_maakasutus",
47-
"viimase_muutmise_aeg",
48-
"taotleja_nimi",
49-
],
50-
"properties": {
51-
"taotletud_kultuur": {"type": "string"},
52-
"taotletud_maakasutus": {"type": "string"},
53-
"niitmise_tuvastamise_staatus": {"type": "string"},
54-
"niitmise_tuvast_ajavahemik": {"type": "string"},
55-
"viimase_muutmise_aeg": {"type": "string"},
56-
"taotletud_toetus": {"type": "string"},
57-
"taotleja_nimi": {"type": "string"},
58-
"taotleja_registrikood": {"type": "string"},
59-
},
60-
}
61-
62-
ID, SHORT_NAME, TITLE, DESCRIPTION, PROVIDERS, EXTENSIONS, COLUMNS, LICENSE = add_eurocrops(
63-
vars(), 2021
64-
)
65-
66-
67-
def convert(output_file, cache=None, **kwargs):
68-
convert_(
69-
output_file,
70-
cache,
71-
SOURCES,
72-
COLUMNS,
73-
ID,
74-
TITLE,
75-
DESCRIPTION,
76-
providers=PROVIDERS,
77-
extensions=EXTENSIONS,
78-
column_migrations=COLUMN_MIGRATIONS,
79-
missing_schemas=MISSING_SCHEMAS,
80-
attribution=ATTRIBUTION,
81-
license=LICENSE,
82-
**kwargs,
83-
)

‎fiboa_cli/datasets/ec_lt.py

+53-75
Original file line numberDiff line numberDiff line change
@@ -1,77 +1,55 @@
1-
from ..convert_utils import convert as convert_
2-
from .commons.ec import add_eurocrops
3-
4-
SOURCES = {"https://zenodo.org/records/6868143/files/LT_2021.zip": ["LT/LT_2021_EC.shp"]}
5-
6-
ID = "ec_lt"
7-
SHORT_NAME = "Lithuania"
8-
TITLE = "Field boundaries for Lithuania"
9-
DESCRIPTION = """
10-
Collection of data on agricultural land and crop areas, cultivated crops in the territory of the Republic of Lithuania.
11-
12-
The download service is a set of personalized spatial data of agricultural land and crop areas, cultivated crops. The service provides object geometry with descriptive (attributive) data.
13-
"""
14-
PROVIDERS = [
15-
{
16-
"name": "Construction Sector Development Agency",
17-
"url": "https://www.geoportal.lt/geoportal/nacionaline-mokejimo-agentura-prie-zemes-ukio-ministerijos#savedSearchId={56542726-DC0B-461E-A32C-3E9A4A693E27}&collapsed=true",
18-
"roles": ["producer", "licensor"],
1+
from ..convert_utils import BaseConverter
2+
from .commons.ec import EuroCropsConverterMixin
3+
4+
5+
class Converter(EuroCropsConverterMixin, BaseConverter):
6+
ec_mapping_csv = "lt_2021.csv"
7+
sources = {"https://zenodo.org/records/6868143/files/LT_2021.zip": ["LT/LT_2021_EC.shp"]}
8+
9+
id = "ec_lt"
10+
short_name = "Lithuania"
11+
title = "Field boundaries for Lithuania"
12+
description = """
13+
Collection of data on agricultural land and crop areas, cultivated crops in the territory of the Republic of Lithuania.
14+
15+
The download service is a set of personalized spatial data of agricultural land and crop areas, cultivated crops. The service provides object geometry with descriptive (attributive) data.
16+
"""
17+
providers = [
18+
{
19+
"name": "Construction Sector Development Agency",
20+
"url": "https://www.geoportal.lt/geoportal/nacionaline-mokejimo-agentura-prie-zemes-ukio-ministerijos#savedSearchId={56542726-DC0B-461E-A32C-3E9A4A693E27}&collapsed=true",
21+
"roles": ["producer", "licensor"],
22+
}
23+
]
24+
# license = {"title": "Non-commercial use only", "href": "https://www.geoportal.lt/metadata-catalog/catalog/search/resource/details.page?uuid=%7B7AF3F5B2-DC58-4EC5-916C-813E994B2DCF%7D", "type": "text/html", "rel": "license"}
25+
26+
columns = {
27+
"NMA_ID": "id",
28+
"GRUPE": "crop:name",
29+
"Shape_Leng": "perimeter",
30+
"Shape_Area": "area",
31+
"geometry": "geometry",
32+
}
33+
add_columns = {"determination_datetime": "2021-10-08T00:00:00Z"}
34+
column_migrations = {"Shape_Area": lambda column: column * 0.0001}
35+
column_filters = {
36+
"GRUPE": lambda col: (
37+
col.isin(
38+
[
39+
"Darþovës",
40+
"Grikiai",
41+
"Ankðtiniai javai",
42+
"Aviþos",
43+
"Þieminiai javai",
44+
"Summer Cereals",
45+
"Vasariniai javai",
46+
"Cukriniai runkeliai",
47+
"Uogynai",
48+
"Kukurûzai",
49+
]
50+
),
51+
False,
52+
)
1953
}
20-
]
21-
# LICENSE = {"title": "Non-commercial use only", "href": "https://www.geoportal.lt/metadata-catalog/catalog/search/resource/details.page?uuid=%7B7AF3F5B2-DC58-4EC5-916C-813E994B2DCF%7D", "type": "text/html", "rel": "license"}
22-
23-
COLUMNS = {
24-
"NMA_ID": "id",
25-
"GRUPE": "crop_name",
26-
"Shape_Leng": "perimeter",
27-
"Shape_Area": "area",
28-
"geometry": "geometry",
29-
}
30-
ADD_COLUMNS = {"determination_datetime": "2021-10-08T00:00:00Z"}
31-
COLUMN_MIGRATIONS = {"Shape_Area": lambda column: column * 0.0001}
32-
COLUMN_FILTERS = {
33-
"GRUPE": lambda col: (
34-
col.isin(
35-
[
36-
"Darþovës",
37-
"Grikiai",
38-
"Ankðtiniai javai",
39-
"Aviþos",
40-
"Þieminiai javai",
41-
"Summer Cereals",
42-
"Vasariniai javai",
43-
"Cukriniai runkeliai",
44-
"Uogynai",
45-
"Kukurûzai",
46-
]
47-
),
48-
False,
49-
)
50-
}
51-
52-
MISSING_SCHEMAS = {"required": [], "properties": {"crop_name": {"type": "string"}}}
53-
54-
ID, SHORT_NAME, TITLE, DESCRIPTION, PROVIDERS, EXTENSIONS, COLUMNS, LICENSE = add_eurocrops(
55-
vars(), 2021
56-
)
57-
5854

59-
# Conversion function, usually no changes required
60-
def convert(output_file, cache=None, **kwargs):
61-
convert_(
62-
output_file,
63-
cache,
64-
SOURCES,
65-
COLUMNS,
66-
ID,
67-
TITLE,
68-
DESCRIPTION,
69-
providers=PROVIDERS,
70-
extensions=EXTENSIONS,
71-
missing_schemas=MISSING_SCHEMAS,
72-
column_additions=ADD_COLUMNS,
73-
column_migrations=COLUMN_MIGRATIONS,
74-
column_filters=COLUMN_FILTERS,
75-
license=LICENSE,
76-
**kwargs,
77-
)
55+
missing_schemas = {"required": [], "properties": {"crop_name": {"type": "string"}}}

0 commit comments

Comments
 (0)