diff --git a/VERSION b/VERSION index 9084fa2f..524cb552 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.1.1 diff --git a/backend/gn_modulator/imports/mixins/check.py b/backend/gn_modulator/imports/mixins/check.py index 76c9bd2e..7108005c 100644 --- a/backend/gn_modulator/imports/mixins/check.py +++ b/backend/gn_modulator/imports/mixins/check.py @@ -113,7 +113,7 @@ def check_types(self): nb_lines = res[0] lines = res[1] error_values = res[2] - str_lines = lines and ", ".join(map(lambda x: str(x), lines)) or "" + # str_lines = lines and ", ".join(map(lambda x: str(x), lines)) or "" if nb_lines == 0: continue self.add_error( @@ -121,7 +121,7 @@ def check_types(self): key=key, lines=lines, error_values=error_values, - error_msg=f"Il y a des valeurs invalides pour la colonne {key} de type {sql_type}. {nb_lines} ligne(s) concernée(s) : [{str_lines}]", + error_msg=f"Valeurs invalides pour `{key}` ({sql_type}).", ) def check_required(self): diff --git a/backend/gn_modulator/imports/mixins/data.py b/backend/gn_modulator/imports/mixins/data.py index 3c5dcac1..20473310 100644 --- a/backend/gn_modulator/imports/mixins/data.py +++ b/backend/gn_modulator/imports/mixins/data.py @@ -31,17 +31,31 @@ def process_data_table(self): self.import_csv_file(self.tables["data"]) # TODO traiter autres types de fichier - else: self.add_error( error_code="ERR_IMPORT_DATA_FILE_TYPE_NOT_FOUND", - error_msg=f"Le type du fichier d'import {self.data_file_path} n'est pas traité", + error_msg=f"Le type de fichier '{Path(self.data_file_path).suffix}' n'est pas traité", ) return # comptage du nombre de ligne et verification de l'intégrité de la table self.count_and_check_table("data", self.tables["data"]) + # # on met à jour la table pour changer les valeurs '' en NULL + set_columns_txt = ", ".join( + map( + lambda x: f"{x}=NULLIF({x}, '')", + filter(lambda x: x != "id_import", self.get_table_columns(self.tables["data"])), + ) + ) + self.sql[ + "process_data" + ] = f""" + UPDATE {self.tables["data"]} SET {set_columns_txt}; + """ + + SchemaMethods.c_sql_exec_txt(self.sql["process_data"]) + def import_csv_file(self, dest_table): """ méthode pour lire les fichiers csv @@ -98,15 +112,6 @@ def import_csv_file(self, dest_table): else: self.copy_csv_data(f, dest_table, import_table_columns) - # on met à jour la table pour changer les valeurs '' en NULL - set_columns_txt = ", ".join("NULLIF({key}, '') AS {key}") - - self.sql[ - "process_data" - ] = f""" - UPDATE {dest_table} SET {set_columns_txt}; - """ - def copy_csv_data(self, f, dest_table, table_columns): """ requete sql pour copier les données en utilisant cursor.copy_expert @@ -120,15 +125,18 @@ def copy_csv_data(self, f, dest_table, table_columns): "data_copy_csv" ] = f"""COPY {dest_table}({columns_fields}) FROM STDIN DELIMITER '{self.csv_delimiter}' QUOTE '"' CSV """ + cursor = db.session.connection().connection.cursor() try: - cursor = db.session.connection().connection.cursor() cursor.copy_expert(sql=self.sql["data_copy_csv"], file=f) except Exception as e: + cursor.close() + db.session.rollback() self.add_error( error_code="ERR_IMPORT_DATA_COPY", error_msg=f"Erreur lors de la copie des données csv : {str(e)}", ) - return + + return def insert_csv_data(self, f, dest_table, table_columns): """ diff --git a/backend/gn_modulator/imports/mixins/utils.py b/backend/gn_modulator/imports/mixins/utils.py index 8883f485..7623d5ed 100644 --- a/backend/gn_modulator/imports/mixins/utils.py +++ b/backend/gn_modulator/imports/mixins/utils.py @@ -136,7 +136,7 @@ def add_error( self.errors.append( { "error_code": error_code, - "msg": error_msg, + "error_msg": error_msg, "key": key, "lines": lines, "valid_values": valid_values, diff --git a/backend/gn_modulator/routes/exports.py b/backend/gn_modulator/routes/exports.py index 7e2ec2b1..9e669c94 100644 --- a/backend/gn_modulator/routes/exports.py +++ b/backend/gn_modulator/routes/exports.py @@ -28,7 +28,11 @@ def api_export(module_code, object_code, export_code): # - export_definition # - on force fields a être # - TODO faire l'intersection de params['fields'] et export_definition['fields'] (si params['fields'] est défini) - params["fields"] = export_definition["fields"] + + sm = SchemaMethods(schema_code) + + _, fields = sm.process_export_fields(export_definition["fields"]) + params["fields"] = fields # - TODO autres paramètres ???? params["process_field_name"] = export_definition.get("process_field_name") @@ -37,8 +41,6 @@ def api_export(module_code, object_code, export_code): cruved_type = params.get("cruved_type") or "R" # recupération de la liste - sm = SchemaMethods(schema_code) - query_list = sm.query_list(module_code=module_code, cruved_type=cruved_type, params=params) # on assume qu'il n'y que des export csv diff --git a/backend/gn_modulator/routes/utils/repository.py b/backend/gn_modulator/routes/utils/repository.py index 2e2ed230..f7ac871f 100644 --- a/backend/gn_modulator/routes/utils/repository.py +++ b/backend/gn_modulator/routes/utils/repository.py @@ -143,7 +143,9 @@ def delete_rest(module_code, object_code, value): dict_out = sm.serialize(m, fields=params.get("fields"), as_geojson=params.get("as_geojson")) try: - sm.delete_row(value, field_name=params.get("field_name"), commit=True) + sm.delete_row( + value, module_code=module_code, field_name=params.get("field_name"), commit=True + ) except sm.errors.SchemaUnsufficientCruvedRigth as e: return "Erreur Cruved : {}".format(str(e)), 403 diff --git a/backend/gn_modulator/schema/export.py b/backend/gn_modulator/schema/export.py index 7528ed2c..6a48b5dc 100644 --- a/backend/gn_modulator/schema/export.py +++ b/backend/gn_modulator/schema/export.py @@ -28,7 +28,7 @@ class SchemaExport: export """ - def process_export_fields(self, fields_in, process_field_name): + def process_export_fields(self, fields_in, process_field_name=False): """ Renvoie - la liste des clé diff --git a/backend/gn_modulator/schema/repositories/base.py b/backend/gn_modulator/schema/repositories/base.py index dcbbeac2..6500eb18 100644 --- a/backend/gn_modulator/schema/repositories/base.py +++ b/backend/gn_modulator/schema/repositories/base.py @@ -221,6 +221,7 @@ def delete_row( # pour être sûr qu'il n'y a qu'une seule ligne de supprimée if not multiple: m.one() + # https://stackoverflow.com/questions/49794899/flask-sqlalchemy-delete-query-failing-with-could-not-evaluate-current-criteria?noredirect=1&lq=1 m.delete(synchronize_session=False) db.session.flush() @@ -276,8 +277,7 @@ def query_list(self, module_code=MODULE_CODE, cruved_type="R", params={}, query_ # clear_query_cache self.clear_query_cache(query) - order_bys, query = self.get_sorters(Model, params.get("sort", []), query) - + order_bys, query = self.get_sorters(params.get("sort", []), query) # if params.get('test'): # print('test') # query = query.options( diff --git a/backend/gn_modulator/schema/repositories/utils.py b/backend/gn_modulator/schema/repositories/utils.py index 42ff863f..11a0e7eb 100644 --- a/backend/gn_modulator/schema/repositories/utils.py +++ b/backend/gn_modulator/schema/repositories/utils.py @@ -1,4 +1,5 @@ -from sqlalchemy import orm, and_, nullslast +import re +from sqlalchemy import orm, and_, nullslast, func, Numeric, cast from sqlalchemy.orm import load_only, Load from gn_modulator.utils.commons import getAttr @@ -168,40 +169,43 @@ def custom_getattr( res, query, condition, field_name, index, only_fields ) - def get_sorters(self, Model, sort, query): + def get_sorters(self, sort, query): order_bys = [] for s in sort: - sort_dir = "+" - sort_field = s - if s[-1] == "-": - sort_field = s[:-1] - sort_dir = s[-1] + sorters, query = self.get_sorter(s, query) + order_bys.extend(sorters) - model_attribute, query = self.custom_getattr(Model, sort_field, query) - - if model_attribute is None: - continue + return order_bys, query - order_by = model_attribute.desc() if sort_dir == "-" else model_attribute.asc() + def get_sorter(self, sorter, query): + orders_by = [] + sort_dir = "-" if "-" in sorter else "+" + sort_spe = "str_num" if "*" in sorter else "num_str" if "%" in sorter else None + sort_field = re.sub(r"[+-\\*%]", "", sorter) - # nullslast - order_by = nullslast(order_by) - order_bys.append(order_by) + model_attribute, query = self.custom_getattr(self.Model(), sort_field, query) - return order_bys, query + if model_attribute is None: + raise Exception(f"Pb avec le tri {self.schema_code()}, field: {sort_field}") - def get_sorter(self, Model, sorter, query): - sort_field = sorter["field"] - sort_dir = sorter["dir"] + if sort_spe is not None: + sort_string = func.substring(model_attribute, "[a-zA-Z]+") + sort_number = cast(func.substring(model_attribute, "[0-9]+"), Numeric) - model_attribute, query = self.custom_getattr(Model, sort_field, query) + if sort_spe == "str_num": + orders_by.extend([sort_string, sort_number]) + else: + orders_by.extend([sort_number, sort_string]) - order_by = model_attribute.desc() if sort_dir == "desc" else model_attribute.asc() + orders_by.append(model_attribute) - order_by = nullslast(order_by) + orders_by = [ + nullslast(order_by.desc() if sort_dir == "-" else order_by.asc()) + for order_by in orders_by + ] - return order_by, query + return orders_by, query def process_page_size(self, page, page_size, query): """ diff --git a/backend/gn_modulator/tests/import_test/pf_errors.csv b/backend/gn_modulator/tests/import_test/pf_errors.csv new file mode 100644 index 00000000..99c5d4ce --- /dev/null +++ b/backend/gn_modulator/tests/import_test/pf_errors.csv @@ -0,0 +1,3 @@ +code_passage_faune;geom;id_nomenclature_ouvrage_specificite +TEST_EX_SIMPLE;POINT(0 45);SPE +;POINT(0 45)__;yuk diff --git a/config/layouts/tests/test_import.layout.yml b/config/layouts/tests/test_import.layout.yml index 2b823079..8959f2ca 100644 --- a/config/layouts/tests/test_import.layout.yml +++ b/config/layouts/tests/test_import.layout.yml @@ -9,5 +9,5 @@ layout: module_code: m_sipaf hidden_options: - enable_update - test_import: true + test_import: false data: \ No newline at end of file diff --git a/config/layouts/utils/utils.import.layout.yml b/config/layouts/utils/utils.import.layout.yml index ff702aa8..a311c82b 100644 --- a/config/layouts/utils/utils.import.layout.yml +++ b/config/layouts/utils/utils.import.layout.yml @@ -78,22 +78,22 @@ layout: items: - type: button color: primary - title: Annuler - description: Annuler + title: "__f__data.status == 'DONE' ? 'Terminer' : 'Annuler'" + description: "Sortir de l'interface d'import" action: close - flex: '0' + flex: "0" - type: button - flex: '0' + flex: "0" icon: refresh color: reset - description: Faire un nouvel import + description: Remise à zero du formulaire pour un nouvel import action: reset hidden: __f__!data.id_import - type: button - flex: '0' + flex: "0" color: success title: Valider description: Valider action: import - disabled: __f__!(formGroup.valid ) + disabled: __f__!formGroup.valid || !!data.errors?.length hidden: __f__data.status == 'DONE' diff --git a/contrib/m_sipaf/config/config.yml b/contrib/m_sipaf/config/config.yml index a2974d1f..02240645 100644 --- a/contrib/m_sipaf/config/config.yml +++ b/contrib/m_sipaf/config/config.yml @@ -70,9 +70,9 @@ site_filters_fields: object_code: ref_geo.linear_group module_code: __REF_MODULE_CODE__ filters: linears.type.type_code = RTE - sort: name + sort: code* reload_on_search: true - page_size: 10 + page_size: 20 site_filters_defs: code_passage_faune: diff --git a/contrib/m_sipaf/config/layouts/m_sipaf.site_list.layout.yml b/contrib/m_sipaf/config/layouts/m_sipaf.site_list.layout.yml index d4fce92a..4da68ca5 100644 --- a/contrib/m_sipaf/config/layouts/m_sipaf.site_list.layout.yml +++ b/contrib/m_sipaf/config/layouts/m_sipaf.site_list.layout.yml @@ -54,11 +54,11 @@ layout: title: "Export complet" description: Télécharger les passages à faune (les filtres sont appliqués) href: __f__o.url_export(x, 'm_sipaf.pf') - - type: button - flex: "0" - title: "Export import" - description: Export destiné à l'import (les filtres sont appliqués) - href: __f__o.url_export(x, 'm_sipaf.pf_import') + # - type: button + # flex: "0" + # title: "Export import" + # description: Export destiné à l'import (les filtres sont appliqués) + # href: __f__o.url_export(x, 'm_sipaf.pf_import') - type: button flex: "0" icon: upload diff --git a/contrib/m_sipaf/doc/import.md b/contrib/m_sipaf/doc/import.md index ba603694..a3627511 100644 --- a/contrib/m_sipaf/doc/import.md +++ b/contrib/m_sipaf/doc/import.md @@ -1,4 +1,4 @@ -# Impot de passage faune +# Import de passages faune ## Définition des champs @@ -8,15 +8,16 @@ - [Exemple simple](/backend/gn_modulator/tests/import_test/pf_simple.csv) - [Exemple complet](/backend/gn_modulator/tests/import_test/pf_complet.csv) -## Procédure d'import sur l'interface web -### Accès au menu d'import +## Procédure d'import depuis l'interface web + +### Accès à l'outil d'import Si l'utilisateur possède des droits de création pour ce module, alors le bouton d'import est visible. ![Bouton d'import](img/boutton_import.png) -Le menu d'import apparait dans une fenêtre modale. +L'outil d'import apparait dans une fenêtre modale. ![Menu d'import](img/menu_import.png) diff --git a/doc/changelog.md b/doc/changelog.md index 88b59124..978f5212 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -1,5 +1,19 @@ # Changelog +## 1.1.1 (2023-06-29) + +**🐛 Corrections** + +- Correction des exports +- Suppression de l'export de test "Export import" +- Correction des tooltips des boutons +- Amélioration de la liste du filtre des infrastructures +- Correction des permissions de suppression d'un objet, s'appuyant sur l'action D du sous-module +- Retrait du bouton de suppression sur les tooltips des cartes +- Correction de l'affichage des imports en frontend +- Import: + - fichier csv: passage des valeurs caractère vide ('') à NULL + - frontend: correction affichage erreur ## 1.1.0 (2023-06-27) Nécessite la version 2.13.0 (ou plus) de GeoNature. diff --git a/frontend/app/components/base/base.scss b/frontend/app/components/base/base.scss index 83e54e7c..26f9bfc0 100644 --- a/frontend/app/components/base/base.scss +++ b/frontend/app/components/base/base.scss @@ -162,6 +162,11 @@ div.layout-items > div { padding: 10px; } +.layout-button { + display: inherit; +} + + .layout-key-title { color: rgb(100, 100, 100); font-style: italic; diff --git a/frontend/app/components/layout/base/layout.component.html b/frontend/app/components/layout/base/layout.component.html index fe7a057e..ccb5a5ea 100644 --- a/frontend/app/components/layout/base/layout.component.html +++ b/frontend/app/components/layout/base/layout.component.html @@ -19,7 +19,8 @@

- {{ computedLayout.title }} : {{ computedLayout.test }} ({{ computedLayout.test_value }}) + {{ computedLayout.title }} : {{ computedLayout.test }} ({{ computedLayout.test_value }})

@@ -79,15 +80,15 @@ > - - - - + + + + @@ -118,46 +119,46 @@ -
+
- - - - - - - {{ computedLayout.icon }} - {{ computedLayout.title }} - +
+ + + + + + {{ computedLayout.icon }} + {{ computedLayout.title }} + +
diff --git a/frontend/app/components/layout/form/list-form.component.html b/frontend/app/components/layout/form/list-form.component.html index 5bb5ff8a..738fb8d5 100644 --- a/frontend/app/components/layout/form/list-form.component.html +++ b/frontend/app/components/layout/form/list-form.component.html @@ -16,7 +16,7 @@ { + this._mLayout.reComputeLayout(''); + }, 50); } } } diff --git a/frontend/app/components/layout/object/layout-object-geojson.component.ts b/frontend/app/components/layout/object/layout-object-geojson.component.ts index baf6aed1..4b95f380 100644 --- a/frontend/app/components/layout/object/layout-object-geojson.component.ts +++ b/frontend/app/components/layout/object/layout-object-geojson.component.ts @@ -172,9 +172,10 @@ export class ModulesLayoutObjectGeoJSONComponent ? '' : ''; - const htmlDelete = this._mObject.checkAction(this.context, 'D', properties.scope).actionAllowed - ? '' - : ''; + const htmlDelete = ''; + // const htmlDelete = this._mObject.checkAction(this.context, 'D', properties.scope).actionAllowed + // ? '' + // : ''; const html = `

${label || ''}

diff --git a/frontend/app/services/import.service.ts b/frontend/app/services/import.service.ts index d975ed1e..4a85d13c 100644 --- a/frontend/app/services/import.service.ts +++ b/frontend/app/services/import.service.ts @@ -82,16 +82,16 @@ export class ModulesImportService { let nbUnchanged = data.res.nb_unchanged.toString().padStart(nbChar, charSpace); if (data.options.enable_update) { - htmlUpdate += `
  • ${nbUpdate} lignes mises à jour
    • `; + htmlUpdate += `
  • ${nbUpdate} ligne(s) mises à jour
    • `; } if (data.res.nb_unchanged) { - htmlUnchanged += `
  • ${nbUnchanged} lignes non modifiées
    • `; + htmlUnchanged += `
  • ${nbUnchanged} ligne(s) non modifiées
    • `; } return `
    • ${nbRaw} lignes dans le fichier
      • -
    • ${nbInsert} lignes ajoutées
      • +
    • ${nbInsert} ligne(s) ajoutées
      • ${htmlUpdate} ${htmlUnchanged}
    `.replace(/_/g, ' '); @@ -104,7 +104,7 @@ export class ModulesImportService { const lines = {}; for (const error of data.errors) { - for (const line of error.lines) { + for (const line of error.lines || []) { lines[line] = lines[line] || {}; lines[line][error.error_code] = lines[line][error.error_code] || { error_msg: error.error_msg, @@ -150,9 +150,10 @@ export class ModulesImportService { for (let error of errorsOfType) { if (error.key) { errors[errorType].keys[error.key] = { lines: error.lines }; - errorHTML += `- ${error.key} : ligne${ - error.lines.length > 1 ? 's' : '' - } ${error.lines.join(', ')}
    `; + + for (const [i, line] of error.lines.entries()) { + errorHTML += `- [ligne ${line}] : ${error.error_values[i]}
    `; + } } } }