Merge pull request #1482 from betagouv/1465-history

Compléter les historiques de Déclarations

data/README.md

@@ -44,7 +44,7 @@ REF_ICA_FONCTION_INGREDIENT     |       |       |       |       |       |
 |ICA_POPULATION_RISQUE_DECLAREE     |       |       |
 |ICA_SUBSTANCE_DECLAREE     |       |       |
 |ICA_USAGER     |       |   🕵️anonymisée (contient Foreign Key vers USR, ADM, ETAB)   |
-|REF_ICA_TYPE_DECLARATION     |       |Enum ? ou obsolète ? (Art 1(, Art 1-, Simplifiée))       |
+|REF_ICA_TYPE_DECLARATION     |       |Enum ? ou obsolète ? (Art 15, Art 16, Simplifiée))       |
 |REF_ICA_TYPE_HERITAGE     |       | Enum ? (Simplifié ou Nouvelle formule)|
 |REF_ICA_TYPE_VERSION_DECLARATION     |       |       |
 |REF_ICA_FORME_GALENIQUE   |       |       |

data/etl/teleicare_history/extractor.py

@@ -4,18 +4,46 @@
 from datetime import date, datetime, timezone
 
 from django.core.exceptions import MultipleObjectsReturned, ValidationError
-from django.db import IntegrityError
+from django.db import IntegrityError, transaction
 
 from phonenumber_field.phonenumber import PhoneNumber
 
-from data.models import GalenicFormulation, SubstanceUnit
+from data.models import (
+    Condition,
+    Effect,
+    GalenicFormulation,
+    Ingredient,
+    Microorganism,
+    Plant,
+    PlantPart,
+    Population,
+    Preparation,
+    Substance,
+    SubstanceUnit,
+)
 from data.models.company import ActivityChoices, Company
-from data.models.declaration import Declaration
+from data.models.declaration import (
+    ComputedSubstance,
+    Declaration,
+    DeclaredIngredient,
+    DeclaredMicroorganism,
+    DeclaredPlant,
+)
 from data.models.teleicare_history.ica_declaration import (
     IcaComplementAlimentaire,
     IcaDeclaration,
+    IcaEffetDeclare,
+    IcaPopulationCibleDeclaree,
+    IcaPopulationRisqueDeclaree,
     IcaVersionDeclaration,
 )
+from data.models.teleicare_history.ica_declaration_composition import (
+    IcaIngredient,
+    IcaIngredientAutre,
+    IcaMicroOrganisme,
+    IcaPreparation,
+    IcaSubstanceDeclaree,
+)
 from data.models.teleicare_history.ica_etablissement import IcaEtablissement
 
 logger = logging.getLogger(__name__)
@@ -177,6 +205,144 @@ def convert_str_date(value, aware=False):
     8: Declaration.DeclarationStatus.ABANDONED,  # 'abandonné'
 }
 
+DECLARATION_TYPE_TO_ARTICLE_MAPPING = {
+    1: Declaration.Article.ARTICLE_15,
+    2: Declaration.Article.ARTICLE_16,
+    3: None,  # Type "simplifié" dans Teleicare, normalement liées à des modifications de déclarations déjà instruites
+}
+
+
+MANY_TO_MANY_PRODUCT_MODELS_MATCHING = {
+    "effects": {"teleIcare_model": IcaEffetDeclare, "teleIcare_pk": "objeff_ident", "CA_model": Effect},
+    "conditions_not_recommended": {
+        "teleIcare_model": IcaPopulationRisqueDeclaree,
+        "teleIcare_pk": "poprs_ident",
+        "CA_model": Condition,
+    },
+    "populations": {
+        "teleIcare_model": IcaPopulationCibleDeclaree,
+        "teleIcare_pk": "popcbl_ident",
+        "CA_model": Population,
+    },
+}
+
+
+def create_declared_plant(declaration, teleIcare_plant, active):
+    declared_plant = DeclaredPlant(
+        declaration=declaration,
+        plant=Plant.objects.get(siccrf_id=teleIcare_plant.plte_ident),
+        active=active,
+        quantity=teleIcare_plant.prepa_qte if active else None,
+        unit=SubstanceUnit.objects.get(siccrf_id=teleIcare_plant.unt_ident) if active else None,
+        preparation=Preparation.objects.get(siccrf_id=teleIcare_plant.typprep_ident) if active else None,
+        used_part=PlantPart.objects.get(siccrf_id=teleIcare_plant.pplan_ident) if active else None,
+    )
+    return declared_plant
+
+
+def create_declared_microorganism(declaration, teleIcare_microorganism, active):
+    declared_microorganism = DeclaredMicroorganism(
+        declaration=declaration,
+        microorganism=Microorganism.objects.get(siccrf_id=teleIcare_microorganism.morg_ident),
+        active=active,
+        activated=True,  # dans TeleIcare, le champ `activated` n'existait pas, les MO inactivés étaient des `ingrédients autres`
+        strain=teleIcare_microorganism.ingmorg_souche,
+        quantity=teleIcare_microorganism.ingmorg_quantite_par_djr,
+    )
+    return declared_microorganism
+
+
+def create_declared_ingredient(declaration, teleIcare_ingredient, active):
+    declared_ingredient = DeclaredIngredient(
+        declaration=declaration,
+        ingredient=Ingredient.objects.get(siccrf_id=teleIcare_ingredient.inga_ident),
+        active=active,
+        quantity=None,  # dans TeleIcare, les ingrédients n'avaient pas de quantité associée
+    )
+    return declared_ingredient
+
+
+def create_computed_substance(declaration, teleIcare_substance):
+    computed_substance = ComputedSubstance(
+        declaration=declaration,
+        substance=Substance.objects.get(siccrf_id=teleIcare_substance.sbsact_ident),
+        quantity=teleIcare_substance.sbsactdecl_quantite_par_djr,
+        # le champ de TeleIcare 'sbsact_commentaires' n'est pas repris
+    )
+    return computed_substance
+
+
+@transaction.atomic
+def add_product_info_from_teleicare_history(declaration, vrsdecl_ident):
+    """
+    Cette function importe les champs ManyToMany des déclarations, relatifs à l'onglet "Produit"
+    Il est nécessaire que les objets soient enregistrés en base (et aient obtenu un id) grâce à la fonction
+    `create_declaration_from_teleicare_history` pour updater leurs champs ManyToMany.
+    """
+    # TODO: other_conditions=conditions_not_recommended,
+    # TODO: other_effects=
+    for CA_field_name, struct in MANY_TO_MANY_PRODUCT_MODELS_MATCHING.items():
+        # par exemple Declaration.populations
+        CA_field = getattr(declaration, CA_field_name)
+        # TODO: utiliser les dataclass ici
+        pk_field = struct["teleIcare_pk"]
+        CA_model = struct["CA_model"]
+        teleIcare_model = struct["teleIcare_model"]
+        CA_field.set(
+            [
+                CA_model.objects.get(siccrf_id=getattr(TI_object, pk_field))
+                for TI_object in (teleIcare_model.objects.filter(vrsdecl_ident=vrsdecl_ident))
+            ]
+        )
+
+
+@transaction.atomic
+def add_composition_from_teleicare_history(declaration, vrsdecl_ident):
+    """
+    Cette function importe les champs ManyToMany des déclarations, relatifs à l'onglet "Composition"
+    Il est nécessaire que les objets soient enregistrés en base (et aient obtenu un id) grâce à la fonction
+    `create_declaration_from_teleicare_history` pour updater leurs champs ManyToMany.
+    """
+    bulk_ingredients = {DeclaredPlant: [], DeclaredMicroorganism: [], DeclaredIngredient: [], ComputedSubstance: []}
+    for ingredient in IcaIngredient.objects.filter(vrsdecl_ident=vrsdecl_ident):
+        if ingredient.tying_ident == 1:
+            declared_plant = create_declared_plant(
+                declaration=declaration,
+                teleIcare_plant=IcaPreparation.objects.get(
+                    ingr_ident=ingredient.ingr_ident,
+                ),
+                active=ingredient.fctingr_ident == 1,
+            )
+            bulk_ingredients[DeclaredPlant].append(declared_plant)
+        elif ingredient.tying_ident == 2:
+            declared_microorganism = create_declared_microorganism(
+                declaration=declaration,
+                teleIcare_microorganism=IcaMicroOrganisme.objects.get(
+                    ingr_ident=ingredient.ingr_ident,
+                ),
+                active=ingredient.fctingr_ident == 1,
+            )
+            bulk_ingredients[DeclaredMicroorganism].append(declared_microorganism)
+        elif ingredient.tying_ident == 3:
+            declared_ingredient = create_declared_ingredient(
+                declaration=declaration,
+                teleIcare_ingredient=IcaIngredientAutre.objects.get(
+                    ingr_ident=ingredient.ingr_ident,
+                ),
+                active=ingredient.fctingr_ident == 1,
+            )
+            bulk_ingredients[DeclaredIngredient].append(declared_ingredient)
+    # dans TeleIcare les `declared_substances` étaient des ingrédients
+    # donc on ne rempli pas le champ declaration.declared_substances grâce à l'historique
+    for teleIcare_substance in IcaSubstanceDeclaree.objects.filter(vrsdecl_ident=vrsdecl_ident):
+        computed_substance = create_computed_substance(
+            declaration=declaration, teleIcare_substance=teleIcare_substance
+        )
+        bulk_ingredients[ComputedSubstance].append(computed_substance)
+
+    for model, bulk_of_objects in bulk_ingredients.items():
+        model.objects.bulk_create(bulk_of_objects)
+
 
 def create_declaration_from_teleicare_history():
     """
@@ -248,12 +414,8 @@ def create_declaration_from_teleicare_history():
                     minimum_duration=latest_ica_version_declaration.vrsdecl_durabilite,
                     instructions=latest_ica_version_declaration.vrsdecl_mode_emploi or "",
                     warning=latest_ica_version_declaration.vrsdecl_mise_en_garde or "",
-                    # TODO: ces champs proviennent de tables pas encore importées
-                    # populations=
-                    # conditions_not_recommended=
-                    # other_conditions=
-                    # effects=
-                    # other_effects=
+                    calculated_article=DECLARATION_TYPE_TO_ARTICLE_MAPPING[latest_ica_declaration.tydcl_ident],
+                    # TODO: ces champs sont à importer
                     # address=
                     # postal_code=
                     # city=
@@ -266,6 +428,12 @@ def create_declaration_from_teleicare_history():
                 try:
                     with suppress_autotime(declaration, ["creation_date", "modification_date"]):
                         declaration.save()
+                        add_product_info_from_teleicare_history(
+                            declaration, latest_ica_version_declaration.vrsdecl_ident
+                        )
+                        add_composition_from_teleicare_history(
+                            declaration, latest_ica_version_declaration.vrsdecl_ident
+                        )
                         nb_created_declarations += 1
                 except IntegrityError:
                     # cette Déclaration a déjà été créée

data/etl/teleicare_history/sql/company_table_creation.sql

@@ -0,0 +1,43 @@
+-- Le but de ce fichier est de permettre la recréation des tables Teleicare correspondant
+-- au modèle de Declaration Compl'Alim
+
+
+-- int int smallint -> integer
+-- vachar -> text
+-- bit -> boolean
+-- datetime -> text (pour une conversion ensuite)
+
+CREATE TABLE ICA_ETABLISSEMENT (
+    ETAB_IDENT integer PRIMARY KEY,
+    COG_IDENT integer NULL,
+    ETAB_IDENT_PARENT integer NULL,
+    PAYS_IDENT integer NOT NULL,
+    ETAB_SIRET text NULL,
+    ETAB_NUMERO_TVA_INTRA text NULL,
+    ETAB_RAISON_SOCIALE text NOT NULL,
+    ETAB_ENSEIGNE text NULL,
+    ETAB_ADRE_VILLE text NULL,
+    ETAB_ADRE_CP text NULL,
+    ETAB_ADRE_VOIE text NULL,
+    ETAB_ADRE_COMP text NULL,
+    ETAB_ADRE_COMP2 text NULL,
+    ETAB_ADRE_DIST text NULL,
+    ETAB_TELEPHONE text NULL,
+    ETAB_FAX text NULL,
+    ETAB_COURRIEL text NULL,
+    ETAB_SITE_INTERNET text NULL,
+    ETAB_ICA_FACONNIER boolean NULL,
+    ETAB_ICA_FABRICANT boolean NULL,
+    ETAB_ICA_CONSEIL boolean NULL,
+    ETAB_ICA_IMPORTATEUR boolean NULL,
+    ETAB_ICA_INTRODUCTEUR boolean NULL,
+    ETAB_ICA_DISTRIBUTEUR boolean NULL,
+    ETAB_ICA_ENSEIGNE text NULL,
+    ETAB_ADRE_REGION text NULL,
+    ETAB_DT_AJOUT_IDENT_PARENT text NULL,
+    ETAB_NUM_ADH_TELE_PROC text NULL,
+    ETAB_COMMENTAIRE_IDENT_PARENT text NULL,
+    ETAB_NOM_DOMAINE text NULL,
+    ETAB_DATE_ADHESION text NULL,
+    ETAB_NB_COMPTE_AUTORISE integer NOT NULL
+);

data/etl/teleicare_history/sql/declaration_composition_table_creation.sql

@@ -0,0 +1,44 @@
+CREATE TABLE ICA_INGREDIENT (
+    INGR_IDENT integer PRIMARY KEY,
+    VRSDECL_IDENT integer NOT NULL,
+    FCTINGR_IDENT integer NOT NULL,
+    TYING_IDENT integer NOT NULL,
+    INGR_COMMENTAIRES text NULL
+);
+
+
+
+
+CREATE TABLE ICA_MICRO_ORGANISME (
+    INGR_IDENT integer PRIMARY KEY,
+    MORG_IDENT integer NOT NULL,
+    INGMORG_SOUCHE text NULL,
+    INGMORG_QUANTITE_PAR_DJR bigint NULL
+);
+
+
+
+CREATE TABLE ICA_INGREDIENT_AUTRE (
+    INGR_IDENT integer PRIMARY KEY,
+    INGA_IDENT integer NOT NULL
+);
+
+
+
+CREATE TABLE ICA_PREPARATION (
+    INGR_IDENT integer PRIMARY KEY,
+    PLTE_IDENT integer NOT NULL,
+    PPLAN_IDENT integer NULL,
+    UNT_IDENT integer NULL,
+    TYPPREP_IDENT integer NULL,
+    PREPA_QTE float NULL
+);
+
+
+CREATE TABLE ICA_SUBSTANCE_DECLAREE(
+    VRSDECL_IDENT integer NOT NULL,
+    SBSACT_IDENT integer NOT NULL,
+    SBSACT_COMMENTAIRES text NULL,
+    SBSACTDECL_QUANTITE_PAR_DJR float NULL,
+    PRIMARY KEY (VRSDECL_IDENT, SBSACT_IDENT)
+);