MEP v12

Pipfile

@@ -57,7 +57,7 @@ docxtpl = "*"
 geopandas = "*"
 gunicorn = "*"
 jenkspy = "*"
-josepy = "==1.15.0"
+josepy = "*"
 mapclassify = "*"
 matplotlib = "*"
 matplotlib-scalebar = "*"
@@ -74,4 +74,4 @@ django-webpack-loader = "*"
 django-two-factor-auth = "*"
 qrcode = "*"
 phonenumbers = "*"
-mozilla-django-oidc = "*"
+mozilla-django-oidc = "5.0.2"

airflow/dags/data_gouv/Config.py

@@ -2,39 +2,61 @@
 
 
 @dataclass(frozen=True, kw_only=True)
-class BaseConfig:
-    frequency: str = "@once"
-    dag_id: str
+class ResourceConfig:
     format: str
     filename: str
-    data_gouv_dataset: str
-    data_gouv_resource: str
+    resource_slug: str
+    resource_title: str
 
 
 @dataclass(frozen=True, kw_only=True)
-class GeopackageConfig(BaseConfig):
+class GeopackageResourceConfig(ResourceConfig):
     sql_to_layer_name_mapping: dict
 
 
 @dataclass(frozen=True, kw_only=True)
-class CSVConfig(BaseConfig):
+class CSVResourceConfig(ResourceConfig):
     sql: str
 
 
+@dataclass(frozen=True, kw_only=True)
+class DatasetConfig:
+    dataset_slug: str
+    dataset_title: str
+    frequency: str = "@once"
+    resources: tuple[ResourceConfig, ...] = ()
+
+
 class ConfigParser:
-    def __get_config_class(self, format: str):
+    def __get_resource_config_class(self, format: str):
         parser_map = {
-            "csv": CSVConfig,
-            "gpkg": GeopackageConfig,
+            "csv": CSVResourceConfig,
+            "gpkg": GeopackageResourceConfig,
         }
 
         if format not in parser_map:
             raise ValueError(f"Le format {format} n'est pas pris en charge")
 
         return parser_map[format]
 
-    def parse_config(self, data: dict):
+    def parse_resource(self, data: dict) -> ResourceConfig:
         if "format" not in data:
             raise ValueError("Le format est manquant dans la configuration", data)
 
-        return self.__get_config_class(data["format"])(**data)
+        return self.__get_resource_config_class(data["format"])(**data)
+
+    def parse_config(self, data: dict) -> DatasetConfig:
+        if "dataset_title" not in data:
+            raise ValueError("Le dataset_title est manquant dans la configuration", data)
+
+        if "dataset_slug" not in data:
+            raise ValueError("Le dataset_slug est manquant dans la configuration", data)
+
+        resources = tuple(self.parse_resource(resource_data) for resource_data in data.get("resources", []))
+
+        return DatasetConfig(
+            dataset_slug=data["dataset_slug"],
+            dataset_title=data["dataset_title"],
+            frequency=data.get("frequency", "@once"),
+            resources=resources,
+        )

airflow/dags/data_gouv/__init__.py

@@ -1,3 +1,2 @@
-from .csv_exports import *  # noqa: F403, F401
-from .geopackage_exports import *  # noqa: F403, F401
-from .utils import get_configs  # noqa: F403, F401
+from .export_all_to_data_gouv import *  # noqa: F403, F401
+from .utils import get_dataset_configs  # noqa: F403, F401

airflow/dags/data_gouv/archive_all_from_data_gouv.py

@@ -0,0 +1,42 @@
+"""
+DAG pour supprimer tous les datasets de data.gouv.fr.
+
+Ce DAG récupère tous les datasets de l'organisation sur data.gouv.fr
+et les supprime.
+"""
+
+import pendulum
+from include.container import DomainContainer as Container
+
+from airflow.decorators import dag, task
+
+
+@dag(
+    dag_id="archive_all_from_data_gouv",
+    start_date=pendulum.datetime(2024, 1, 1),
+    schedule="@once",
+    catchup=False,
+    doc_md=__doc__,
+    max_active_runs=1,
+    default_args={"owner": "Alexis Athlani", "retries": 3},
+    max_active_tasks=10,
+    tags=["data_gouv", "archive"],
+)
+def archive_all_from_data_gouv():
+    @task.python
+    def get_all_dataset_ids() -> list[str]:
+        """Récupère tous les IDs de datasets depuis data.gouv.fr."""
+        datasets = Container().data_gouv().get_organization_datasets()
+        return [ds["id"] for ds in datasets]
+
+    @task.python
+    def delete_dataset(dataset_id: str) -> str:
+        """Supprime un dataset de data.gouv.fr."""
+        Container().data_gouv().delete_dataset(dataset_id)
+        return dataset_id
+
+    dataset_ids = get_all_dataset_ids()
+    delete_dataset.expand(dataset_id=dataset_ids)
+
+
+archive_all_from_data_gouv()

airflow/dags/data_gouv/configs.py

@@ -1,53 +1,206 @@
-from os import getenv
-
-if getenv("ENVIRONMENT") == "production":
-    configs = [
-        {
-            "dag_id": "create_and_publish_france_impermeabilisation_communes_geopackage",
-            "format": "gpkg",
-            "filename": "france_impermeabilisation_communes.gpkg",
-            "sql_to_layer_name_mapping": {
-                "impermeabilisation": "SELECT * FROM public_for_export.commune_imper_by_year"
-            },
-            "data_gouv_dataset": "67bddc715e3badfc49d5df9b",
-            "data_gouv_resource": "b0bbf493-a078-46eb-b901-7d0754bf6e77",
-        },
-        {
-            "dag_id": "create_and_publish_france_impermeabilisation_communes_csv",
-            "format": "csv",
-            "filename": "france_impermeabilisation_communes.csv",
-            "sql": "SELECT * FROM public_for_export.commune_imper_by_year",
-            "data_gouv_dataset": "67bddc715e3badfc49d5df9b",
-            "data_gouv_resource": "800c36e1-8c12-46af-b495-4689edc40c3b",
-        },
-        {
-            "dag_id": "create_and_publish_france_artificialisation_communes_geopackage",
-            "format": "gpkg",
-            "filename": "france_artificialisation_communes.gpkg",
-            "sql_to_layer_name_mapping": {
-                "impermeabilisation": "SELECT * FROM public_for_export.commune_artif_by_year"
-            },
-            "data_gouv_dataset": "67daf7413e4c8cbd0ff2da24",
-            "data_gouv_resource": "7bf29725-6cdc-4f68-a10f-b9a99bf6365e",
-        },
-        {
-            "dag_id": "create_and_publish_part_territoire_francais_couvert_par_ocsge_csv",
-            "format": "csv",
-            "filename": "part_territoire_francais_couvert_par_ocsge.csv",
-            "sql": "SELECT * FROM public_for_export.ocsge_surface_cover_percent",
-            "data_gouv_dataset": "68348c9afa867945df5968e5",
-            "data_gouv_resource": "3a37cc78-4aa7-4a8e-814f-4e3714eb9a90",
-            "frequency": "@weekly",
-        },
-        {
-            "dag_id": "create_and_publish_trajectoires_communes_csv",
-            "format": "csv",
-            "filename": "trajectoires_communes.csv",
-            "sql": "SELECT * FROM public_for_export.trajectoires_communes",
-            "data_gouv_dataset": "688a1dcd9eaa8e9b2bd6bfc6",
-            "data_gouv_resource": "e4fa6a1d-48f1-44dd-bde9-b3c7814487bd",
-            "frequency": "@once",
-        },
-    ]
-else:
-    configs = []
+configs = [
+    {
+        "dataset_slug": "ocsge-coverage-2026",
+        "dataset_title": "Part du territoire Francais couvert par OCS GE",
+        "frequency": "@weekly",
+        "resources": [
+            {
+                "format": "csv",
+                "filename": "part_territoire_francais_couvert_par_ocsge.csv",
+                "sql": "SELECT * FROM public_for_export.ocsge_surface_cover_percent",
+                "resource_slug": "ocsge-coverage-csv-2026",
+                "resource_title": "part_territoire_francais_couvert_par_ocsge.csv",
+            },
+        ],
+    },
+    {
+        "dataset_slug": "trajectoires-zan-2031-2026",
+        "dataset_title": "Etat des lieux national de la trajectoire de consommation d'espaces NAF (naturels, agricoles et forestiers) à horizon 2031",  # noqa: E501
+        "frequency": "@once",
+        "resources": [
+            {
+                "format": "csv",
+                "filename": "trajectoires_communes.csv",
+                "sql": "SELECT * FROM public_for_export.trajectoires_communes",
+                "resource_slug": "trajectoires-communes-csv-2026",
+                "resource_title": "trajectoires_communes.csv",
+            },
+        ],
+    },
+    # === ARTIFICIALISATION ===
+    {
+        "dataset_slug": "artificialisation-collectivites-2026",
+        "dataset_title": "Artificialisation des collectivites de France",
+        "resources": [
+            # Commune
+            {
+                "format": "csv",
+                "filename": "artif_commune.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_artif_commune",
+                "resource_slug": "artif-commune-csv-2026",
+                "resource_title": "Artificialisation par commune (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "artif_commune.gpkg",
+                "sql_to_layer_name_mapping": {
+                    "artif_commune": "SELECT * FROM public_for_export.for_export_artif_commune"
+                },
+                "resource_slug": "artif-commune-gpkg-2026",
+                "resource_title": "Artificialisation par commune (GeoPackage)",
+            },
+            # EPCI
+            {
+                "format": "csv",
+                "filename": "artif_epci.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_artif_epci",
+                "resource_slug": "artif-epci-csv-2026",
+                "resource_title": "Artificialisation par EPCI (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "artif_epci.gpkg",
+                "sql_to_layer_name_mapping": {"artif_epci": "SELECT * FROM public_for_export.for_export_artif_epci"},
+                "resource_slug": "artif-epci-gpkg-2026",
+                "resource_title": "Artificialisation par EPCI (GeoPackage)",
+            },
+            # SCOT
+            {
+                "format": "csv",
+                "filename": "artif_scot.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_artif_scot",
+                "resource_slug": "artif-scot-csv-2026",
+                "resource_title": "Artificialisation par SCOT (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "artif_scot.gpkg",
+                "sql_to_layer_name_mapping": {"artif_scot": "SELECT * FROM public_for_export.for_export_artif_scot"},
+                "resource_slug": "artif-scot-gpkg-2026",
+                "resource_title": "Artificialisation par SCOT (GeoPackage)",
+            },
+            # Departement
+            {
+                "format": "csv",
+                "filename": "artif_departement.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_artif_departement",
+                "resource_slug": "artif-departement-csv-2026",
+                "resource_title": "Artificialisation par departement (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "artif_departement.gpkg",
+                "sql_to_layer_name_mapping": {
+                    "artif_departement": "SELECT * FROM public_for_export.for_export_artif_departement"
+                },
+                "resource_slug": "artif-departement-gpkg-2026",
+                "resource_title": "Artificialisation par departement (GeoPackage)",
+            },
+            # Region
+            {
+                "format": "csv",
+                "filename": "artif_region.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_artif_region",
+                "resource_slug": "artif-region-csv-2026",
+                "resource_title": "Artificialisation par region (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "artif_region.gpkg",
+                "sql_to_layer_name_mapping": {
+                    "artif_region": "SELECT * FROM public_for_export.for_export_artif_region"
+                },
+                "resource_slug": "artif-region-gpkg-2026",
+                "resource_title": "Artificialisation par region (GeoPackage)",
+            },
+        ],
+    },
+    # === IMPERMEABILISATION ===
+    {
+        "dataset_slug": "impermeabilisation-collectivites-2026",
+        "dataset_title": "Impermeabilisation des collectivites de France",
+        "resources": [
+            # Commune
+            {
+                "format": "csv",
+                "filename": "imper_commune.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_imper_commune",
+                "resource_slug": "imper-commune-csv-2026",
+                "resource_title": "Impermeabilisation par commune (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "imper_commune.gpkg",
+                "sql_to_layer_name_mapping": {
+                    "imper_commune": "SELECT * FROM public_for_export.for_export_imper_commune"
+                },
+                "resource_slug": "imper-commune-gpkg-2026",
+                "resource_title": "Impermeabilisation par commune (GeoPackage)",
+            },
+            # EPCI
+            {
+                "format": "csv",
+                "filename": "imper_epci.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_imper_epci",
+                "resource_slug": "imper-epci-csv-2026",
+                "resource_title": "Impermeabilisation par EPCI (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "imper_epci.gpkg",
+                "sql_to_layer_name_mapping": {"imper_epci": "SELECT * FROM public_for_export.for_export_imper_epci"},
+                "resource_slug": "imper-epci-gpkg-2026",
+                "resource_title": "Impermeabilisation par EPCI (GeoPackage)",
+            },
+            # SCOT
+            {
+                "format": "csv",
+                "filename": "imper_scot.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_imper_scot",
+                "resource_slug": "imper-scot-csv-2026",
+                "resource_title": "Impermeabilisation par SCOT (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "imper_scot.gpkg",
+                "sql_to_layer_name_mapping": {"imper_scot": "SELECT * FROM public_for_export.for_export_imper_scot"},
+                "resource_slug": "imper-scot-gpkg-2026",
+                "resource_title": "Impermeabilisation par SCOT (GeoPackage)",
+            },
+            # Departement
+            {
+                "format": "csv",
+                "filename": "imper_departement.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_imper_departement",
+                "resource_slug": "imper-departement-csv-2026",
+                "resource_title": "Impermeabilisation par departement (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "imper_departement.gpkg",
+                "sql_to_layer_name_mapping": {
+                    "imper_departement": "SELECT * FROM public_for_export.for_export_imper_departement"
+                },
+                "resource_slug": "imper-departement-gpkg-2026",
+                "resource_title": "Impermeabilisation par departement (GeoPackage)",
+            },
+            # Region
+            {
+                "format": "csv",
+                "filename": "imper_region.csv",
+                "sql": "SELECT * FROM public_for_export.for_export_imper_region",
+                "resource_slug": "imper-region-csv-2026",
+                "resource_title": "Impermeabilisation par region (CSV)",
+            },
+            {
+                "format": "gpkg",
+                "filename": "imper_region.gpkg",
+                "sql_to_layer_name_mapping": {
+                    "imper_region": "SELECT * FROM public_for_export.for_export_imper_region"
+                },
+                "resource_slug": "imper-region-gpkg-2026",
+                "resource_title": "Impermeabilisation par region (GeoPackage)",
+            },
+        ],
+    },
+]