Skip to content

Commit c949713

Browse files
alexisigalexisig
authored
Merge pull request #1444 from MTES-MCT/staging
MEP v12
2 parents f8f9df6 + 4cb80ca commit c949713

196 files changed

Lines changed: 7805 additions & 3253 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Pipfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ docxtpl = "*"
5757
geopandas = "*"
5858
gunicorn = "*"
5959
jenkspy = "*"
60-
josepy = "==1.15.0"
60+
josepy = "*"
6161
mapclassify = "*"
6262
matplotlib = "*"
6363
matplotlib-scalebar = "*"
@@ -74,4 +74,4 @@ django-webpack-loader = "*"
7474
django-two-factor-auth = "*"
7575
qrcode = "*"
7676
phonenumbers = "*"
77-
mozilla-django-oidc = "*"
77+
mozilla-django-oidc = "5.0.2"

‎Pipfile.lock‎

Lines changed: 1275 additions & 1244 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎airflow/dags/data_gouv/Config.py‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,61 @@
22

33

44
@dataclass(frozen=True, kw_only=True)
5-
class BaseConfig:
6-
frequency: str = "@once"
7-
dag_id: str
5+
class ResourceConfig:
86
format: str
97
filename: str
10-
data_gouv_dataset: str
11-
data_gouv_resource: str
8+
resource_slug: str
9+
resource_title: str
1210

1311

1412
@dataclass(frozen=True, kw_only=True)
15-
class GeopackageConfig(BaseConfig):
13+
class GeopackageResourceConfig(ResourceConfig):
1614
sql_to_layer_name_mapping: dict
1715

1816

1917
@dataclass(frozen=True, kw_only=True)
20-
class CSVConfig(BaseConfig):
18+
class CSVResourceConfig(ResourceConfig):
2119
sql: str
2220

2321

22+
@dataclass(frozen=True, kw_only=True)
23+
class DatasetConfig:
24+
dataset_slug: str
25+
dataset_title: str
26+
frequency: str = "@once"
27+
resources: tuple[ResourceConfig, ...] = ()
28+
29+
2430
class ConfigParser:
25-
def __get_config_class(self, format: str):
31+
def __get_resource_config_class(self, format: str):
2632
parser_map = {
27-
"csv": CSVConfig,
28-
"gpkg": GeopackageConfig,
33+
"csv": CSVResourceConfig,
34+
"gpkg": GeopackageResourceConfig,
2935
}
3036

3137
if format not in parser_map:
3238
raise ValueError(f"Le format {format} n'est pas pris en charge")
3339

3440
return parser_map[format]
3541

36-
def parse_config(self, data: dict):
42+
def parse_resource(self, data: dict) -> ResourceConfig:
3743
if "format" not in data:
3844
raise ValueError("Le format est manquant dans la configuration", data)
3945

40-
return self.__get_config_class(data["format"])(**data)
46+
return self.__get_resource_config_class(data["format"])(**data)
47+
48+
def parse_config(self, data: dict) -> DatasetConfig:
49+
if "dataset_title" not in data:
50+
raise ValueError("Le dataset_title est manquant dans la configuration", data)
51+
52+
if "dataset_slug" not in data:
53+
raise ValueError("Le dataset_slug est manquant dans la configuration", data)
54+
55+
resources = tuple(self.parse_resource(resource_data) for resource_data in data.get("resources", []))
56+
57+
return DatasetConfig(
58+
dataset_slug=data["dataset_slug"],
59+
dataset_title=data["dataset_title"],
60+
frequency=data.get("frequency", "@once"),
61+
resources=resources,
62+
)

‎airflow/dags/data_gouv/__init__.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
1-
from .csv_exports import * # noqa: F403, F401
2-
from .geopackage_exports import * # noqa: F403, F401
3-
from .utils import get_configs # noqa: F403, F401
1+
from .export_all_to_data_gouv import * # noqa: F403, F401
2+
from .utils import get_dataset_configs # noqa: F403, F401

‎airflow/dags/data_gouv/archive_all_from_data_gouv.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""
2+
DAG pour supprimer tous les datasets de data.gouv.fr.
3+
4+
Ce DAG récupère tous les datasets de l'organisation sur data.gouv.fr
5+
et les supprime.
6+
"""
7+
8+
import pendulum
9+
from include.container import DomainContainer as Container
10+
11+
from airflow.decorators import dag, task
12+
13+
14+
@dag(
15+
dag_id="archive_all_from_data_gouv",
16+
start_date=pendulum.datetime(2024, 1, 1),
17+
schedule="@once",
18+
catchup=False,
19+
doc_md=__doc__,
20+
max_active_runs=1,
21+
default_args={"owner": "Alexis Athlani", "retries": 3},
22+
max_active_tasks=10,
23+
tags=["data_gouv", "archive"],
24+
)
25+
def archive_all_from_data_gouv():
26+
@task.python
27+
def get_all_dataset_ids() -> list[str]:
28+
"""Récupère tous les IDs de datasets depuis data.gouv.fr."""
29+
datasets = Container().data_gouv().get_organization_datasets()
30+
return [ds["id"] for ds in datasets]
31+
32+
@task.python
33+
def delete_dataset(dataset_id: str) -> str:
34+
"""Supprime un dataset de data.gouv.fr."""
35+
Container().data_gouv().delete_dataset(dataset_id)
36+
return dataset_id
37+
38+
dataset_ids = get_all_dataset_ids()
39+
delete_dataset.expand(dataset_id=dataset_ids)
40+
41+
42+
archive_all_from_data_gouv()

‎airflow/dags/data_gouv/configs.py‎

Lines changed: 206 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,206 @@
1-
from os import getenv
2-
3-
if getenv("ENVIRONMENT") == "production":
4-
configs = [
5-
{
6-
"dag_id": "create_and_publish_france_impermeabilisation_communes_geopackage",
7-
"format": "gpkg",
8-
"filename": "france_impermeabilisation_communes.gpkg",
9-
"sql_to_layer_name_mapping": {
10-
"impermeabilisation": "SELECT * FROM public_for_export.commune_imper_by_year"
11-
},
12-
"data_gouv_dataset": "67bddc715e3badfc49d5df9b",
13-
"data_gouv_resource": "b0bbf493-a078-46eb-b901-7d0754bf6e77",
14-
},
15-
{
16-
"dag_id": "create_and_publish_france_impermeabilisation_communes_csv",
17-
"format": "csv",
18-
"filename": "france_impermeabilisation_communes.csv",
19-
"sql": "SELECT * FROM public_for_export.commune_imper_by_year",
20-
"data_gouv_dataset": "67bddc715e3badfc49d5df9b",
21-
"data_gouv_resource": "800c36e1-8c12-46af-b495-4689edc40c3b",
22-
},
23-
{
24-
"dag_id": "create_and_publish_france_artificialisation_communes_geopackage",
25-
"format": "gpkg",
26-
"filename": "france_artificialisation_communes.gpkg",
27-
"sql_to_layer_name_mapping": {
28-
"impermeabilisation": "SELECT * FROM public_for_export.commune_artif_by_year"
29-
},
30-
"data_gouv_dataset": "67daf7413e4c8cbd0ff2da24",
31-
"data_gouv_resource": "7bf29725-6cdc-4f68-a10f-b9a99bf6365e",
32-
},
33-
{
34-
"dag_id": "create_and_publish_part_territoire_francais_couvert_par_ocsge_csv",
35-
"format": "csv",
36-
"filename": "part_territoire_francais_couvert_par_ocsge.csv",
37-
"sql": "SELECT * FROM public_for_export.ocsge_surface_cover_percent",
38-
"data_gouv_dataset": "68348c9afa867945df5968e5",
39-
"data_gouv_resource": "3a37cc78-4aa7-4a8e-814f-4e3714eb9a90",
40-
"frequency": "@weekly",
41-
},
42-
{
43-
"dag_id": "create_and_publish_trajectoires_communes_csv",
44-
"format": "csv",
45-
"filename": "trajectoires_communes.csv",
46-
"sql": "SELECT * FROM public_for_export.trajectoires_communes",
47-
"data_gouv_dataset": "688a1dcd9eaa8e9b2bd6bfc6",
48-
"data_gouv_resource": "e4fa6a1d-48f1-44dd-bde9-b3c7814487bd",
49-
"frequency": "@once",
50-
},
51-
]
52-
else:
53-
configs = []
1+
configs = [
2+
{
3+
"dataset_slug": "ocsge-coverage-2026",
4+
"dataset_title": "Part du territoire Francais couvert par OCS GE",
5+
"frequency": "@weekly",
6+
"resources": [
7+
{
8+
"format": "csv",
9+
"filename": "part_territoire_francais_couvert_par_ocsge.csv",
10+
"sql": "SELECT * FROM public_for_export.ocsge_surface_cover_percent",
11+
"resource_slug": "ocsge-coverage-csv-2026",
12+
"resource_title": "part_territoire_francais_couvert_par_ocsge.csv",
13+
},
14+
],
15+
},
16+
{
17+
"dataset_slug": "trajectoires-zan-2031-2026",
18+
"dataset_title": "Etat des lieux national de la trajectoire de consommation d'espaces NAF (naturels, agricoles et forestiers) à horizon 2031", # noqa: E501
19+
"frequency": "@once",
20+
"resources": [
21+
{
22+
"format": "csv",
23+
"filename": "trajectoires_communes.csv",
24+
"sql": "SELECT * FROM public_for_export.trajectoires_communes",
25+
"resource_slug": "trajectoires-communes-csv-2026",
26+
"resource_title": "trajectoires_communes.csv",
27+
},
28+
],
29+
},
30+
# === ARTIFICIALISATION ===
31+
{
32+
"dataset_slug": "artificialisation-collectivites-2026",
33+
"dataset_title": "Artificialisation des collectivites de France",
34+
"resources": [
35+
# Commune
36+
{
37+
"format": "csv",
38+
"filename": "artif_commune.csv",
39+
"sql": "SELECT * FROM public_for_export.for_export_artif_commune",
40+
"resource_slug": "artif-commune-csv-2026",
41+
"resource_title": "Artificialisation par commune (CSV)",
42+
},
43+
{
44+
"format": "gpkg",
45+
"filename": "artif_commune.gpkg",
46+
"sql_to_layer_name_mapping": {
47+
"artif_commune": "SELECT * FROM public_for_export.for_export_artif_commune"
48+
},
49+
"resource_slug": "artif-commune-gpkg-2026",
50+
"resource_title": "Artificialisation par commune (GeoPackage)",
51+
},
52+
# EPCI
53+
{
54+
"format": "csv",
55+
"filename": "artif_epci.csv",
56+
"sql": "SELECT * FROM public_for_export.for_export_artif_epci",
57+
"resource_slug": "artif-epci-csv-2026",
58+
"resource_title": "Artificialisation par EPCI (CSV)",
59+
},
60+
{
61+
"format": "gpkg",
62+
"filename": "artif_epci.gpkg",
63+
"sql_to_layer_name_mapping": {"artif_epci": "SELECT * FROM public_for_export.for_export_artif_epci"},
64+
"resource_slug": "artif-epci-gpkg-2026",
65+
"resource_title": "Artificialisation par EPCI (GeoPackage)",
66+
},
67+
# SCOT
68+
{
69+
"format": "csv",
70+
"filename": "artif_scot.csv",
71+
"sql": "SELECT * FROM public_for_export.for_export_artif_scot",
72+
"resource_slug": "artif-scot-csv-2026",
73+
"resource_title": "Artificialisation par SCOT (CSV)",
74+
},
75+
{
76+
"format": "gpkg",
77+
"filename": "artif_scot.gpkg",
78+
"sql_to_layer_name_mapping": {"artif_scot": "SELECT * FROM public_for_export.for_export_artif_scot"},
79+
"resource_slug": "artif-scot-gpkg-2026",
80+
"resource_title": "Artificialisation par SCOT (GeoPackage)",
81+
},
82+
# Departement
83+
{
84+
"format": "csv",
85+
"filename": "artif_departement.csv",
86+
"sql": "SELECT * FROM public_for_export.for_export_artif_departement",
87+
"resource_slug": "artif-departement-csv-2026",
88+
"resource_title": "Artificialisation par departement (CSV)",
89+
},
90+
{
91+
"format": "gpkg",
92+
"filename": "artif_departement.gpkg",
93+
"sql_to_layer_name_mapping": {
94+
"artif_departement": "SELECT * FROM public_for_export.for_export_artif_departement"
95+
},
96+
"resource_slug": "artif-departement-gpkg-2026",
97+
"resource_title": "Artificialisation par departement (GeoPackage)",
98+
},
99+
# Region
100+
{
101+
"format": "csv",
102+
"filename": "artif_region.csv",
103+
"sql": "SELECT * FROM public_for_export.for_export_artif_region",
104+
"resource_slug": "artif-region-csv-2026",
105+
"resource_title": "Artificialisation par region (CSV)",
106+
},
107+
{
108+
"format": "gpkg",
109+
"filename": "artif_region.gpkg",
110+
"sql_to_layer_name_mapping": {
111+
"artif_region": "SELECT * FROM public_for_export.for_export_artif_region"
112+
},
113+
"resource_slug": "artif-region-gpkg-2026",
114+
"resource_title": "Artificialisation par region (GeoPackage)",
115+
},
116+
],
117+
},
118+
# === IMPERMEABILISATION ===
119+
{
120+
"dataset_slug": "impermeabilisation-collectivites-2026",
121+
"dataset_title": "Impermeabilisation des collectivites de France",
122+
"resources": [
123+
# Commune
124+
{
125+
"format": "csv",
126+
"filename": "imper_commune.csv",
127+
"sql": "SELECT * FROM public_for_export.for_export_imper_commune",
128+
"resource_slug": "imper-commune-csv-2026",
129+
"resource_title": "Impermeabilisation par commune (CSV)",
130+
},
131+
{
132+
"format": "gpkg",
133+
"filename": "imper_commune.gpkg",
134+
"sql_to_layer_name_mapping": {
135+
"imper_commune": "SELECT * FROM public_for_export.for_export_imper_commune"
136+
},
137+
"resource_slug": "imper-commune-gpkg-2026",
138+
"resource_title": "Impermeabilisation par commune (GeoPackage)",
139+
},
140+
# EPCI
141+
{
142+
"format": "csv",
143+
"filename": "imper_epci.csv",
144+
"sql": "SELECT * FROM public_for_export.for_export_imper_epci",
145+
"resource_slug": "imper-epci-csv-2026",
146+
"resource_title": "Impermeabilisation par EPCI (CSV)",
147+
},
148+
{
149+
"format": "gpkg",
150+
"filename": "imper_epci.gpkg",
151+
"sql_to_layer_name_mapping": {"imper_epci": "SELECT * FROM public_for_export.for_export_imper_epci"},
152+
"resource_slug": "imper-epci-gpkg-2026",
153+
"resource_title": "Impermeabilisation par EPCI (GeoPackage)",
154+
},
155+
# SCOT
156+
{
157+
"format": "csv",
158+
"filename": "imper_scot.csv",
159+
"sql": "SELECT * FROM public_for_export.for_export_imper_scot",
160+
"resource_slug": "imper-scot-csv-2026",
161+
"resource_title": "Impermeabilisation par SCOT (CSV)",
162+
},
163+
{
164+
"format": "gpkg",
165+
"filename": "imper_scot.gpkg",
166+
"sql_to_layer_name_mapping": {"imper_scot": "SELECT * FROM public_for_export.for_export_imper_scot"},
167+
"resource_slug": "imper-scot-gpkg-2026",
168+
"resource_title": "Impermeabilisation par SCOT (GeoPackage)",
169+
},
170+
# Departement
171+
{
172+
"format": "csv",
173+
"filename": "imper_departement.csv",
174+
"sql": "SELECT * FROM public_for_export.for_export_imper_departement",
175+
"resource_slug": "imper-departement-csv-2026",
176+
"resource_title": "Impermeabilisation par departement (CSV)",
177+
},
178+
{
179+
"format": "gpkg",
180+
"filename": "imper_departement.gpkg",
181+
"sql_to_layer_name_mapping": {
182+
"imper_departement": "SELECT * FROM public_for_export.for_export_imper_departement"
183+
},
184+
"resource_slug": "imper-departement-gpkg-2026",
185+
"resource_title": "Impermeabilisation par departement (GeoPackage)",
186+
},
187+
# Region
188+
{
189+
"format": "csv",
190+
"filename": "imper_region.csv",
191+
"sql": "SELECT * FROM public_for_export.for_export_imper_region",
192+
"resource_slug": "imper-region-csv-2026",
193+
"resource_title": "Impermeabilisation par region (CSV)",
194+
},
195+
{
196+
"format": "gpkg",
197+
"filename": "imper_region.gpkg",
198+
"sql_to_layer_name_mapping": {
199+
"imper_region": "SELECT * FROM public_for_export.for_export_imper_region"
200+
},
201+
"resource_slug": "imper-region-gpkg-2026",
202+
"resource_title": "Impermeabilisation par region (GeoPackage)",
203+
},
204+
],
205+
},
206+
]

0 commit comments

Comments
 (0)