Skip to content

Commit 093623e

Browse files
alexisigalexisig
authored
Merge pull request #1434 from MTES-MCT/feat-carroyage
Ajout du carroyage à la page conso
2 parents eb88a4f + 8fb8976 commit 093623e

29 files changed

Lines changed: 1632 additions & 6 deletions

File tree

‎airflow/dags/ingest_carroyage_lea.py‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
"""
2+
DAG pour ingérer le carroyage LEA de consommation d'espaces NAF.
3+
4+
Source: data.gouv.fr - Consommation d'espaces naturels, agricoles et forestiers (2009-2024)
5+
"""
6+
7+
import subprocess
8+
from zipfile import ZipFile
9+
10+
import requests
11+
from include.container import DomainContainer
12+
from include.container import InfraContainer as Container
13+
from include.pools import DBT_POOL
14+
from include.utils import (
15+
get_dbt_command_from_directory,
16+
get_first_shapefile_path_in_dir,
17+
multiline_string_to_single_line,
18+
)
19+
from pendulum import datetime
20+
21+
from airflow.decorators import dag, task
22+
23+
URL = "https://www.data.gouv.fr/api/1/datasets/r/088789cb-a9c0-4069-a4b4-7d8078911d38"
24+
TABLE_NAME = "majic_carroyage_lea"
25+
TMP_PATH = "/tmp/carroyage_lea"
26+
ZIP_FILENAME = "carroyage_lea.zip"
27+
GEOJSON_FILENAME = "carroyage_lea.geojson"
28+
PMTILES_FILENAME = "carroyage_lea.pmtiles"
29+
VECTOR_TILES_DIR = "vector_tiles"
30+
31+
32+
@dag(
33+
start_date=datetime(2026, 1, 28),
34+
schedule="@once",
35+
catchup=False,
36+
doc_md=__doc__,
37+
max_active_runs=1,
38+
default_args={"owner": "Alexis Athlani", "retries": 3},
39+
tags=["Majic", "Cerema"],
40+
)
41+
def ingest_carroyage_lea():
42+
bucket_name = Container().bucket_name()
43+
s3_key = f"majic/{ZIP_FILENAME}"
44+
localpath = f"{TMP_PATH}/{ZIP_FILENAME}"
45+
46+
@task.python
47+
def download() -> str:
48+
"""Télécharge le fichier zip depuis data.gouv.fr et l'upload sur S3."""
49+
import os
50+
51+
os.makedirs(TMP_PATH, exist_ok=True)
52+
53+
response = requests.get(URL, allow_redirects=True)
54+
response.raise_for_status()
55+
56+
with open(localpath, "wb") as f:
57+
f.write(response.content)
58+
59+
Container().s3().put_file(localpath, f"{bucket_name}/{s3_key}")
60+
return s3_key
61+
62+
@task.python
63+
def ingest() -> None:
64+
"""Extrait le shapefile et l'ingère dans PostgreSQL."""
65+
import os
66+
import shutil
67+
68+
os.makedirs(TMP_PATH, exist_ok=True)
69+
70+
# Download from S3
71+
Container().s3().get_file(f"{bucket_name}/{s3_key}", localpath)
72+
73+
# Extract zip
74+
extract_path = f"{TMP_PATH}/extracted"
75+
with ZipFile(localpath, "r") as zip_file:
76+
zip_file.extractall(extract_path)
77+
78+
# Find shapefile
79+
shapefile_path = get_first_shapefile_path_in_dir(extract_path)
80+
81+
# Load to PostgreSQL with ogr2ogr (source is in EPSG:3035)
82+
cmd = [
83+
"ogr2ogr",
84+
"-f",
85+
'"PostgreSQL"',
86+
f'"{Container().gdal_dbt_conn().encode()}"',
87+
"-overwrite",
88+
"-lco",
89+
"GEOMETRY_NAME=geom",
90+
"-lco",
91+
"SRID=3035",
92+
"-a_srs",
93+
"EPSG:3035",
94+
"-nlt",
95+
"MULTIPOLYGON",
96+
"-nlt",
97+
"PROMOTE_TO_MULTI",
98+
"-nln",
99+
TABLE_NAME,
100+
shapefile_path,
101+
"--config",
102+
"PG_USE_COPY",
103+
"YES",
104+
]
105+
subprocess.run(" ".join(cmd), shell=True, check=True)
106+
107+
# Cleanup
108+
shutil.rmtree(TMP_PATH)
109+
110+
@task.bash(retries=0, trigger_rule="all_success", pool=DBT_POOL)
111+
def dbt_build():
112+
return get_dbt_command_from_directory(cmd="dbt build -s carroyage_lea for_vector_tiles_carroyage_lea")
113+
114+
@task.python
115+
def postgis_to_geojson():
116+
"""Exporte les données vers GeoJSONSeq."""
117+
sql = """
118+
SELECT
119+
*
120+
FROM
121+
public_for_vector_tiles.for_vector_tiles_carroyage_lea
122+
"""
123+
return (
124+
DomainContainer()
125+
.sql_to_geojsonseq_on_s3_handler()
126+
.export_sql_result_to_geojsonseq_on_s3(
127+
sql=multiline_string_to_single_line(sql),
128+
s3_key=f"{VECTOR_TILES_DIR}/{GEOJSON_FILENAME}",
129+
s3_bucket=bucket_name,
130+
)
131+
)
132+
133+
@task.bash
134+
def geojson_to_pmtiles():
135+
"""Convertit le GeoJSON en PMTiles avec tippecanoe."""
136+
local_input = f"/tmp/{GEOJSON_FILENAME}"
137+
local_output = f"/tmp/{PMTILES_FILENAME}"
138+
Container().s3().get_file(f"{bucket_name}/{VECTOR_TILES_DIR}/{GEOJSON_FILENAME}", local_input)
139+
140+
cmd = [
141+
"tippecanoe",
142+
"-o",
143+
local_output,
144+
local_input,
145+
"--read-parallel",
146+
"--force",
147+
"--no-simplification-of-shared-nodes",
148+
"--no-tiny-polygon-reduction",
149+
"--no-line-simplification",
150+
"--no-feature-limit",
151+
"--no-tile-size-limit",
152+
"--detect-shared-borders",
153+
"--extra-detail=15",
154+
"-z16",
155+
]
156+
return " ".join(cmd)
157+
158+
@task.python
159+
def upload_pmtiles():
160+
"""Upload le fichier PMTiles sur S3."""
161+
local_path = f"/tmp/{PMTILES_FILENAME}"
162+
path_on_s3 = f"{bucket_name}/{VECTOR_TILES_DIR}/{PMTILES_FILENAME}"
163+
Container().s3().put(local_path, path_on_s3)
164+
165+
@task.bash
166+
def cleanup():
167+
"""Supprime les fichiers temporaires."""
168+
return f"rm -f /tmp/{GEOJSON_FILENAME} /tmp/{PMTILES_FILENAME}"
169+
170+
@task.python
171+
def make_pmtiles_public():
172+
"""Rend le fichier PMTiles accessible publiquement."""
173+
pmtiles_key = f"{VECTOR_TILES_DIR}/{PMTILES_FILENAME}"
174+
s3_handler = DomainContainer().s3_handler()
175+
s3_handler.set_key_publicly_visible(pmtiles_key, bucket_name)
176+
177+
(
178+
download()
179+
>> ingest()
180+
>> dbt_build()
181+
>> postgis_to_geojson()
182+
>> geojson_to_pmtiles()
183+
>> upload_pmtiles()
184+
>> cleanup()
185+
>> make_pmtiles_public()
186+
)
187+
188+
189+
ingest_carroyage_lea()

‎airflow/include/sql/sparte/models/for_vector_tiles/for_vector_tiles_carroyage_lea.sql‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
{{
2+
config(
3+
materialized="table",
4+
)
5+
}}
6+
7+
SELECT
8+
carroyage.idcarreau,
9+
-- 2011
10+
carroyage.conso_2011,
11+
carroyage.conso_2011_habitat,
12+
carroyage.conso_2011_activite,
13+
carroyage.conso_2011_mixte,
14+
carroyage.conso_2011_route,
15+
carroyage.conso_2011_ferroviaire,
16+
carroyage.conso_2011_inconnu,
17+
-- 2012
18+
carroyage.conso_2012,
19+
carroyage.conso_2012_habitat,
20+
carroyage.conso_2012_activite,
21+
carroyage.conso_2012_mixte,
22+
carroyage.conso_2012_route,
23+
carroyage.conso_2012_ferroviaire,
24+
carroyage.conso_2012_inconnu,
25+
-- 2013
26+
carroyage.conso_2013,
27+
carroyage.conso_2013_habitat,
28+
carroyage.conso_2013_activite,
29+
carroyage.conso_2013_mixte,
30+
carroyage.conso_2013_route,
31+
carroyage.conso_2013_ferroviaire,
32+
carroyage.conso_2013_inconnu,
33+
-- 2014
34+
carroyage.conso_2014,
35+
carroyage.conso_2014_habitat,
36+
carroyage.conso_2014_activite,
37+
carroyage.conso_2014_mixte,
38+
carroyage.conso_2014_route,
39+
carroyage.conso_2014_ferroviaire,
40+
carroyage.conso_2014_inconnu,
41+
-- 2015
42+
carroyage.conso_2015,
43+
carroyage.conso_2015_habitat,
44+
carroyage.conso_2015_activite,
45+
carroyage.conso_2015_mixte,
46+
carroyage.conso_2015_route,
47+
carroyage.conso_2015_ferroviaire,
48+
carroyage.conso_2015_inconnu,
49+
-- 2016
50+
carroyage.conso_2016,
51+
carroyage.conso_2016_habitat,
52+
carroyage.conso_2016_activite,
53+
carroyage.conso_2016_mixte,
54+
carroyage.conso_2016_route,
55+
carroyage.conso_2016_ferroviaire,
56+
carroyage.conso_2016_inconnu,
57+
-- 2017
58+
carroyage.conso_2017,
59+
carroyage.conso_2017_habitat,
60+
carroyage.conso_2017_activite,
61+
carroyage.conso_2017_mixte,
62+
carroyage.conso_2017_route,
63+
carroyage.conso_2017_ferroviaire,
64+
carroyage.conso_2017_inconnu,
65+
-- 2018
66+
carroyage.conso_2018,
67+
carroyage.conso_2018_habitat,
68+
carroyage.conso_2018_activite,
69+
carroyage.conso_2018_mixte,
70+
carroyage.conso_2018_route,
71+
carroyage.conso_2018_ferroviaire,
72+
carroyage.conso_2018_inconnu,
73+
-- 2019
74+
carroyage.conso_2019,
75+
carroyage.conso_2019_habitat,
76+
carroyage.conso_2019_activite,
77+
carroyage.conso_2019_mixte,
78+
carroyage.conso_2019_route,
79+
carroyage.conso_2019_ferroviaire,
80+
carroyage.conso_2019_inconnu,
81+
-- 2020
82+
carroyage.conso_2020,
83+
carroyage.conso_2020_habitat,
84+
carroyage.conso_2020_activite,
85+
carroyage.conso_2020_mixte,
86+
carroyage.conso_2020_route,
87+
carroyage.conso_2020_ferroviaire,
88+
carroyage.conso_2020_inconnu,
89+
-- 2021
90+
carroyage.conso_2021,
91+
carroyage.conso_2021_habitat,
92+
carroyage.conso_2021_activite,
93+
carroyage.conso_2021_mixte,
94+
carroyage.conso_2021_route,
95+
carroyage.conso_2021_ferroviaire,
96+
carroyage.conso_2021_inconnu,
97+
-- 2022
98+
carroyage.conso_2022,
99+
carroyage.conso_2022_habitat,
100+
carroyage.conso_2022_activite,
101+
carroyage.conso_2022_mixte,
102+
carroyage.conso_2022_route,
103+
carroyage.conso_2022_ferroviaire,
104+
carroyage.conso_2022_inconnu,
105+
-- 2023
106+
carroyage.conso_2023,
107+
carroyage.conso_2023_habitat,
108+
carroyage.conso_2023_activite,
109+
carroyage.conso_2023_mixte,
110+
carroyage.conso_2023_route,
111+
carroyage.conso_2023_ferroviaire,
112+
carroyage.conso_2023_inconnu,
113+
-- Géométrie (transformée de EPSG:3035 vers EPSG:4326)
114+
st_transform(geom, 4326) as geom,
115+
-- Territoires
116+
communes.communes as "{{ var('COMMUNE')}}",
117+
communes.epcis as "{{ var('EPCI')}}",
118+
communes.departements as "{{ var('DEPARTEMENT')}}",
119+
communes.regions as "{{ var('REGION')}}",
120+
communes.scots as "{{ var('SCOT')}}",
121+
custom_lands.custom_lands as "{{ var('CUSTOM')}}"
122+
FROM
123+
{{ ref("carroyage_lea") }} as carroyage
124+
LEFT JOIN LATERAL (
125+
SELECT
126+
array_agg(DISTINCT commune.code) as communes,
127+
array_agg(DISTINCT commune.epci) as epcis,
128+
array_agg(DISTINCT commune.departement) as departements,
129+
array_agg(DISTINCT commune.region) as regions,
130+
array_agg(DISTINCT commune.scot) as scots
131+
FROM
132+
{{ ref('commune') }} as commune
133+
WHERE
134+
st_intersects(commune.geom_4326, st_transform(st_setsrid(carroyage.geom, 3035), 4326))
135+
) communes ON TRUE
136+
LEFT JOIN LATERAL (
137+
SELECT array_agg(DISTINCT ccl.custom_land_id) as custom_lands
138+
FROM
139+
{{ ref('commune_custom_land') }} as ccl
140+
WHERE
141+
ccl.commune_code = ANY(communes.communes)
142+
) custom_lands ON TRUE
143+
WHERE
144+
carroyage.geom IS NOT NULL

0 commit comments

Comments
 (0)