Develop

.github/workflows/deploy-to-scaleway.yml

@@ -26,13 +26,28 @@ jobs:
           SCW_USER: ${{ secrets.SCW_USER }}
         run: |
           ssh -o StrictHostKeyChecking=no $SCW_USER@$SCW_HOST << 'EOF'
-            cd /home/${{ secrets.SCW_USER }}/aigle-api
-            git pull origin ${{ github.ref_name }}
+            set -e  # This will make the script exit immediately if any command fails
+
+            cd /home/${{ secrets.SCW_USER }}/aigle-api || exit 1
+
+            # Fetch and reset to remote branch
+            git fetch origin ${{ github.ref_name }} || exit 1
+            git reset --hard origin/${{ github.ref_name }} || exit 1
+
             set -a  # Enable auto-export of variables
-            source .env
+            source .env || exit 1
             set +a # Disable auto-export of variables
-            source venv/bin/activate
-            pip install -r requirements.txt
-            python manage.py migrate
-            sudo systemctl restart gunicorn_aigle
+
+            source venv/bin/activate || exit 1
+            pip install --break-system-packages -r requirements.txt || exit 1
+            python manage.py migrate || exit 1
+            sudo systemctl restart gunicorn_aigle || exit 1
+
+            echo "Deployment completed successfully!"
           EOF
+
+          # Check the exit status of the SSH command
+          if [ $? -ne 0 ]; then
+            echo "Deployment failed!"
+            exit 1
+          fi

core/management/commands/import_detections.py

@@ -125,8 +125,8 @@ def add_arguments(self, parser):
         parser.add_argument("--clean-step", type=bool, default=False)
         parser.add_argument("--batch-id", type=str)
         parser.add_argument("--file-path", type=str)
-        parser.add_argument("--table-name", type=str)
-        parser.add_argument("--table-schema", type=str, default="import_detections")
+        parser.add_argument("--table-name", type=str, default="inference")
+        parser.add_argument("--table-schema", type=str, default="detections")
 
     def validate_arguments(self, options):
         if not options.get("file_path") and not options.get("table_name"):
@@ -483,7 +483,7 @@ def insert_detections(self, force=False):
 
         if self.total:
             print(
-                f"Inserted {self.total_inserted_detections}/{self.total} detections in total ({(self.total_inserted_detections/self.total)*100:.2f})"
+                f"Inserted {self.total_inserted_detections}/{self.total} detections in total ({(self.total_inserted_detections/self.total)*100:.2f}%)"
             )
         else:
             print(f"Inserted {

core/management/commands/import_geoepcis.py

@@ -0,0 +1,110 @@
+from typing import Any, Dict, Iterable
+from django.core.management.base import BaseCommand
+from rest_framework import serializers
+from django.db import connection
+from django.db.models import F
+
+from rest_framework_gis.serializers import GeometryField
+from django.contrib.gis.db.models.functions import Area, Intersection
+
+
+from core.models.geo_commune import GeoCommune
+from core.models.geo_department import GeoDepartment
+from core.models.geo_epci import GeoEpci
+
+PERCENTAGE_COMMUNE_INCLUDED_THRESHOLD = 0.6
+
+
+class EpciRowSerializer(serializers.Serializer):
+    id = serializers.CharField()
+    code_siren = serializers.CharField()
+    geometry = GeometryField()
+    nom = serializers.CharField()
+
+
+TABLE_COLUMNS = [col for col in EpciRowSerializer().get_fields().keys()]
+
+
+class Command(BaseCommand):
+    help = "Import EPCIs from another schema, generated from adminexpress"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--table-name", type=str, default="epci")
+        parser.add_argument("--table-schema", type=str, default="epci")
+
+    def get_rows_to_insert_from_table(
+        self, table_name: str, table_schema: str
+    ) -> Iterable[Dict[str, Any]]:
+        self.cursor = connection.cursor()
+
+        self.cursor.execute("SELECT count(*) FROM %s.%s" % (table_schema, table_name))
+        self.total = self.cursor.fetchone()[0]
+        self.cursor.execute(
+            "SELECT %s FROM %s.%s ORDER BY id"
+            % (", ".join(TABLE_COLUMNS), table_schema, table_name)
+        )
+        return map(lambda row: dict(zip(TABLE_COLUMNS, row)), self.cursor)
+
+    def handle(self, *args, **options):
+        table_name = options["table_name"]
+        table_schema = options["table_schema"]
+
+        print(f"Starting importing epcis from {table_schema}.{table_name}")
+
+        rows_to_insert = self.get_rows_to_insert_from_table(
+            table_name=table_name,
+            table_schema=table_schema,
+        )
+
+        for index, row in enumerate(rows_to_insert):
+            serializer = EpciRowSerializer(data=row)
+            if not serializer.is_valid():
+                print(f"Invalid row: {row}, errors: {serializer.errors} skipping...")
+
+            epci_serialized = serializer.validated_data
+
+            # get the department that have the biggest surface in common with epci
+            department = (
+                GeoDepartment.objects.filter(
+                    geometry__intersects=epci_serialized["geometry"]
+                )
+                .annotate(
+                    intersection_area=Area(
+                        Intersection("geometry", epci_serialized["geometry"])
+                    )
+                )
+                .order_by("-intersection_area")
+                .only("id")
+                .first()
+            )
+
+            epci = GeoEpci(
+                name=epci_serialized["nom"],
+                siren_code=epci_serialized["code_siren"],
+                geometry=epci_serialized["geometry"],
+                department_id=department.id,
+            )
+            epci.save()
+            communes = (
+                GeoCommune.objects.filter(
+                    geometry__intersects=epci_serialized["geometry"]
+                )
+                .annotate(
+                    intersection_area=Area(
+                        Intersection("geometry", epci_serialized["geometry"])
+                    ),
+                    total_area=Area("geometry"),
+                    intersection_percentage=F("intersection_area") / F("total_area"),
+                )
+                .filter(
+                    intersection_percentage__gte=PERCENTAGE_COMMUNE_INCLUDED_THRESHOLD
+                )
+            )
+            for commune in communes:
+                commune.epci_id = epci.id
+
+            GeoCommune.objects.bulk_update(communes, ["epci_id"])
+
+            print(f"EPCIs inserted: {index+1}/{self.total}")
+
+        self.cursor.close()

core/management/commands/import_pvs.py

@@ -8,7 +8,7 @@
 
 from aigle.settings import PASSWORD_MIN_LENGTH
 from core.models.user import User, UserRole
-from core.utils.string import normalize, slugify
+from core.utils.string import slugify
 
 from core.models.detection_data import DetectionControlStatus
 from core.models.parcel import Parcel
@@ -26,7 +26,7 @@
 
 
 class PvRow(TypedDict):
-    COMMUNE: str
+    COMMUNE: Optional[str]
     CODE_INSEE: str
     REF_CADAST: str
     DATE_PV: Optional[str]
@@ -81,10 +81,7 @@ def handle(self, *args, **options):
                 continue
 
             parcel = (
-                Parcel.objects.filter(
-                    Q(commune__iso_code=row["CODE_INSEE"])
-                    | Q(commune__name_normalized=normalize(row["COMMUNE"]))
-                )
+                Parcel.objects.filter(Q(commune__iso_code=row["CODE_INSEE"]))
                 .filter(
                     section=cadastre_letters,
                     num_parcel=cadastre_numbers,

core/management/commands/update_custom_zones.py

@@ -2,6 +2,7 @@
 
 from core.models.detection import Detection
 from core.models.geo_custom_zone import GeoCustomZone
+from core.models.geo_sub_custom_zone import GeoSubCustomZone
 from core.models.tile_set import TileSet, TileSetStatus, TileSetType
 
 
@@ -23,7 +24,12 @@ def handle(self, *args, **options):
         if zones_uuids:
             custom_zones_queryset = custom_zones_queryset.filter(uuid__in=zones_uuids)
 
-        custom_zones = custom_zones_queryset.filter(geometry__isnull=False).all()
+        custom_zones = (
+            custom_zones_queryset.filter(geometry__isnull=False)
+            .prefetch_related("sub_custom_zones")
+            .defer("geometry", "sub_custom_zones__geometry")
+            .all()
+        )
 
         print(
             f"Starting updating detection data for zones: {", ".join([zone.name for zone in custom_zones])}"
@@ -49,33 +55,75 @@ def handle(self, *args, **options):
 
             GeoCustomZone.objects.raw(
                 """
-            insert into core_detectionobject_geo_custom_zones(
-                    detectionobject_id,
-                    geocustomzone_id
-                )
-            select
-                distinct
-                dobj.id as detectionobject_id,
-                %s as geocustomzone_id
-            from
-                core_detectionobject dobj
-            join core_detection detec on
-                detec.detection_object_id = dobj.id
-            WHERE
-                detec.batch_id = ANY(%s) and
-                detec.tile_set_id = ANY(%s) and
-                ST_Intersects(
-                    detec.geometry,
-                    (
+                    insert into core_detectionobject_geo_custom_zones(
+                            detectionobject_id,
+                            geocustomzone_id
+                        )
+                    select
+                        distinct
+                        dobj.id as detectionobject_id,
+                        %s as geocustomzone_id
+                    from
+                        core_detectionobject dobj
+                    join core_detection detec on
+                        detec.detection_object_id = dobj.id
+                    WHERE
+                        detec.batch_id = ANY(%s) and
+                        detec.tile_set_id = ANY(%s) and
+                        ST_Intersects(
+                            detec.geometry,
+                            (
+                                select
+                                    geozone.geometry
+                                from
+                                    core_geozone geozone
+                                where
+                                    id = %s
+                            )
+                        )
+                    on conflict do nothing;
+            """,
+                [zone.id, batch_uuids, tile_set_uuids, zone.id],
+            )
+
+        sub_custom_zones = [
+            sub_custom_zone
+            for zone in custom_zones
+            for sub_custom_zone in zone.sub_custom_zones
+        ]
+
+        for zone in sub_custom_zones:
+            print(f"Updating detection data for sub-zone: {zone.name}")
+
+            GeoSubCustomZone.objects.raw(
+                """
+                    insert into core_detectionobject_geo_sub_custom_zones(
+                            detectionobject_id,
+                            geosubcustomzone_id
+                        )
                     select
-                        geozone.geometry
+                        distinct
+                        dobj.id as detectionobject_id,
+                        %s as geosubcustomzone_id
                     from
-                        core_geozone geozone
-                    where
-                        id = %s
-                    )
-                )
-            on conflict do nothing;
+                        core_detectionobject dobj
+                    join core_detection detec on
+                        detec.detection_object_id = dobj.id
+                    WHERE
+                        detec.batch_id = ANY(%s) and
+                        detec.tile_set_id = ANY(%s) and
+                        ST_Intersects(
+                            detec.geometry,
+                            (
+                                select
+                                    geosubcustomzone.geometry
+                                from
+                                    core_geosubcustomzone geosubcustomzone
+                                where
+                                    id = %s
+                            )
+                        )
+                    on conflict do nothing;
             """,
                 [zone.id, batch_uuids, tile_set_uuids, zone.id],
             )

core/migrations/0096_alter_geozone_geo_zone_type_geosubcustomzone.py

@@ -0,0 +1,60 @@
+# Generated by Django 5.0.6 on 2025-05-20 13:11
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0095_detectionobject_commune_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="geozone",
+            name="geo_zone_type",
+            field=models.CharField(
+                choices=[
+                    ("COMMUNE", "COMMUNE"),
+                    ("DEPARTMENT", "DEPARTMENT"),
+                    ("REGION", "REGION"),
+                    ("CUSTOM", "CUSTOM"),
+                    ("SUB_CUSTOM", "SUB_CUSTOM"),
+                ],
+                editable=False,
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="GeoSubCustomZone",
+            fields=[
+                (
+                    "geozone_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="core.geozone",
+                    ),
+                ),
+                (
+                    "custom_zone",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="sub_custom_zones",
+                        to="core.geocustomzone",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["geozone_ptr"], name="core_geosub_geozone_7bdbef_idx"
+                    )
+                ],
+            },
+            bases=("core.geozone",),
+        ),
+    ]