Added models for Column/Filter descriptions (now they can be edited from admin page)

Author: dmytrotsko

src/fixtures/column_descriptions.json

@@ -0,0 +1 @@
+[{"model": "indicatorsets.columndescription", "pk": 18, "fields": {"name": "name", "description": "Hover over the indicator's name to see a brief description."}}, {"model": "indicatorsets.columndescription", "pk": 19, "fields": {"name": "geographic_coverage", "description": "The countries or world regions covered by this indicator.  These are typically covered at finer geographic levels / jurisdictions."}}, {"model": "indicatorsets.columndescription", "pk": 20, "fields": {"name": "geographic_levels", "description": "All the geographic levels at which this indicator is available.  Larger jurisdictions are often based on aggregation of data from constituent jurisdictions."}}, {"model": "indicatorsets.columndescription", "pk": 21, "fields": {"name": "temporal_scope_start", "description": "The earliest date for which this indicator is available."}}, {"model": "indicatorsets.columndescription", "pk": 22, "fields": {"name": "temporal_scope_end", "description": "The latest date for which this indicator is available."}}, {"model": "indicatorsets.columndescription", "pk": 23, "fields": {"name": "temporal_granularity", "description": "The temporal resolution of this indicator (not of the reporting).  Might not be the same as Reporting Cadence (e.g. a daily indicator may be reported only once a week)."}}, {"model": "indicatorsets.columndescription", "pk": 24, "fields": {"name": "reporting_cadence", "description": "The frequency with which this indicator is reported.  This may be different from the temporal granularity."}}, {"model": "indicatorsets.columndescription", "pk": 25, "fields": {"name": "reporting_lag", "description": "The number of days from the last day of a reported period until the first reported value for that period is usually available in Delphi Epidata.  E.g. if reporting U.S. epiweeks (Sunday through Saturday), and the first report is usually available in Delphi Epidata on the following Friday, The Reporting Lag is 6. By \"usually available\" we mean when it\\'s \"supposed to be\" available based on our current understanding of the data provider\\'s operations and Delphi\\'s ingestion pipeline.  That is the date on which we think of the data as showing up \"on time\", and relative to which we track unusual delays."}}, {"model": "indicatorsets.columndescription", "pk": 26, "fields": {"name": "revision_cadende", "description": "How frequently are revised values (e.g. \"backfill\") usually reported (if any)?"}}, {"model": "indicatorsets.columndescription", "pk": 27, "fields": {"name": "population", "description": "The population or demographic group reflected by the indicator (\"All\" means the entire population)"}}, {"model": "indicatorsets.columndescription", "pk": 28, "fields": {"name": "population_stratifiers", "description": "What population or demographic stratifiers are available, if any?"}}, {"model": "indicatorsets.columndescription", "pk": 29, "fields": {"name": "surveillance_categories", "description": "Which surveillance categories or rungs in the Severity Pyramid does this indicator attempt to track?  Some indicators may approximately track multiple categories."}}, {"model": "indicatorsets.columndescription", "pk": 30, "fields": {"name": "original_data_provider", "description": "The owner or supplier of the original or raw data used to create this indicator."}}, {"model": "indicatorsets.columndescription", "pk": 31, "fields": {"name": "pre_processing", "description": "Brief description of main data processing used in creating this set of indicators, including smoothing and aggregation.  For more details, see the documentation."}}, {"model": "indicatorsets.columndescription", "pk": 32, "fields": {"name": "censoring", "description": "Is any of the data being censored (e.g. small counts)?  If so how, and how much impact does it have (e.g. approximate fraction of values affected)."}}, {"model": "indicatorsets.columndescription", "pk": 33, "fields": {"name": "dua_required", "description": "Applicable data use terms (may apply even to publicly accessible indicators)."}}]

src/fixtures/filter_descriptions.json

@@ -0,0 +1 @@
+[{"model": "indicatorsets.filterdescription", "pk": 9, "fields": {"name": "pathogens", "description": "List only indicators related to these pathogens, syndromes or diseases."}}, {"model": "indicatorsets.filterdescription", "pk": 10, "fields": {"name": "geographic_scope", "description": "List only indicators that cover any of the selected countries or world regions."}}, {"model": "indicatorsets.filterdescription", "pk": 11, "fields": {"name": "geographic_levels", "description": "List only indicators that are available at any of the selected geographic levels."}}, {"model": "indicatorsets.filterdescription", "pk": 12, "fields": {"name": "severity_pyramid_rungs", "description": "List only indicators that are directly related to any of the selected rungs."}}, {"model": "indicatorsets.filterdescription", "pk": 13, "fields": {"name": "original_data_provider", "description": "List only indicator that are based on data from one of the selected sources."}}, {"model": "indicatorsets.filterdescription", "pk": 14, "fields": {"name": "temporal_granularity", "description": "The temporal resolution of this indicator (not of the reporting).  Might not be the same as Reporting Cadence (e.g. a daily indicator may be reported only once a week)."}}, {"model": "indicatorsets.filterdescription", "pk": 15, "fields": {"name": "temporal_scope_end", "description": "The latest date for which this indicator is available."}}, {"model": "indicatorsets.filterdescription", "pk": 16, "fields": {"name": "location_search", "description": "Enter one or more locations for which you are looking for indicator coverage, or leave empty for all locations.  Start entering a location name to see all compatible locations.  Auto-complete with [Tab] or [Enter].  Currently works only for U.S. locations."}}]

src/indicatorsets/admin.py

@@ -1,7 +1,12 @@
 from django.contrib import admin
 
 from import_export.admin import ImportExportModelAdmin
-from indicatorsets.models import IndicatorSet, NonDelphiIndicatorSet
+from indicatorsets.models import (
+    IndicatorSet,
+    NonDelphiIndicatorSet,
+    FilterDescription,
+    ColumnDescription,
+)
 from indicatorsets.resources import IndicatorSetResource, NonDelphiIndicatorSetResource
 
 
@@ -55,3 +60,29 @@ class NonDelphiIndicatorSetAdmin(ImportExportModelAdmin):
     search_fields = ("name", "short_name", "description")
     ordering = ["name"]
     list_filter = ["original_data_provider", "source_type"]
+
+
+@admin.register(FilterDescription)
+class FilterDescriptionAdmin(admin.ModelAdmin):
+    """
+    Admin interface for the FilterDescription model.
+    """
+
+    list_display = ("name", "description")
+    search_fields = ("name", "description")
+    ordering = ["name"]
+    list_filter = ["name"]
+    list_editable = ("description",)
+
+
+@admin.register(ColumnDescription)
+class ColumnDescriptionAdmin(admin.ModelAdmin):
+    """
+    Admin interface for the ColumnDescription model.
+    """
+
+    list_display = ("name", "description")
+    search_fields = ("name", "description")
+    ordering = ["name"]
+    list_filter = ["name"]
+    list_editable = ("description",)

src/indicatorsets/migrations/0004_columndescription_filterdescription.py

@@ -0,0 +1,41 @@
+# Generated by Django 5.0.7 on 2025-05-13 12:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('indicatorsets', '0003_indicatorset_source_type'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ColumnDescription',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, unique=True, verbose_name='Name')),
+                ('description', models.TextField(blank=True, verbose_name='Description')),
+            ],
+            options={
+                'verbose_name': 'Column Description',
+                'verbose_name_plural': 'Column Descriptions',
+                'ordering': ['name'],
+                'indexes': [models.Index(fields=['name'], name='column_description_name_idx')],
+            },
+        ),
+        migrations.CreateModel(
+            name='FilterDescription',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, unique=True, verbose_name='Name')),
+                ('description', models.TextField(blank=True, verbose_name='Description')),
+            ],
+            options={
+                'verbose_name': 'Filter Description',
+                'verbose_name_plural': 'Filter Descriptions',
+                'ordering': ['name'],
+                'indexes': [models.Index(fields=['name'], name='filter_description_name_idx')],
+            },
+        ),
+    ]

src/indicatorsets/models.py

@@ -223,3 +223,53 @@ class Meta:
         proxy = True
         verbose_name = "Non-Delphi Indicator Set"
         verbose_name_plural = "Non-Delphi Indicators Sets"
+
+
+class FilterDescription(models.Model):
+
+    name = models.CharField(verbose_name="Name", max_length=255, unique=True)
+    description: models.TextField = models.TextField(
+        verbose_name="Description", blank=True
+    )
+
+    class Meta:
+        verbose_name = "Filter Description"
+        verbose_name_plural = "Filter Descriptions"
+        ordering = ["name"]
+        indexes = [
+            models.Index(fields=["name"], name="filter_description_name_idx"),
+        ]
+
+    def __str__(self):
+        return self.name
+
+    @classmethod
+    def get_all_descriptions_as_dict(cls):
+        descriptions = cls.objects.values("name", "description")
+        return {desc["name"]: desc["description"] for desc in descriptions}
+
+
+class ColumnDescription(models.Model):
+
+    name: models.CharField = models.CharField(
+        verbose_name="Name", max_length=255, unique=True
+    )
+    description: models.TextField = models.TextField(
+        verbose_name="Description", blank=True
+    )
+
+    class Meta:
+        verbose_name = "Column Description"
+        verbose_name_plural = "Column Descriptions"
+        ordering = ["name"]
+        indexes = [
+            models.Index(fields=["name"], name="column_description_name_idx"),
+        ]
+
+    def __str__(self):
+        return self.name
+
+    @classmethod
+    def get_all_descriptions_as_dict(cls):
+        descriptions = cls.objects.values("name", "description")
+        return {desc["name"]: desc["description"] for desc in descriptions}

src/indicatorsets/views.py

@@ -2,7 +2,6 @@
 import json
 import logging
 import requests
-from bs4 import BeautifulSoup
 
 
 from django.conf import settings
@@ -12,7 +11,7 @@
 from base.models import Geography, GeographyUnit
 from indicatorsets.filters import IndicatorSetFilter
 from indicatorsets.forms import IndicatorSetFilterForm
-from indicatorsets.models import IndicatorSet
+from indicatorsets.models import IndicatorSet, FilterDescription, ColumnDescription
 from indicatorsets.utils import (
     generate_epivis_custom_title,
     generate_random_color,
@@ -21,36 +20,6 @@
 
 logger = logging.getLogger(__name__)
 
-FILTERS_DESCRIPTIONS = {
-    "pathogens": "List only indicators related to these pathogens, syndromes or diseases.",
-    "geographic_scope": "List only indicators that cover any of the selected countries or world regions.",
-    "geographic_levels": "List only indicators that are available at any of the selected geographic levels.",
-    "severity_pyramid_rungs": "List only indicators that are directly related to any of the selected rungs.",
-    "original_data_provider": "List only indicator that are based on data from one of the selected sources.",
-    "temporal_granularity": "The temporal resolution of this indicator (not of the reporting).  Might not be the same as Reporting Cadence (e.g. a daily indicator may be reported only once a week).",
-    "temporal_scope_end": "The latest date for which this indicator is available.",
-    "location_search": "Enter one or more locations for which you are looking for indicator coverage, or leave empty for all locations.  Start entering a location name to see all compatible locations.  Auto-complete with [Tab] or [Enter].  Currently works only for U.S. locations.",
-}
-
-COLUMNS_DESCRIPTIONS = {
-    "name": "Hover over the indicator's name to see a brief description.",
-    "geographic_coverage": "The countries or world regions covered by this indicator.  These are typically covered at finer geographic levels / jurisdictions.",
-    "geographic_levels": "All the geographic levels at which this indicator is available.  Larger jurisdictions are often based on aggregation of data from constituent jurisdictions.",
-    "temporal_scope_start": "The earliest date for which this indicator is available.",
-    "temporal_scope_end": "The latest date for which this indicator is available.",
-    "temporal_granularity": "The temporal resolution of this indicator (not of the reporting).  Might not be the same as Reporting Cadence (e.g. a daily indicator may be reported only once a week).",
-    "reporting_cadence": "The frequency with which this indicator is reported.  This may be different from the temporal granularity.",
-    "reporting_lag": 'The number of days from the last day of a reported period until the first reported value for that period is usually available in Delphi Epidata.  E.g. if reporting U.S. epiweeks (Sunday through Saturday), and the first report is usually available in Delphi Epidata on the following Friday, The Reporting Lag is 6. By "usually available" we mean when it\'s "supposed to be" available based on our current understanding of the data provider\'s operations and Delphi\'s ingestion pipeline.  That is the date on which we think of the data as showing up "on time", and relative to which we track unusual delays.',
-    "revision_cadende": 'How frequently are revised values (e.g. "backfill") usually reported (if any)?',
-    "population": 'The population or demographic group reflected by the indicator ("All" means the entire population)',
-    "population_stratifiers": "What population or demographic stratifiers are available, if any?",
-    "surveillance_categories": "Which surveillance categories or rungs in the Severity Pyramid does this indicator attempt to track?  Some indicators may approximately track multiple categories.",
-    "original_data_provider": "The owner or supplier of the original or raw data used to create this indicator.",
-    "pre_processing": "Brief description of main data processing used in creating this set of indicators, including smoothing and aggregation.  For more details, see the documentation.",
-    "censoring": "Is any of the data being censored (e.g. small counts)?  If so how, and how much impact does it have (e.g. approximate fraction of values affected).",
-    "dua_required": "Applicable data use terms (may apply even to publicly accessible indicators).",
-}
-
 HEADER_DESCRIPTION = "Discover, display and download real-time infectious disease indicators (time series) that track a variety of pathogens, diseases and syndromes in a variety of locations (primarily within the USA). Browse the list, or filter it first by locations and pathogens of interest, by surveillance categories, and more. Expand any row to expose and select from a set of related indicators, then hit 'Show Selected Indicators' at bottom to plot or export your selected indicators, or to generate code snippets to retrieve them from the Delphi Epidata API. Most indicators are served from the Delphi Epidata real-time repository, but some may be available only from third parties or may require prior approval."
 
 FLUVIEW_INDICATORS_MAPPING = {"wili": "%wILI", "ili": "%ILI"}
@@ -193,8 +162,12 @@ def get_context_data(self, **kwargs):
                 filter.indicators_qs, filter.qs.values_list("id", flat=True)
             )
         )
-        context["filters_descriptions"] = FILTERS_DESCRIPTIONS
-        context["columns_descriptions"] = COLUMNS_DESCRIPTIONS
+        context["filters_descriptions"] = (
+            FilterDescription.get_all_descriptions_as_dict()
+        )
+        context["columns_descriptions"] = (
+            ColumnDescription.get_all_descriptions_as_dict()
+        )
         context["header_description"] = HEADER_DESCRIPTION
         context["available_geographies"] = Geography.objects.filter(
             used_in="indicators"