Support downloading a DMR file in earthdata-varinfo (#60)

eni-awowale · web-flow · commit 988e4ba7f49f · 2025-07-25T15:32:31.000-04:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,5 @@
-## v3.1.1
-### Unreleased
+## v3.2.0
+### 2025-07-25
 
 ### Changed:
 
@@ -11,6 +11,11 @@
   rule. There is no filtering based on the variable path of the requested
   variables in the input set to `VarInfoBase.get_required_variables`.
 
+### Added:
+
+* Support for getting an OPeNDAP url with `cmr_search.get_dmr_xml_url` and
+  a `use_dmr=True` flag to `varinfo/generate_umm_var.generate_collection_umm_var`
+
 ## v3.1.0
 ### 2025-03-25
 
diff --git a/README.md b/README.md
@@ -120,6 +120,16 @@ umm_var_json = generate_collection_umm_var(<Production collection concept ID>,
 umm_var_json = generate_collection_umm_var(<UAT collection concept ID>,
                                            <authorization header>,
                                            publish=True)
+
+# Use a DMR file to generate UMM-Var, defaults to UAT, and not to publish:
+umm_var_json = generate_collection_umm_var(<UAT collection concept ID>,
+                                           <authorization header>)
+
+# To generate and publish records from a DMR file for a UAT collection
+# (note the authorization header must contain a LaunchPad token):
+umm_var_json = generate_collection_umm_var(<UAT collection concept ID>,
+                                           <authorization header>,
+                                           publish=True, use_dmr=True)
 ```
 
 Expected outputs:
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-3.1.0
+3.2.0
diff --git a/tests/unit/test_cmr_search.py b/tests/unit/test_cmr_search.py
@@ -11,6 +11,7 @@
 from varinfo.cmr_search import (
     get_granules,
     get_granule_link,
+    get_dmr_xml_url,
     download_granule,
     get_edl_token_from_launchpad,
     get_edl_token_header,
@@ -519,3 +520,120 @@ def test_get_edl_token_header_with_edl_token(self):
         """
         test_bearer_token = get_edl_token_header(self.bearer_token_header, CMR_UAT)
         self.assertEqual(test_bearer_token, self.bearer_token_header)
+
+    def test_get_dmr_xml_url_raises(self):
+        """Check if MissingGranuleDownloadLinks is raised with get_dmr_xml_url,
+        when the "links" attribute in the "RelatedUrls" of a granule response don't contain
+        the correct fields for an OPeNDAP service url.
+        """
+
+        # Nested dict does not contain key `links`
+        granule_response_no_links = [
+            {'no_links': [{'rel': 'http://esipfed.org/ns/fedsearch/1.1/s3#'}]}
+        ]
+
+        # Nested dict contains `links` but `rel` does not end in '/service#'
+        granule_response_links_no_rel = [
+            {'links': [{'rel': 'http://cool-science-data.#'}]}
+        ]
+
+        # Nested dict contains "inherited" key
+        granule_response_links_rel_inherit = [
+            {
+                'links': [
+                    {
+                        'rel': 'http://cool-science-data/1.1/service#',
+                        'href': 'http://cool-science-data/data/.nc4',
+                        'inherited': True,
+                    }
+                ]
+            }
+        ]
+
+        granule_response_links_empty = [{'links': []}]
+
+        granule_response_links_no_opendap = [
+            {
+                'links': [
+                    {
+                        'rel': 'http://cool-science-data/1.1/service#',
+                        'title': 'OPeNDAP request URL (GET DATA : OPENDAP DATA)',
+                        'hreflang': 'en-US',
+                        'href': 'https://fake.earthdata.nasa.gov/example.nc4',
+                    }
+                ]
+            }
+        ]
+
+        with self.subTest('Granule has no `links` key'):
+            with self.assertRaises(MissingGranuleDownloadLinks) as context_manager:
+                get_dmr_xml_url(granule_response_no_links)
+            self.assertEqual(
+                f'No links for granule record: {str(granule_response_no_links)}',
+                str(context_manager.exception),
+            )
+
+        with self.subTest(
+            'Granule has `links`' 'but `rel` does not end with `/service#`'
+        ):
+            with self.assertRaises(MissingGranuleDownloadLinks):
+                get_dmr_xml_url(granule_response_links_no_rel)
+
+        with self.subTest(
+            'Granule has `links` and `rel` ends in `/service#`, but contains `inherited` field.'
+        ):
+            with self.assertRaises(MissingGranuleDownloadLinks):
+                get_dmr_xml_url(granule_response_links_rel_inherit)
+
+        with self.subTest('Granule has `links` but it is an empty list'):
+            with self.assertRaises(MissingGranuleDownloadLinks):
+                get_dmr_xml_url(granule_response_links_empty)
+
+        with self.subTest(
+            'Granule has `links` and `rel` ends in `/service#`, but opendap is NOT in url.'
+        ):
+            with self.assertRaises(MissingGranuleDownloadLinks):
+                get_dmr_xml_url(granule_response_links_no_opendap)
+
+    def test_get_dmr_xml_url(self):
+        """Assert the correct OPeNDAP url is returned for a good granule response
+        (i.e. when the "links" attribute in the "RelatedUrls" of a granule response has
+        the correct fields for an OPeNDAP service url).
+        """
+        granule_response_links_correct_cloud_opendap = [
+            {
+                'links': [
+                    {
+                        'rel': 'http://cool-science-data/1.1/service#',
+                        'title': 'OPeNDAP request URL (GET DATA : OPENDAP DATA)',
+                        'hreflang': 'en-US',
+                        'href': 'https://fake.opendap.earthdata.nasa.gov/example.hdf',
+                    }
+                ]
+            }
+        ]
+
+        granule_response_links_correct_onprem_opendap = [
+            {
+                'links': [
+                    {
+                        'rel': 'http://cool-science-data/1.1/service#',
+                        'type': 'application/x-hdf',
+                        'title': 'OPeNDAP request URL (GET DATA : OPENDAP DATA)',
+                        'hreflang': 'en-US',
+                        'href': 'https://some.fake.server.onprem.opendap.nasa.gov/example.hdf',
+                    }
+                ]
+            }
+        ]
+        with self.subTest('Cloud OPeNDAP Related Urls response'):
+            self.assertEqual(
+                get_dmr_xml_url(granule_response_links_correct_cloud_opendap),
+                'https://fake.opendap.earthdata.nasa.gov/example.hdf.dmr.xml',
+            )
+
+        with self.subTest('Onprem OPeNDAP Related Urls response'):
+            self.assertEqual(
+                get_dmr_xml_url(granule_response_links_correct_onprem_opendap),
+                'https://some.fake.server.onprem.opendap.nasa.gov/example.hdf.dmr.xml',
+            )
diff --git a/tests/unit/test_generate_umm_var.py b/tests/unit/test_generate_umm_var.py
@@ -20,6 +20,11 @@ def setUpClass(cls):
         cls.launchpad_token_header = 'launchpad-token'
         cls.netcdf4_basename = 'f16_ssmis_20210426v7.nc'
         cls.netcdf4_url = f'https://example.com/{cls.netcdf4_basename}'
+        cls.example_dmr_basename = 'foo_HDF_EOS_granule.hdf'
+        cls.opendap_url = (
+            f'https://fake.opendap.earthdata.nasa.gov/{cls.example_dmr_basename}'
+        )
+        cls.opendap_xml_download_url = f'https://fake.opendap.earthdata.nasa.gov/{cls.example_dmr_basename}.dmr.xml'
         cls.query_granule_return = [
             {
                 'links': [
@@ -30,6 +35,18 @@ def setUpClass(cls):
                 ]
             }
         ]
+        cls.query_granule_return_opendap = [
+            {
+                'links': [
+                    {
+                        'rel': 'http://esipfed.org/ns/fedsearch/1.1/service#',
+                        'title': 'OPeNDAP request URL (GET DATA : OPENDAP DATA)',
+                        'hreflang': 'en-US',
+                        'href': cls.opendap_url,
+                    }
+                ]
+            }
+        ]
         cls.rssmif16d_variables = [
             'atmosphere_cloud_liquid_water_content',
             'atmosphere_water_vapor_content',
@@ -53,6 +70,18 @@ def download_granule_side_effect(granule_link, auth_header, out_directory):
         netcdf4_file_path = 'tests/unit/data/f16_ssmis_20210426v7.nc'
         return copy(netcdf4_file_path, out_directory)
 
+    @staticmethod
+    def download_dmr_side_effect(granule_link, auth_header, out_directory):
+        """A helper method that will copy a test dmr file to the temporary
+        directory being used for a specific test.
+
+        Static methods do not have access to class attributes, so the test
+        file path is defined in this method as well as setUpClass.
+
+        """
+        dmr_file_path = 'tests/unit/data/M2I3NPASM_example.dmr'
+        return copy(dmr_file_path, out_directory)
+
     @patch('varinfo.umm_var.publish_umm_var')
     @patch('varinfo.cmr_search.GranuleQuery')
     @patch('varinfo.generate_umm_var.download_granule')
@@ -144,6 +173,52 @@ def test_generate_collection_umm_var_with_publication(
         # Ensure the output looks as expected
         self.assertSetEqual(set(published_umm_var), set(expected_concept_ids))
 
+    @patch('varinfo.umm_var.publish_umm_var')
+    @patch('varinfo.generate_umm_var.get_dmr_xml_url')
+    @patch('varinfo.cmr_search.GranuleQuery')
+    @patch('varinfo.generate_umm_var.download_granule')
+    def test_generate_collection_umm_var_dmr(
+        self,
+        mock_download_granule,
+        mock_granule_query,
+        mock_get_dmr_xml_url,
+        mock_publish_umm_var,
+    ):
+        """Test an end-to-end request for a DMR file."""
+        mock_granule_query.return_value.get.return_value = (
+            self.query_granule_return_opendap
+        )
+
+        # Add side effect that will copy test file to the temporary directory,
+        # simulating a download.
+        mock_download_granule.side_effect = self.download_dmr_side_effect
+
+        # Set return value for get_dmr_xml_url
+        mock_get_dmr_xml_url.return_value = self.opendap_xml_download_url
+
+        # Check call arguments when use_dmr=True
+        generate_collection_umm_var(
+            self.collection_concept_id,
+            self.bearer_token_header,
+            use_dmr=True,
+        )
+
+        mock_get_dmr_xml_url.assert_called_once_with(self.query_granule_return_opendap)
+        # Ensure the granule query used expected query parameters
+        mock_granule_query.return_value.parameters.assert_called_once_with(
+            downloadable=True,
+            sort_key='-start_date',
+            concept_id=self.collection_concept_id,
+        )
+
+        # Ensure the call to download the granule had correct parameters
+        mock_download_granule.assert_called_once_with(
+            self.opendap_xml_download_url, self.bearer_token_header, out_directory=ANY
+        )
+
+        # Check that no attempt was made to publish a UMM-Var record to CMR:
+        mock_publish_umm_var.assert_not_called()
+
     @patch('varinfo.umm_var.publish_umm_var')
     @patch('varinfo.cmr_search.GranuleQuery')
     @patch('varinfo.generate_umm_var.download_granule')
diff --git a/varinfo/cmr_search.py b/varinfo/cmr_search.py
@@ -190,3 +190,22 @@ def get_edl_token_header(auth_header: str, cmr_env: CmrEnvType) -> str:
     else:
         edl_auth_header = auth_header
     return edl_auth_header
+
+
+def get_dmr_xml_url(granule_response: Sequence) -> str:
+    """Gets the OPeNDAP url from a CMR granule response and appends `.dmr.xml`"""
+    granule_link = next(
+        (
+            link['href']
+            for link in granule_response[0].get('links', [])
+            if link['rel'].endswith('/service#')
+            and 'inherited' not in link
+            and 'opendap' in link['href']
+        ),
+        None,
+    )
+
+    if granule_link is None:
+        raise MissingGranuleDownloadLinks(granule_response)
+
+    return granule_link + '.dmr.xml'
diff --git a/varinfo/generate_umm_var.py b/varinfo/generate_umm_var.py
@@ -18,13 +18,14 @@
 
 from cmr import CMR_UAT
 
-from varinfo import VarInfoFromNetCDF4
+from varinfo import VarInfoFromNetCDF4, VarInfoFromDmr
 from varinfo.cmr_search import (
     CmrEnvType,
     download_granule,
     get_granule_link,
     get_granules,
     get_edl_token_header,
+    get_dmr_xml_url,
 )
 from varinfo.umm_var import get_all_umm_var, publish_all_umm_var
 
@@ -40,6 +41,7 @@ def generate_collection_umm_var(
     cmr_env: CmrEnvType = CMR_UAT,
     publish: bool = False,
     config_file: str | None = None,
+    use_dmr: bool = False,
 ) -> UmmVarReturnType:
     """Run all the of the functions for downloading and publishing
     a UMM-Var entry to CMR given:
@@ -64,20 +66,29 @@ def generate_collection_umm_var(
         collection_concept_id, cmr_env=cmr_env, auth_header=auth_header_edl_token
     )
 
-    # Get the data download URL for the most recent granule (NetCDF-4 file)
-    granule_link = get_granule_link(granule_response)
+    if use_dmr:
+        # Get OPeNDAP data URL with `.dml.xml` appended
+        granule_link = get_dmr_xml_url(granule_response)
+
+    else:
+        # Get the data download URL for the most recent granule (NetCDF-4 file)
+        granule_link = get_granule_link(granule_response)
 
     with TemporaryDirectory() as temp_dir:
         # Download file to runtime environment
         local_granule = download_granule(
             granule_link, auth_header_edl_token, out_directory=temp_dir
         )
 
-        # Parse the granule with VarInfo to map all variables and relations:
-        var_info = VarInfoFromNetCDF4(local_granule, config_file=config_file)
+        if use_dmr:
+            # Parse the granule with VarInfoFromDmr to map all variables and relations:
+            var_info = VarInfoFromDmr(local_granule, config_file=config_file)
+        else:
+            # Parse the granule with VarInfoFromNetCDF4 to map all variables and relations:
+            var_info = VarInfoFromNetCDF4(local_granule, config_file=config_file)
 
-        # Generate all the UMM-Var records:
-        all_umm_var_records = get_all_umm_var(var_info)
+    # Generate all the UMM-Var records:
+    all_umm_var_records = get_all_umm_var(var_info)
 
     if publish:
         # Publish to CMR and construct an output object that is a list of
