Euclid: new method to retrieve scientific LE3 products

CHANGES.rst

@@ -24,6 +24,12 @@ alma
 - Bug fix in ``footprint_to_reg`` that did not allow regions to be plotted. [#3285]
 
 
+esa.euclid
+^^^^^^^^^^
+
+- New method, get_scientific_product_list, to retrieve scientific LE3
+  products. [#3313]
+
 gaia
 ^^^^
 
@@ -82,10 +88,10 @@ mast
 
 - Fix bug in ``utils.remove_duplicate_products`` that does not retain the order of the products in an input table. [#3314]
 
-- Added ``return_uri_map`` parameter to ``Observations.get_cloud_uris`` to return a mapping of the input data product URIs 
+- Added ``return_uri_map`` parameter to ``Observations.get_cloud_uris`` to return a mapping of the input data product URIs
   to the returned cloud URIs. [#3314]
 
-- Added ``verbose`` parameter to ``Observations.get_cloud_uris`` to control whether warnings are logged when a product cannot 
+- Added ``verbose`` parameter to ``Observations.get_cloud_uris`` to control whether warnings are logged when a product cannot
   be found in the cloud. [#3314]
 
 

astroquery/esa/euclid/__init__.py

@@ -59,6 +59,70 @@ class Conf(_config.ConfigNamespace):
 
     SIR_SCIENCE_FRAME_PRODUCTS = ['dpdSirScienceFrame']
 
+    VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS = {
+        'Clusters of Galaxies': {
+            'GrpCatalog': ['DpdLE3clAMICOModel', 'DpdLE3clDetMergeParams', 'DpdLE3clDetOnMockParams',
+                           'DpdLE3clDetInputParams', 'DpdLE3clAmicoAux', 'DpdLE3clAssociations', 'DpdLE3clPzwavAux',
+                           'DpdLE3clPZWAVDensity', 'DpdLE3clDetClusters', 'DpdLE3FullDet', 'DpdLE3clCatMergeParams',
+                           'DpdLE3clCATParams', 'DpdLE3clCcpInputParams', 'DpdLE3clRichMembers', 'DpdLE3clZClParams',
+                           'DpdLE3clGlueMatchParams', 'DpdLE3clMockGlueMatchParams'],
+            'GrpClustering': ['DDpdLE3clPkDOA', 'DpdLE3clCovmatTwoPointCov2', 'DpdLE3clPkYam',
+                              'DpdLE3clTwoPointAutoCorrPol', 'DpolDpdLE3clCovmatPKCov1'],
+            'GrpCOMB': ['DpdLE3clCombConfigurationSet', 'DpdLE3clCombCovMatDeltaSigmaCosmoDep',
+                        'DpdLE3clCombCovMatReducedShearCosmoDep', 'DpdLE3clCombCovMatReducedShearCosmoIndep',
+                        'DpdLE3clCombRedSheProf', 'DpdLE3clCombStackingCosmoDep', 'DpdLE3clCombStackingCosmoInd',
+                        'DpdLE3clCombUCovRedSheProf', 'DpdLE3clCombWLME'],
+            'GrpLMF': ['DpdLE3clLMFOutput', 'DpdLE3clLMFParams'],
+            'GrpSEL': ['DpdLE3clMatchForSelParams', 'DpdLE3clMockClusters', 'DpdLE3clRedshiftDistrib',
+                       'DpdLE3clSelRandom', 'DpdLE3clSelRandomParams', 'DpdLE3clSelSelFunc',
+                       'DpdLE3clSelSelFuncInputParams', 'DpdLE3clSelSinfoniaIniClMockInputParams',
+                       'DpdLE3clSelSinfoniaMockInputParams', 'DpdLE3clSelSubSample', 'DpdLE3clSinfoniaEllipticity'],
+            'GrpTiling': ['DpdLE3clCLTile', 'DpdLE3clCommon', 'DpdLE3clConfigurationSet']},
+        'External Data Products': {
+            'LE3-ED configuration catalog': ['DpdLE3edConfigurationFile'],
+            'LE3-ED match catalog': ['DpdLE3edMatchedCatalog']},
+        'Galaxy Clustering Products': {
+            '2PCF_PK': ['DpdLE3gcPkCross', 'DpdLE3gcPkDOA', 'DpdLE3gcPkYam', 'DpdLE3gcTwoPointAutoCorr',
+                        'DpdLE3gcTwoPointAutoCorrCart', 'DpdLE3gcTwoPointAutoCorrPol', 'DpdLE3gcTwoPointCrossCorr',
+                        'DpdLE3gcTwoPointCrossCorrCart', 'DpdLE3gcTwoPointCrossCorrPol', 'DpdLE3gcTwoPointRecAutoCorr',
+                        'DpdLE3gcTwoPointRecAutoCorrCart', 'DpdLE3gcTwoPointRecAutoCorrPol',
+                        'DpdLE3gcTwoPointRecCrossCorr', 'DpdLE3gcTwoPointRecCrossCorrCart',
+                        'DpdLE3gcTwoPointRecCrossCorrPol'],
+            '3PCF_BK': ['DpdLE3gcBkMonopole', 'DpdLE3gcBkMultipole', 'DpdLE3gcThreePointAll', 'DpdLE3gcThreePointSin'],
+            'CM-2PCF': ['DpdLE3gcCovmatTwoPointCov1D', 'DpdLE3gcCovmatTwoPointCov2Dcart',
+                        'DpdLE3gcCovmatTwoPointCov2Dpol', 'DpdLE3gcCovmatTwoPointCovMu',
+                        'DpdLE3gcCovmatTwoPointCovPro'],
+            'CM-PK': ['DpdLE3gcCovmatPKCov1D', 'DpdLE3gcCovmatPKCov2Dcart', 'DpdLE3gcCovmatPKCov2Dpol']},
+        'Internal Data Products': {
+            'SEL Config/Stats': ['DpdLE3IDSELConfigurationSet', 'DpdLE3IDSELIDStatistics'],
+            'SEL Wide Main': ['DpdLE3IDSELIDCatalog'],
+            'SEL Wide': ['DpdLE3IDSELIDSubsampledCatalog'],
+            'VMSP Group': ['DpdLE3IDVMSPConfiguration', 'DpdLE3IDVMSPDetectionModel', 'DpdLE3IDVMSPDistModel',
+                           'DpdLE3IDVMSPRandomCatalog']},
+        'Weak Lensing Products': {
+            '2D-MASS': ['DpdTwoDMassConvergenceClusters', 'DpdTwoDMassConvergencePatch',
+                        'DpdTwoDMassConvergencePatchesToSphere', 'DpdTwoDMassConvergenceSphere',
+                        'DpdTwoDMassParamsConvergenceClusters', 'DpdTwoDMassParamsConvergencePatch',
+                        'DpdTwoDMassParamsConvergencePatchesToSphere', 'DpdTwoDMassParamsConvergenceSphere',
+                        'DpdTwoDMassParamsPeakCatalogConvergence', 'DpdTwoDMassParamsPeakCatalogMassAperture',
+                        'DpdTwoDMassPeakCatalog'],
+            '2PCF': ['DpdTwoPCFWLCOSEBIFilter', 'DpdTwoPCFWLParamsCOSEBIShearShear2D', 'DpdTwoPCFWLParamsClPosPos2D',
+                     'DpdTwoPCFWLParamsPEBPosShear2D', 'DpdTwoPCFWLParamsPEBShearShear2D', 'DpdTwoPCFWLParamsPosPos2D',
+                     'DpdTwoPCFWLParamsPosShear2D', 'DpdTwoPCFWLParamsShearShear2D', 'DpdTwoPCFWLCOSEBIShearShear2D',
+                     'DpdTwoPCFWLClPosPos2D', 'DpdTwoPCFWLPEBPosShear2D', 'DpdTwoPCFWLPEBShearShear2D',
+                     'DpdTwoPCFWLPosPos2D', 'DpdTwoPCFWLPosShear2D', 'DpdTwoPCFWLShearShear2D',
+                     'DpdCovarTwoPCFWLParams', 'DpdCovarTwoPCFWLClPosPos2D', 'DpdCovarTwoPCFWLCOSEBIShearShear2D',
+                     'DpdCovarTwoPCFWLPEBPosShear2D', 'DpdCovarTwoPCFWLPEBShearShear2D',
+                     'DpdCovarTwoPCFWLPosPos2D', 'DpdCovarTwoPCFWLPosShear2D', 'DpdCovarTwoPCFWLShearShear2D',
+                     'DpdCovarTwoPCFWLResampleCOSEBIShearShear2D', 'DpdCovarTwoPCFWLResampleClPosPos2D',
+                     'DpdCovarTwoPCFWLResamplePEBPosShear2D', 'DpdCovarTwoPCFWLResamplePEBShearShear2D',
+                     'DpdCovarTwoPCFWLResamplePosPos2D', 'DpdCovarTwoPCFWLResamplePosShear2D',
+                     'DpdCovarTwoPCFWLResampleShearShear2D'],
+            'PK': ['DpdPKWLAlms', 'DpdPKWLCovMatrix2D', 'DpdPKWLEstimate2D', 'DpdPKWLMaps', 'DpdPKWLMixingMatrix2D']},
+        'PHZ': {
+            'PHZ': ['DpdBinMeanRedshift', 'DpdReferenceSample', 'DpdTomographicBins']}
+    }
+
     PRODUCT_TYPES = ['observation', 'mosaic']
 
     SCHEMAS = ['sedm']

astroquery/esa/euclid/core.py

@@ -8,6 +8,7 @@
 """
 import binascii
 import os
+import pprint
 import tarfile
 import zipfile
 from collections.abc import Iterable
@@ -472,6 +473,10 @@ def login(self, *, user=None, password=None, credentials_file=None, verbose=Fals
             file containing user and password in two lines
         verbose : bool, optional, default 'False'
             flag to display information about the process
+
+        Returns
+        -------
+        None
         """
         try:
             log.info(f"Login to Euclid TAP server: {self._TapPlus__getconnhandler().get_host_url()}")
@@ -503,6 +508,10 @@ def login_gui(self, verbose=False):
         ----------
         verbose : bool, optional, default 'False'
             flag to display information about the process
+
+        Returns
+        -------
+        None
         """
         try:
             log.info(f"Login to Euclid TAP server: {self._TapPlus__getconnhandler().get_host_url()}")
@@ -540,6 +549,11 @@ def logout(self, verbose=False):
         ----------
         verbose : bool, optional, default 'False'
             flag to display information about the process
+
+        Returns
+        -------
+        None
+
         """
         try:
             super().logout(verbose=verbose)
@@ -779,7 +793,7 @@ def __get_tile_catalogue_list(self, *, tile_index, product_type, verbose=False):
 
         Returns
         -------
-        The list of products (astropy.table)
+        The products in an astropy.table.Table
         """
 
         if tile_index is None:
@@ -884,7 +898,7 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 #. NISP
                     DpdNispRawFrame: NISP Raw Frame Product
 
-            #. Euclid LE2/LE3 products:
+            #. Euclid LE2 products:
 
                 #. VIS
                     DpdVisCalibratedQuadFrame: VIS Calibrated Frame Product
@@ -1276,5 +1290,157 @@ def get_datalinks(self, ids, *, linking_parameter='SOURCE_ID', verbose=False):
 
         return self.__eucliddata.get_datalinks(ids=ids, linking_parameter=linking_parameter, verbose=verbose)
 
+    def get_scientific_product_list(self, *, observation_id=None, tile_index=None, category=None, group=None,
+                                    product_type=None, dataset_release='REGREPROC1_R2', verbose=False):
+        """ Gets the LE3 products (the high-level science data products).
+
+        Please note that not all combinations of category, group, and product_type are valid. Check the available values
+        in https://astroquery.readthedocs.io/en/latest/esa/euclid/euclid.html#appendix
+
+        Parameters
+        ----------
+        observation_id: str, optional, default None.
+            It is not compatible with parameter tile_index.
+        tile_index: str, optional, default None.
+            It is not compatible with parameter observation_id.
+        category: str, optional, default None.
+        group : str, optional, default None
+        product_type : str, optional, default None
+        dataset_release : str, mandatory. Default REGREPROC1_R2
+            Data release from which data should be taken.
+        verbose : bool, optional, default 'False'
+            flag to display information about the process
+
+        Returns
+        -------
+        The products in an astropy.table.Table
+
+        """
+
+        query_extra_condition = ""
+
+        if (observation_id is None and tile_index is None and category is None and group is None and product_type is
+                None):
+            raise ValueError("Include a valid parameter to retrieve a LE3 product.")
+
+        if dataset_release is None:
+            raise ValueError("The release is required.")
+
+        if observation_id is not None and tile_index is not None:
+            raise ValueError(self.__ERROR_MSG_REQUESTED_OBSERVATION_ID_AND_TILE_ID)
+
+        if tile_index is not None:
+            query_extra_condition = f" AND '{tile_index}' = ANY(tile_index_list) "
+
+        if observation_id is not None:
+            query_extra_condition = f" AND '{observation_id}' = ANY(observation_id_list) "
+
+        if category is not None:
+
+            try:
+                _ = conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[category]
+            except KeyError:
+                raise ValueError(
+                    f"Invalid combination of parameters: category={category}. Valid values:\n "
+                    f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+            if group is not None:
+
+                try:
+                    product_type_for_category_group_list = conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[category][
+                        group]
+                except KeyError:
+                    raise ValueError(
+                        f"Invalid combination of parameters: category={category}; group={group}. Valid "
+                        f"values:\n {pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+                if product_type is not None:
+
+                    if product_type not in product_type_for_category_group_list:
+                        raise ValueError(
+                            f"Invalid combination of parameters: category={category}; group={group}; "
+                            f"product_type={product_type}. Valid values:\n "
+                            f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+                    query_extra_condition = query_extra_condition + f" AND product_type ='{product_type}' "
+                else:
+
+                    final_products = ', '.join(f"'{w}'" for w in product_type_for_category_group_list)
+                    query_extra_condition = query_extra_condition + f" AND product_type IN ({final_products}) "
+            else:  # category is not None and group is None
+
+                product_type_for_category_group_list = [item for row in
+                                                        conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[category]
+                                                        .values() for item in row]
+                if product_type is not None:
+
+                    if product_type not in product_type_for_category_group_list:
+                        raise ValueError(
+                            f"Invalid combination of parameters: category={category}; product_type={product_type}."
+                            f" Valid values:\n {pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+                    query_extra_condition = query_extra_condition + f" AND product_type = '{product_type}' "
+
+                else:  # category is not None and group is None and product_type is None
+                    final_products = ', '.join(f"'{w}'" for w in product_type_for_category_group_list)
+                    query_extra_condition = query_extra_condition + f" AND product_type IN ({final_products}) "
+        else:  # category is None
+
+            all_groups_dict = {}
+            for i in conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS.keys():
+                all_groups_dict.update(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[i])
+
+            if group is not None:
+
+                try:
+                    _ = all_groups_dict[group]
+                except KeyError:
+                    raise ValueError(
+                        f"Invalid combination of parameters: group={group}. Valid values:\n "
+                        f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+                if product_type is not None:
+
+                    if product_type not in all_groups_dict[group]:
+                        raise ValueError(
+                            f"Invalid combination of parameters: group={group}; product_type={product_type}. Valid "
+                            f"values:\n {pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+                    query_extra_condition = query_extra_condition + f" AND product_type = '{product_type}' "
+                else:  # group is not None and product_type is None
+
+                    product_type_for_group_list = all_groups_dict[group]
+                    final_products = ', '.join(f"'{w}'" for w in product_type_for_group_list)
+                    query_extra_condition = query_extra_condition + f" AND product_type IN ({final_products}) "
+
+            else:  # category is None and group is None
+
+                product_type_for_category_group_list = [element for sublist in all_groups_dict.values() for element
+                                                        in sublist]
+
+                if product_type is not None:
+                    if product_type not in product_type_for_category_group_list:
+                        raise ValueError(
+                            f"Invalid combination of parameters: product_type={product_type}. Valid values:\n "
+                            f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
+
+                    query_extra_condition = query_extra_condition + f" AND product_type = '{product_type}' "
+
+                else:
+                    query_extra_condition = query_extra_condition + ""
+
+        query = (
+            f"SELECT basic_download_data.basic_download_data_oid, basic_download_data.product_type, "
+            f"basic_download_data.product_id, CAST(basic_download_data.observation_id_list as text) AS "
+            f"observation_id_list, CAST(basic_download_data.tile_index_list as text) AS tile_index_list, "
+            f"CAST(basic_download_data.patch_id_list as text) AS patch_id_list, "
+            f"CAST(basic_download_data.filter_name as text) AS filter_name FROM sedm.basic_download_data WHERE "
+            f"release_name='{dataset_release}' {query_extra_condition} ORDER BY observation_id_list ASC")
+
+        job = super().launch_job(query=query, output_format='votable_plain', verbose=verbose,
+                                 format_with_results_compressed=('votable_gzip',))
+
+        return job.get_results()
+
 
 Euclid = EuclidClass()