Adding support of multiple operators in one fragment

[CLOUDDST-22242]

Author: lipoja

iib/workers/tasks/fbc_utils.py

@@ -4,7 +4,7 @@
 import logging
 import shutil
 
-from typing import Tuple
+from typing import Tuple, List
 from iib.exceptions import IIBError
 from iib.workers.config import get_worker_config
 from iib.common.tracing import instrument_tracing
@@ -91,13 +91,13 @@ def merge_catalogs_dirs(src_config: str, dest_config: str):
     shutil.copytree(src_config, dest_config, dirs_exist_ok=True)
 
 
-def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]:
+def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, List[str]]:
     """
-    Extract operator package from the fbc_fragment image.
+    Extract operator packages from the fbc_fragment image.
 
     :param str temp_dir: base temp directory for IIB request.
     :param str fbc_fragment: pull specification of fbc_fragment in the IIB request.
-    :return: fbc_fragment path, fbc_operator_package.
+    :return: fbc_fragment path, fbc_operator_packages.
     :rtype: tuple
     """
     from iib.workers.tasks.build import _copy_files_from_image
@@ -111,10 +111,8 @@ def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]:
 
     log.info("fbc_fragment extracted at %s", fbc_fragment_path)
     operator_packages = os.listdir(fbc_fragment_path)
-    log.info("fbc_fragment contains package %s", operator_packages)
+    log.info("fbc_fragment contains packages %s", operator_packages)
     if not operator_packages:
         raise IIBError("No operator packages in fbc_fragment %s", fbc_fragment)
-    if len(operator_packages) > 1:
-        raise IIBError("More than 1 package is present in fbc_fragment %s", fbc_fragment)
 
-    return fbc_fragment_path, operator_packages[0]
+    return fbc_fragment_path, operator_packages

iib/workers/tasks/opm_operations.py

@@ -819,57 +819,56 @@ def opm_registry_add_fbc_fragment(
     """
     set_request_state(request_id, 'in_progress', 'Extracting operator package from fbc_fragment')
     # fragment path will look like /tmp/iib-**/fbc-fragment
-    fragment_path, fragment_operator = extract_fbc_fragment(
+    fragment_path, fragment_operators = extract_fbc_fragment(
         temp_dir=temp_dir, fbc_fragment=fbc_fragment
     )
 
-    is_operator_in_db, index_db_path = verify_operator_exists(
+    # the dir where all the configs from from_index are stored
+    # this will look like /tmp/iib-**/configs
+    from_index_configs_dir = get_catalog_dir(from_index=from_index, base_dir=temp_dir)
+    log.info("The content of from_index configs located at %s", from_index_configs_dir)
+
+    operators_in_db, index_db_path = verify_operators_exists(
         from_index=from_index,
         base_dir=temp_dir,
-        operator_package=fragment_operator,
+        operator_packages=fragment_operators,
         overwrite_from_index_token=overwrite_from_index_token,
     )
 
-    # the dir where all the configs from from_index is stored
-    # this will look like /tmp/iib-**/configs
-    from_index_configs_dir = get_catalog_dir(from_index=from_index, base_dir=temp_dir)
-
-    log.info("The content of from_index configs located at %s", from_index_configs_dir)
+    if operators_in_db:
+        log.info('Removing %s from %s index.db ', operators_in_db, from_index)
+        _opm_registry_rm(index_db_path=index_db_path, operators=operators_in_db, base_dir=temp_dir)
 
-    if is_operator_in_db:
-        log.info('Removing %s from %s index.db ', fragment_operator, from_index)
-        _opm_registry_rm(
-            index_db_path=index_db_path, operators=[fragment_operator], base_dir=temp_dir
-        )
-        # migated_catalog_dir path will look like /tmp/iib-**/catalog
-        migated_catalog_dir, _ = opm_migrate(
+        # migrated_catalog_dir path will look like /tmp/iib-**/catalog
+        migrated_catalog_dir, _ = opm_migrate(
             index_db=index_db_path,
             base_dir=temp_dir,
             generate_cache=False,
         )
-        log.info("Migated catalog after removing from db at %s", migated_catalog_dir)
+        log.info("Migrated catalog after removing from db at %s", migrated_catalog_dir)
 
         # copy the content of migrated_catalog to from_index's config
-        log.info("Copying content of %s to %s", migated_catalog_dir, from_index_configs_dir)
-        for operator_package in os.listdir(migated_catalog_dir):
+        log.info("Copying content of %s to %s", migrated_catalog_dir, from_index_configs_dir)
+        for operator_package in os.listdir(migrated_catalog_dir):
             shutil.copytree(
-                os.path.join(migated_catalog_dir, operator_package),
+                os.path.join(migrated_catalog_dir, operator_package),
                 os.path.join(from_index_configs_dir, operator_package),
                 dirs_exist_ok=True,
             )
 
-    # copy fragment_operator to from_index configs
-    set_request_state(request_id, 'in_progress', 'Adding fbc_fragment to from_index')
-    fragment_opr_src_path = os.path.join(fragment_path, fragment_operator)
-    fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator)
-    if os.path.exists(fragment_opr_dest_path):
-        shutil.rmtree(fragment_opr_dest_path)
-    log.info(
-        "Copying content of %s to %s",
-        fragment_opr_src_path,
-        fragment_opr_dest_path,
-    )
-    shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path)
+    for fragment_operator in fragment_operators:
+        # copy fragment_operator to from_index configs
+        set_request_state(request_id, 'in_progress', 'Adding fbc_fragment to from_index')
+        fragment_opr_src_path = os.path.join(fragment_path, fragment_operator)
+        fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator)
+        if os.path.exists(fragment_opr_dest_path):
+            shutil.rmtree(fragment_opr_dest_path)
+        log.info(
+            "Copying content of %s to %s",
+            fragment_opr_src_path,
+            fragment_opr_dest_path,
+        )
+        shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path)
 
     local_cache_path = os.path.join(temp_dir, 'cache')
     generate_cache_locally(
@@ -886,32 +885,32 @@ def opm_registry_add_fbc_fragment(
     )
 
 
-def verify_operator_exists(
+def verify_operators_exists(
     from_index: str,
     base_dir: str,
-    operator_package: str,
+    operator_packages: List[str],
     overwrite_from_index_token: Optional[str],
 ):
     """
-    Check if operator exists in index image.
+    Check if operators exists in index image.
 
     :param str from_index: index in which operator existence is checked
     :param str base_dir: base temp directory for IIB request
-    :param str operator_package: operator_package to check
+    :param list(str) operator_packages: operator_package to check
     :param str overwrite_from_index_token: token used to access the image
-    :return: is_package_in_index, index_db_path
-    :rtype: (str, str)
+    :return: packages_in_index, index_db_path
+    :rtype: (list, str)
     """
     from iib.workers.tasks.build import terminate_process, get_bundle_json
     from iib.workers.tasks.iib_static_types import BundleImage
     from iib.workers.tasks.utils import run_cmd
     from iib.workers.tasks.utils import set_registry_token
 
-    is_package_in_index = False
+    packages_in_index: List[str] = []
 
-    log.info("Verifying if operator package %s exists in index %s", operator_package, from_index)
+    log.info("Verifying if operator packages %s exists in index %s", operator_packages, from_index)
 
-    # check if operater package exists in hidden index.db
+    # check if operator packages exists in hidden index.db
     with set_registry_token(overwrite_from_index_token, from_index, append=True):
         index_db_path = get_hidden_index_database(from_index=from_index, base_dir=base_dir)
 
@@ -924,10 +923,13 @@ def verify_operator_exists(
     present_bundles: List[BundleImage] = get_bundle_json(bundles)
 
     for bundle in present_bundles:
-        if bundle['packageName'] == operator_package:
-            is_package_in_index = True
-            log.info("operator package %s found in index_db %s", operator_package, index_db_path)
-    return is_package_in_index, index_db_path
+        if bundle['packageName'] in operator_packages:
+            packages_in_index.append(bundle['packageName'])
+
+    if packages_in_index:
+        log.info("operator packages found in index_db %s:  %s", index_db_path, packages_in_index)
+
+    return packages_in_index, index_db_path
 
 
 @retry(

tests/test_workers/test_tasks/test_fbc_utils.py

@@ -144,15 +144,14 @@ def test_merge_catalogs_dirs_raise(mock_isdir, mock_cpt, tmpdir):
 @pytest.mark.parametrize('ldr_output', [['testoperator'], ['test1', 'test2'], []])
 @mock.patch('os.listdir')
 @mock.patch('iib.workers.tasks.build._copy_files_from_image')
-def test_extract_fbc_fragment_one_operator(mock_cffi, mock_osldr, ldr_output, tmpdir):
+def test_extract_fbc_fragment(mock_cffi, mock_osldr, ldr_output, tmpdir):
     test_fbc_fragment = "example.com/test/fbc_fragment:latest"
     mock_osldr.return_value = ldr_output
     fbc_fragment_path = os.path.join(tmpdir, get_worker_config()['temp_fbc_fragment_path'])
 
-    if len(ldr_output) != 1:
+    if not ldr_output:
         with pytest.raises(IIBError):
             extract_fbc_fragment(tmpdir, test_fbc_fragment)
-
     else:
         extract_fbc_fragment(tmpdir, test_fbc_fragment)
     mock_cffi.assert_has_calls([mock.call(test_fbc_fragment, '/configs', fbc_fragment_path)])

tests/test_workers/test_tasks/test_opm_operations.py

@@ -648,7 +648,8 @@ def test_verify_cache_insertion_edit_dockerfile_failed():
 
 
 @pytest.mark.parametrize(
-    'is_operator_exists, index_db_path', [(True, "index_path"), (False, "index_path")]
+    'operators_exists, index_db_path',
+    [(['test-operator'], "index_path"), ([], "index_path")],
 )
 @mock.patch('iib.workers.tasks.opm_operations.opm_generate_dockerfile')
 @mock.patch('iib.workers.tasks.opm_operations.generate_cache_locally')
@@ -658,7 +659,7 @@ def test_verify_cache_insertion_edit_dockerfile_failed():
 @mock.patch('iib.workers.tasks.opm_operations.opm_migrate')
 @mock.patch('iib.workers.tasks.opm_operations._opm_registry_rm')
 @mock.patch('iib.workers.tasks.opm_operations.get_catalog_dir')
-@mock.patch('iib.workers.tasks.opm_operations.verify_operator_exists')
+@mock.patch('iib.workers.tasks.opm_operations.verify_operators_exists')
 @mock.patch('iib.workers.tasks.opm_operations.extract_fbc_fragment')
 @mock.patch('iib.workers.tasks.opm_operations.set_request_state')
 def test_opm_registry_add_fbc_fragment(
@@ -673,16 +674,16 @@ def test_opm_registry_add_fbc_fragment(
     mock_rmt,
     mock_gcc,
     mock_ogd,
-    is_operator_exists,
+    operators_exists,
     index_db_path,
     tmpdir,
 ):
     from_index = "example.com/test/index"
     binary_image = "example.com/ose/binary"
     fbc_fragment = "example.com/test/fragment"
-    fbc_fragment_operator = "test-operator"
-    mock_eff.return_value = (os.path.join(tmpdir, "fbc_fragment"), fbc_fragment_operator)
-    mock_voe.return_value = is_operator_exists, index_db_path
+    fbc_fragment_operators = ["test-operator"]
+    mock_eff.return_value = (os.path.join(tmpdir, "fbc_fragment"), fbc_fragment_operators)
+    mock_voe.return_value = operators_exists, index_db_path
     mock_gcr.return_value = os.path.join(tmpdir, "configs")
     mock_om.return_value = os.path.join(tmpdir, "catalog"), None
     mock_ldr.return_value = [
@@ -695,13 +696,13 @@ def test_opm_registry_add_fbc_fragment(
     mock_voe.assert_called_with(
         from_index=from_index,
         base_dir=tmpdir,
-        operator_package=fbc_fragment_operator,
+        operator_packages=fbc_fragment_operators,
         overwrite_from_index_token=None,
     )
     mock_gcr.assert_called_with(from_index=from_index, base_dir=tmpdir)
-    if is_operator_exists:
+    if operators_exists:
         mock_orr.assert_called_with(
-            index_db_path=index_db_path, operators=[fbc_fragment_operator], base_dir=tmpdir
+            index_db_path=index_db_path, operators=fbc_fragment_operators, base_dir=tmpdir
         )
         mock_om.assert_called_with(index_db=index_db_path, base_dir=tmpdir, generate_cache=False)
         mock_cpt.assert_has_calls(
@@ -721,8 +722,8 @@ def test_opm_registry_add_fbc_fragment(
     mock_cpt.assert_has_calls(
         [
             mock.call(
-                os.path.join(tmpdir, "fbc_fragment", fbc_fragment_operator),
-                os.path.join(tmpdir, "configs", fbc_fragment_operator),
+                os.path.join(tmpdir, "fbc_fragment", fbc_fragment_operators[0]),
+                os.path.join(tmpdir, "configs", fbc_fragment_operators[0]),
             )
         ]
     )
@@ -736,12 +737,12 @@ def test_opm_registry_add_fbc_fragment(
         (
             '{"packageName": "test-operator", "version": "v1.0", "bundlePath":"bundle1"\n}'
             '\n{\n"packageName": "package2", "version": "v2.0", "bundlePath":"bundle2"}',
-            True,
+            ["test-operator"],
         ),
         (
             '{"packageName": "package1", "version": "v1.0", "bundlePath":"bundle1"\n}'
             '\n{\n"packageName": "package2", "version": "v2.0", "bundlePath":"bundle2"}',
-            False,
+            [],
         ),
     ],
 )
@@ -757,7 +758,7 @@ def test_verify_operator_exists(
     mock_ors.return_value = 500, mock.MagicMock()
     mock_rc.return_value = bundles_in_db
     index_db_path = os.path.join(tmpdir, get_worker_config()['temp_index_db_path'])
-    package_exists, index_db_path = opm_operations.verify_operator_exists(
+    package_exists, index_db_path = opm_operations.verify_operators_exists(
         from_index, tmpdir, 'test-operator', None
     )
     mock_ors.assert_has_calls([mock.call(db_path=index_db_path)])