diff --git a/src/parselmouth/cli/cli.py b/src/parselmouth/cli/cli.py index af845160..fd4c2358 100644 --- a/src/parselmouth/cli/cli.py +++ b/src/parselmouth/cli/cli.py @@ -29,16 +29,14 @@ def main(): @app.command() def updater_producer( output_dir: str = "output_index", - check_if_exists: bool = True, + force: bool = False, channel: SupportedChannels = SupportedChannels.CONDA_FORGE, ): """ Generate the subdir@letter list. """ - updater_producer_main( - output_dir=output_dir, check_if_exists=check_if_exists, channel=channel - ) + updater_producer_main(output_dir=output_dir, force=force, channel=channel) @app.command() @@ -53,6 +51,7 @@ def updater( partial_output_dir: str = "output", channel: SupportedChannels = SupportedChannels.CONDA_FORGE, upload: bool = False, + force: bool = False, ): """ Get all the packages based on subdir@letter and save it in partial_output_dir. @@ -68,6 +67,7 @@ def updater( partial_output_dir=partial_output_dir, channel=channel, upload=upload, + force=force, ) diff --git a/src/parselmouth/internals/updater.py b/src/parselmouth/internals/updater.py index 4f48eb07..b277c927 100644 --- a/src/parselmouth/internals/updater.py +++ b/src/parselmouth/internals/updater.py @@ -88,6 +88,7 @@ def main( partial_output_dir: str = "output", channel: SupportedChannels = SupportedChannels.CONDA_FORGE, upload: bool = False, + force: bool = False, ): subdir, letter = subdir_letter.split("@") @@ -107,7 +108,7 @@ def main( package = repodatas[package_name] sha256 = package["sha256"] - if sha256 not in existing_mapping_data: + if sha256 not in existing_mapping_data or force: # trying to get packages info using all backends. # note: streamed is not supported for .tar.gz if package_name.endswith(".conda"): @@ -117,7 +118,7 @@ def main( total_packages.add(package_name) total = 0 - logging.warning(f"Total packages for processing: {len(all_packages)} for {subdir}") + logging.warning(f"Total packages for processing: {len(all_packages)} for {subdir}.") with concurrent.futures.ThreadPoolExecutor() as executor: futures = { executor.submit( diff --git a/src/parselmouth/internals/updater_producer.py b/src/parselmouth/internals/updater_producer.py index 9d3508d5..b52682fc 100644 --- a/src/parselmouth/internals/updater_producer.py +++ b/src/parselmouth/internals/updater_producer.py @@ -18,16 +18,13 @@ egg_pattern_compiled = re.compile(egg_info_pattern) -def main(output_dir: str, check_if_exists: bool, channel: SupportedChannels): +def main(output_dir: str, force: bool, channel: SupportedChannels): subdirs = get_all_archs_available(channel) all_packages: list[tuple[str, str]] = [] - if check_if_exists: - existing_mapping_data = s3_client.get_channel_index(channel=channel) - if not existing_mapping_data: - existing_mapping_data = IndexMapping(root={}) - else: + existing_mapping_data = s3_client.get_channel_index(channel=channel) + if not existing_mapping_data: existing_mapping_data = IndexMapping(root={}) letters = set() @@ -43,7 +40,11 @@ def main(output_dir: str, check_if_exists: bool, channel: SupportedChannels): package = repodatas[package_name] sha256 = package["sha256"] - if sha256 not in existing_mapping_data.root: + if force: + all_packages.append(package_name) + letters.add(f"{subdir}@{package_name[0]}") + + elif sha256 not in existing_mapping_data.root: all_packages.append(package_name) letters.add(f"{subdir}@{package_name[0]}")