Skip to content

Commit 621b17b

Browse files
lhoestqlhoestq
committed
retry intermediate commits too
1 parent adaddcd commit 621b17b

File tree

3 files changed

+112
-36
lines changed

3 files changed

+112
-36
lines changed

‎src/datasets/arrow_dataset.py‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5963,15 +5963,34 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
59635963
operations = additions[
59645964
i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
59655965
]
5966-
commit_info = api.create_commit(
5967-
repo_id,
5968-
operations=operations,
5969-
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
5970-
commit_description=commit_description,
5971-
repo_type="dataset",
5972-
revision=revision,
5973-
create_pr=create_pr,
5974-
)
5966+
for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
5967+
# We need to retry if another commit happens at the same time
5968+
sleep_time *= 1 + random.random()
5969+
try:
5970+
commit_info = api.create_commit(
5971+
repo_id,
5972+
operations=operations,
5973+
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
5974+
commit_description=commit_description,
5975+
repo_type="dataset",
5976+
revision=revision,
5977+
create_pr=create_pr,
5978+
)
5979+
except HfHubHTTPError as err:
5980+
if (
5981+
err.__context__
5982+
and isinstance(err.__context__, HTTPError)
5983+
and err.__context__.response.status_code == 409
5984+
):
5985+
# 409 is Conflict (another commit is in progress)
5986+
time.sleep(sleep_time)
5987+
logger.info(
5988+
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
5989+
)
5990+
continue
5991+
else:
5992+
raise
5993+
break
59755994
logger.info(
59765995
f"Commit #{i + 1} completed"
59775996
+ (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")

‎src/datasets/dataset_dict.py‎

Lines changed: 56 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1895,15 +1895,34 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
18951895
operations = additions[
18961896
i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
18971897
]
1898-
commit_info = api.create_commit(
1899-
repo_id,
1900-
operations=operations,
1901-
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
1902-
commit_description=commit_description,
1903-
repo_type="dataset",
1904-
revision=revision,
1905-
create_pr=create_pr,
1906-
)
1898+
for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
1899+
# We need to retry if another commit happens at the same time
1900+
sleep_time *= 1 + random.random()
1901+
try:
1902+
commit_info = api.create_commit(
1903+
repo_id,
1904+
operations=operations,
1905+
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
1906+
commit_description=commit_description,
1907+
repo_type="dataset",
1908+
revision=revision,
1909+
create_pr=create_pr,
1910+
)
1911+
except HfHubHTTPError as err:
1912+
if (
1913+
err.__context__
1914+
and isinstance(err.__context__, HTTPError)
1915+
and err.__context__.response.status_code == 409
1916+
):
1917+
# 409 is Conflict (another commit is in progress)
1918+
time.sleep(sleep_time)
1919+
logger.info(
1920+
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
1921+
)
1922+
continue
1923+
else:
1924+
raise
1925+
break
19071926
logger.info(
19081927
f"Commit #{i + 1} completed"
19091928
+ (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
@@ -2745,15 +2764,34 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
27452764
operations = additions[
27462765
i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
27472766
]
2748-
commit_info = api.create_commit(
2749-
repo_id,
2750-
operations=operations,
2751-
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
2752-
commit_description=commit_description,
2753-
repo_type="dataset",
2754-
revision=revision,
2755-
create_pr=create_pr,
2756-
)
2767+
for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
2768+
# We need to retry if another commit happens at the same time
2769+
sleep_time *= 1 + random.random()
2770+
try:
2771+
commit_info = api.create_commit(
2772+
repo_id,
2773+
operations=operations,
2774+
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
2775+
commit_description=commit_description,
2776+
repo_type="dataset",
2777+
revision=revision,
2778+
create_pr=create_pr,
2779+
)
2780+
except HfHubHTTPError as err:
2781+
if (
2782+
err.__context__
2783+
and isinstance(err.__context__, HTTPError)
2784+
and err.__context__.response.status_code == 409
2785+
):
2786+
# 409 is Conflict (another commit is in progress)
2787+
time.sleep(sleep_time)
2788+
logger.info(
2789+
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
2790+
)
2791+
continue
2792+
else:
2793+
raise
2794+
break
27572795
logger.info(
27582796
f"Commit #{i + 1} completed"
27592797
+ (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")

‎src/datasets/iterable_dataset.py‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4246,15 +4246,34 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
42464246
operations = additions[
42474247
i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
42484248
]
4249-
commit_info = api.create_commit(
4250-
repo_id,
4251-
operations=operations,
4252-
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
4253-
commit_description=commit_description,
4254-
repo_type="dataset",
4255-
revision=revision,
4256-
create_pr=create_pr,
4257-
)
4249+
for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
4250+
# We need to retry if another commit happens at the same time
4251+
sleep_time *= 1 + random.random()
4252+
try:
4253+
commit_info = api.create_commit(
4254+
repo_id,
4255+
operations=operations,
4256+
commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
4257+
commit_description=commit_description,
4258+
repo_type="dataset",
4259+
revision=revision,
4260+
create_pr=create_pr,
4261+
)
4262+
except HfHubHTTPError as err:
4263+
if (
4264+
err.__context__
4265+
and isinstance(err.__context__, HTTPError)
4266+
and err.__context__.response.status_code == 409
4267+
):
4268+
# 409 is Conflict (another commit is in progress)
4269+
time.sleep(sleep_time)
4270+
logger.info(
4271+
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
4272+
)
4273+
continue
4274+
else:
4275+
raise
4276+
break
42584277
logger.info(
42594278
f"Commit #{i + 1} completed"
42604279
+ (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")

0 commit comments

Comments
 (0)