Skip to content

Commit 16c9bce

Browse files
warsawSurenNihalanidi
authored
Document the project and file upload limits in pypi on the FAQ page (#16200)
* Document the project and file upload limits in pypi on the FAQ page * Test it * Add license * sort * Test config * make translations * Update warehouse/templates/pages/help.html * Update translations --------- Co-authored-by: Suren Nihalani <1093911+SurenNihalani@users.noreply.github.com> Co-authored-by: Dustin Ingram <di@users.noreply.github.com>
1 parent 69f8cb1 commit 16c9bce

File tree

13 files changed

+202
-165
lines changed

13 files changed

+202
-165
lines changed

‎docs/dev/development/getting-started.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -639,6 +639,8 @@ Building the docs requires Python 3.8. If it is not installed, the
639639
make: *** [.state/env/pyvenv.cfg] Error 127
640640
641641
642+
.. _building-translations:
643+
642644
Building translations
643645
---------------------
644646

‎docs/dev/translations.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,6 @@ the view's configuration:
121121
has_translations=True,
122122
)
123123
class SampleViews:
124+
125+
126+
You may have to :ref:`rebuild the translation files <building-translations>`.

‎tests/unit/admin/views/test_projects.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
from pyramid.httpexceptions import HTTPBadRequest, HTTPMovedPermanently, HTTPSeeOther
2222
from sqlalchemy.orm import joinedload
2323

24+
import warehouse.constants
25+
2426
from tests.common.db.oidc import GitHubPublisherFactory
2527
from warehouse.admin.views import projects as views
2628
from warehouse.observations.models import ObservationKind
@@ -102,10 +104,10 @@ def test_gets_project(self, db_request):
102104
"maintainers": roles,
103105
"journal": journals[:30],
104106
"oidc_publishers": oidc_publishers,
105-
"ONE_MB": views.ONE_MB,
106-
"MAX_FILESIZE": views.MAX_FILESIZE,
107-
"MAX_PROJECT_SIZE": views.MAX_PROJECT_SIZE,
108-
"ONE_GB": views.ONE_GB,
107+
"ONE_MIB": views.ONE_MIB,
108+
"MAX_FILESIZE": warehouse.constants.MAX_FILESIZE,
109+
"MAX_PROJECT_SIZE": warehouse.constants.MAX_PROJECT_SIZE,
110+
"ONE_GIB": views.ONE_GIB,
109111
"UPLOAD_LIMIT_CAP": views.UPLOAD_LIMIT_CAP,
110112
"observation_kinds": ObservationKind,
111113
"observations": [],
@@ -575,11 +577,11 @@ def test_sets_total_size_limitwith_integer(self, db_request):
575577
pretend.call("Set the total size limit on 'foo'", queue="success")
576578
]
577579

578-
assert project.total_size_limit == 150 * views.ONE_GB
580+
assert project.total_size_limit == 150 * views.ONE_GIB
579581

580582
def test_sets_total_size_limitwith_none(self, db_request):
581583
project = ProjectFactory.create(name="foo")
582-
project.total_size_limit = 150 * views.ONE_GB
584+
project.total_size_limit = 150 * views.ONE_GIB
583585

584586
db_request.route_path = pretend.call_recorder(
585587
lambda *a, **kw: "/admin/projects/"
@@ -627,7 +629,7 @@ def test_sets_limitwith_integer(self, db_request):
627629
flash=pretend.call_recorder(lambda *a, **kw: None)
628630
)
629631
db_request.matchdict["project_name"] = project.normalized_name
630-
new_upload_limit = views.MAX_FILESIZE // views.ONE_MB
632+
new_upload_limit = warehouse.constants.MAX_FILESIZE // views.ONE_MIB
631633
db_request.POST["upload_limit"] = str(new_upload_limit)
632634

633635
views.set_upload_limit(project, db_request)
@@ -636,11 +638,11 @@ def test_sets_limitwith_integer(self, db_request):
636638
pretend.call("Set the upload limit on 'foo'", queue="success")
637639
]
638640

639-
assert project.upload_limit == new_upload_limit * views.ONE_MB
641+
assert project.upload_limit == new_upload_limit * views.ONE_MIB
640642

641643
def test_sets_limit_with_none(self, db_request):
642644
project = ProjectFactory.create(name="foo")
643-
project.upload_limit = 90 * views.ONE_MB
645+
project.upload_limit = 90 * views.ONE_MIB
644646

645647
db_request.route_path = pretend.call_recorder(
646648
lambda *a, **kw: "/admin/projects/"

‎tests/unit/forklift/test_legacy.py‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
from trove_classifiers import classifiers
3131
from webob.multidict import MultiDict
3232

33+
import warehouse.constants
34+
3335
from warehouse.accounts.utils import UserContext
3436
from warehouse.admin.flags import AdminFlag, AdminFlagValue
3537
from warehouse.classifiers.models import Classifier
@@ -236,7 +238,9 @@ def test_zipfile_exceeds_compression_threshold(self, tmpdir):
236238

237239
with zipfile.ZipFile(f, "w") as zfp:
238240
zfp.writestr("PKG-INFO", b"this is the package info")
239-
zfp.writestr("1.dat", b"0" * 65 * legacy.ONE_MB, zipfile.ZIP_DEFLATED)
241+
zfp.writestr(
242+
"1.dat", b"0" * 65 * warehouse.constants.ONE_MIB, zipfile.ZIP_DEFLATED
243+
)
240244

241245
assert not legacy._is_valid_dist_file(f, "")
242246

@@ -1626,8 +1630,8 @@ def test_upload_fails_with_too_large_project_size_default_limit(
16261630
EmailFactory.create(user=user)
16271631
project = ProjectFactory.create(
16281632
name="foobar",
1629-
upload_limit=legacy.MAX_FILESIZE,
1630-
total_size=legacy.MAX_PROJECT_SIZE - 1,
1633+
upload_limit=warehouse.constants.MAX_FILESIZE,
1634+
total_size=warehouse.constants.MAX_PROJECT_SIZE - 1,
16311635
)
16321636
release = ReleaseFactory.create(project=project, version="1.0")
16331637
RoleFactory.create(user=user, project=project)
@@ -1673,10 +1677,13 @@ def test_upload_fails_with_too_large_project_size_custom_limit(
16731677
one_megabyte = 1 * 1024 * 1024
16741678
project = ProjectFactory.create(
16751679
name="foobar",
1676-
upload_limit=legacy.MAX_FILESIZE,
1677-
total_size=legacy.MAX_PROJECT_SIZE,
1678-
total_size_limit=legacy.MAX_PROJECT_SIZE
1679-
+ one_megabyte, # Custom Limit for the project
1680+
upload_limit=warehouse.constants.MAX_FILESIZE,
1681+
total_size=warehouse.constants.MAX_PROJECT_SIZE,
1682+
total_size_limit=(
1683+
warehouse.constants.MAX_PROJECT_SIZE
1684+
+ one_megabyte
1685+
# Custom Limit for the project
1686+
),
16801687
)
16811688
release = ReleaseFactory.create(project=project, version="1.0")
16821689
RoleFactory.create(user=user, project=project)
@@ -1726,10 +1733,11 @@ def test_upload_succeeds_custom_project_size_limit(
17261733
one_megabyte = 1 * 1024 * 1024
17271734
project = ProjectFactory.create(
17281735
name="foobar",
1729-
upload_limit=legacy.MAX_FILESIZE,
1730-
total_size=legacy.MAX_PROJECT_SIZE,
1731-
total_size_limit=legacy.MAX_PROJECT_SIZE
1732-
+ (one_megabyte * 60), # Custom Limit for the project
1736+
upload_limit=warehouse.constants.MAX_FILESIZE,
1737+
total_size=warehouse.constants.MAX_PROJECT_SIZE,
1738+
total_size_limit=(
1739+
warehouse.constants.MAX_PROJECT_SIZE + (one_megabyte * 60)
1740+
), # Custom Limit for the project
17331741
)
17341742
release = ReleaseFactory.create(project=project, version="1.0")
17351743
RoleFactory.create(user=user, project=project)
@@ -2167,7 +2175,7 @@ def test_upload_fails_with_invalid_filetype(
21672175
}[filetype],
21682176
"content": pretend.stub(
21692177
filename=filename,
2170-
file=io.BytesIO(b"a" * (legacy.MAX_FILESIZE + 1)),
2178+
file=io.BytesIO(b"a" * (warehouse.constants.MAX_FILESIZE + 1)),
21712179
type="application/tar",
21722180
),
21732181
}
@@ -2205,7 +2213,7 @@ def test_upload_fails_with_invalid_extension(self, pyramid_config, db_request):
22052213
"md5_digest": "nope!",
22062214
"content": pretend.stub(
22072215
filename=filename,
2208-
file=io.BytesIO(b"a" * (legacy.MAX_FILESIZE + 1)),
2216+
file=io.BytesIO(b"a" * (warehouse.constants.MAX_FILESIZE + 1)),
22092217
type="application/tar",
22102218
),
22112219
}
@@ -2246,7 +2254,7 @@ def test_upload_fails_with_unsafe_filename(
22462254
"md5_digest": "nope!",
22472255
"content": pretend.stub(
22482256
filename=filename,
2249-
file=io.BytesIO(b"a" * (legacy.MAX_FILESIZE + 1)),
2257+
file=io.BytesIO(b"a" * (warehouse.constants.MAX_FILESIZE + 1)),
22502258
type="application/tar",
22512259
),
22522260
}
@@ -2283,7 +2291,7 @@ def test_upload_fails_with_disallowed_in_filename(
22832291
"md5_digest": "nope!",
22842292
"content": pretend.stub(
22852293
filename=filename,
2286-
file=io.BytesIO(b"a" * (legacy.MAX_FILESIZE + 1)),
2294+
file=io.BytesIO(b"a" * (warehouse.constants.MAX_FILESIZE + 1)),
22872295
type="application/tar",
22882296
),
22892297
}
@@ -2322,7 +2330,7 @@ def test_upload_fails_without_user_permission(self, pyramid_config, db_request):
23222330
"md5_digest": "nope!",
23232331
"content": pretend.stub(
23242332
filename=filename,
2325-
file=io.BytesIO(b"a" * (legacy.MAX_FILESIZE + 1)),
2333+
file=io.BytesIO(b"a" * (warehouse.constants.MAX_FILESIZE + 1)),
23262334
type="application/tar",
23272335
),
23282336
}
@@ -2364,7 +2372,7 @@ def test_upload_fails_without_oidc_publisher_permission(
23642372
"md5_digest": "nope!",
23652373
"content": pretend.stub(
23662374
filename=filename,
2367-
file=io.BytesIO(b"a" * (legacy.MAX_FILESIZE + 1)),
2375+
file=io.BytesIO(b"a" * (warehouse.constants.MAX_FILESIZE + 1)),
23682376
type="application/tar",
23692377
),
23702378
}

‎tests/unit/manage/test_views.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@
3939
TokenExpired,
4040
)
4141
from warehouse.admin.flags import AdminFlagValue
42+
from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE
4243
from warehouse.events.tags import EventTag
43-
from warehouse.forklift.legacy import MAX_FILESIZE, MAX_PROJECT_SIZE
4444
from warehouse.macaroons import caveats
4545
from warehouse.macaroons.interfaces import IMacaroonService
4646
from warehouse.manage import views

‎tests/unit/test_config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,8 @@ def __init__(self):
254254
"reconcile_file_storages.batch_size": 100,
255255
"metadata_backfill.batch_size": 500,
256256
"gcloud.service_account_info": {},
257+
"warehouse.forklift.legacy.MAX_FILESIZE_MIB": 100,
258+
"warehouse.forklift.legacy.MAX_PROJECT_SIZE_GIB": 10,
257259
}
258260
if environment == config.Environment.development:
259261
expected_settings.update(

‎warehouse/admin/views/projects.py‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,16 @@
2222
from warehouse.accounts.interfaces import IUserService
2323
from warehouse.accounts.models import User
2424
from warehouse.authnz import Permissions
25+
from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB
2526
from warehouse.events.tags import EventTag
26-
from warehouse.forklift.legacy import MAX_FILESIZE, MAX_PROJECT_SIZE
2727
from warehouse.observations.models import OBSERVATION_KIND_MAP, ObservationKind
2828
from warehouse.packaging.models import JournalEntry, Project, Release, Role
2929
from warehouse.packaging.tasks import update_release_description
3030
from warehouse.search.tasks import reindex_project as _reindex_project
3131
from warehouse.utils.paginate import paginate_url_factory
3232
from warehouse.utils.project import confirm_project, remove_project
3333

34-
ONE_MB = 1024 * 1024 # bytes
35-
ONE_GB = 1024 * 1024 * 1024 # bytes
36-
UPLOAD_LIMIT_CAP = 1073741824 # 1 GiB
34+
UPLOAD_LIMIT_CAP = ONE_GIB
3735

3836

3937
@view_config(
@@ -142,9 +140,9 @@ def project_detail(project, request):
142140
"maintainers": maintainers,
143141
"journal": journal,
144142
"oidc_publishers": project.oidc_publishers,
145-
"ONE_MB": ONE_MB,
143+
"ONE_MIB": ONE_MIB,
146144
"MAX_FILESIZE": MAX_FILESIZE,
147-
"ONE_GB": ONE_GB,
145+
"ONE_GIB": ONE_GIB,
148146
"MAX_PROJECT_SIZE": MAX_PROJECT_SIZE,
149147
"UPLOAD_LIMIT_CAP": UPLOAD_LIMIT_CAP,
150148
"observation_kinds": ObservationKind,
@@ -476,19 +474,19 @@ def set_upload_limit(project, request):
476474
f"must be integer or empty string."
477475
)
478476

479-
# The form is in MB, but the database field is in bytes.
480-
upload_limit *= ONE_MB
477+
# The form is in MiB, but the database field is in bytes.
478+
upload_limit *= ONE_MIB
481479

482480
if upload_limit > UPLOAD_LIMIT_CAP:
483481
raise HTTPBadRequest(
484482
f"Upload limit can not be more than the overall limit of "
485-
f"{UPLOAD_LIMIT_CAP / ONE_MB}MiB."
483+
f"{UPLOAD_LIMIT_CAP / ONE_MIB}MiB."
486484
)
487485

488486
if upload_limit < MAX_FILESIZE:
489487
raise HTTPBadRequest(
490488
f"Upload limit can not be less than the default limit of "
491-
f"{MAX_FILESIZE / ONE_MB}MB."
489+
f"{MAX_FILESIZE / ONE_MIB}MiB."
492490
)
493491

494492
project.upload_limit = upload_limit
@@ -521,13 +519,13 @@ def set_total_size_limit(project, request):
521519
f"must be integer or empty string."
522520
)
523521

524-
# The form is in GB, but the database field is in bytes.
525-
total_size_limit *= ONE_GB
522+
# The form is in GiB, but the database field is in bytes.
523+
total_size_limit *= ONE_GIB
526524

527525
if total_size_limit < MAX_PROJECT_SIZE:
528526
raise HTTPBadRequest(
529527
f"Total project size can not be less than the default limit of "
530-
f"{MAX_PROJECT_SIZE / ONE_GB}GB."
528+
f"{MAX_PROJECT_SIZE / ONE_GIB}GiB."
531529
)
532530

533531
project.total_size_limit = total_size_limit

‎warehouse/config.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from pyramid_rpc.xmlrpc import XMLRPCRenderer
3131

3232
from warehouse.authnz import Permissions
33+
from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB
3334
from warehouse.utils.static import ManifestCacheBuster
3435
from warehouse.utils.wsgi import ProxyFixer, VhmRootRemover
3536

@@ -216,6 +217,10 @@ def from_base64_encoded_json(configuration):
216217
def configure(settings=None):
217218
if settings is None:
218219
settings = {}
220+
settings["warehouse.forklift.legacy.MAX_FILESIZE_MIB"] = MAX_FILESIZE / ONE_MIB
221+
settings["warehouse.forklift.legacy.MAX_PROJECT_SIZE_GIB"] = (
222+
MAX_PROJECT_SIZE / ONE_GIB
223+
)
219224

220225
# Allow configuring the log level. See `warehouse/logging.py` for more
221226
maybe_set(settings, "logging.level", "LOG_LEVEL")

‎warehouse/constants.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License.
12+
13+
ONE_MIB = 1 * 1024 * 1024
14+
ONE_GIB = 1 * 1024 * 1024 * 1024
15+
MAX_FILESIZE = 100 * ONE_MIB
16+
MAX_SIGSIZE = 8 * 1024
17+
MAX_PROJECT_SIZE = 10 * ONE_GIB

‎warehouse/forklift/legacy.py‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
from warehouse.admin.flags import AdminFlagValue
4646
from warehouse.authnz import Permissions
4747
from warehouse.classifiers.models import Classifier
48+
from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB
4849
from warehouse.email import (
4950
send_api_token_used_in_trusted_publisher_project_email,
5051
send_two_factor_not_yet_enabled_email,
@@ -70,15 +71,9 @@
7071
from warehouse.rate_limiting.interfaces import RateLimiterException
7172
from warehouse.utils import readme
7273

73-
ONE_MB = 1 * 1024 * 1024
74-
ONE_GB = 1 * 1024 * 1024 * 1024
75-
76-
MAX_FILESIZE = 100 * ONE_MB
77-
MAX_PROJECT_SIZE = 10 * ONE_GB
78-
7974
PATH_HASHER = "blake2_256"
8075

81-
COMPRESSION_RATIO_MIN_SIZE = 64 * ONE_MB
76+
COMPRESSION_RATIO_MIN_SIZE = 64 * ONE_MIB
8277

8378
# If the zip file decompressed to 50x more space
8479
# than it is uncompressed, consider it a ZIP bomb.
@@ -853,7 +848,7 @@ def file_upload(request):
853848
HTTPBadRequest,
854849
"File too large. "
855850
+ "Limit for project {name!r} is {limit} MB. ".format(
856-
name=project.name, limit=file_size_limit // ONE_MB
851+
name=project.name, limit=file_size_limit // ONE_MIB
857852
)
858853
+ "See "
859854
+ request.help_url(_anchor="file-size-limit")
@@ -864,7 +859,7 @@ def file_upload(request):
864859
HTTPBadRequest,
865860
"Project size too large. Limit for "
866861
+ "project {name!r} total size is {limit} GB. ".format(
867-
name=project.name, limit=project_size_limit // ONE_GB
862+
name=project.name, limit=project_size_limit // ONE_GIB
868863
)
869864
+ "See "
870865
+ request.help_url(_anchor="project-size-limit"),

0 commit comments

Comments
 (0)