inform pants about 3rd party dependencies and constraints

Author: cognifloyd

BUILD

@@ -1,5 +1,42 @@
 python_requirements(
-    name="root",
+    name="reqs",
+    source="requirements-pants.txt",
+    module_mapping={
+        "gitpython": ["git"],
+        "python-editor": ["editor"],
+        "python-json-logger": ["pythonjsonlogger"],
+        "python-statsd": ["statsd"],
+        "sseclient-py": ["sseclient"],
+        "oslo.config": ["oslo_config"],
+        "RandomWords": ["random_words"],
+    },
+    overrides={
+        # flex uses pkg_resources w/o declaring the dep
+        "flex": {
+            "dependencies": [
+                "//:reqs#setuptools",
+            ]
+        },
+        # do not use the prance[flex] extra as that pulls in an old version of flex
+        "prance": {
+            "dependencies": [
+                "//:reqs#flex",
+            ]
+        },
+        # stevedore uses pkg_resources w/o declaring the dep
+        "stevedore": {
+            "dependencies": [
+                "//:reqs#setuptools",
+            ]
+        },
+        # tooz needs one or more backends (tooz is used by the st2 coordination backend)
+        "tooz": {
+            "dependencies": [
+                "//:reqs#redis",
+                "//:reqs#zake",
+            ]
+        },
+    },
 )
 
 python_test_utils(

contrib/core/BUILD

@@ -3,4 +3,5 @@ python_sources()
 python_requirements(
     name="reqs",
     source="requirements-tests.txt",
+    module_mapping={"mail-parser": ["mailparser"]},
 )

contrib/runners/winrm_runner/BUILD

@@ -0,0 +1,5 @@
+python_requirement(
+    name="winrm",
+    requirements=["pywinrm"],
+    modules=["winrm"],
+)

lockfiles/st2-constraints.txt

@@ -0,0 +1,68 @@
+# Add/remove version constraints for transitive dependencies in this file
+# (transitive dependencies are dependencies of our direct dependencies).
+# Then run `./pants generate-lockfiles --resolve=st2` to regenerate the lockfile.
+#
+# Direct dependencies should be recorded in `requirements-pants.txt`, not here.
+
+# ############################################ #
+# pinned transitive deps from requirements.txt #
+# ############################################ #
+#
+# required by jinja2:
+# fixed-requirements.txt:
+#     Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode
+#     >=0.23 was from jinja2
+MarkupSafe<2.1.0,>=0.23
+#
+# required by kombu:
+#amqp==5.0.6
+#
+# required by cryptography, paramiko, passlib:
+#bcrypt==3.2.0
+#
+# required by bcrypt, cryptography, pynacl, zstandard:
+#cffi<1.15.0
+#
+# required by orquesta, prance, requests:
+# fixed-requirements.txt:
+#     requests 2.23 requires chardet < 3.1.0
+#chardet<3.1.0
+#
+# required by jsonpath-rw, networkx:
+# fixed-requireements.txt:
+#     networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2
+#     but the wheel on pypi does not say it supports python3.8, so pip gets
+#     confused. For now, pin decorator to work around pip's confusion.
+#decorator==4.4.2
+#
+# required by eventlet, pymongo:
+# fixed-requireements.txt:
+#     NOTE: 2.0 version breaks pymongo work with hosts
+#dnspython>=1.16.0,<2.0.0
+#
+# required by eventlet:
+#greenlet==1.0.0
+#
+# required by argcomplete, click, debtcollector, kombu, pluggy, prettytable,
+# pytest, virtualenv:
+#importlib-metadata==3.10.1
+#
+# required by tooz:
+#oslo.utils<5.0,>=4.0.0
+#tenacity>=3.2.1,<7.0.0
+#
+# required by st2-auth-backend-flat-file:
+#passlib==1.7.4
+#
+# required by pymongo, urllib3:
+# fixed-requireements.txt:
+#     pyOpenSSL 22.0.0 requires cryptography>=35.0
+#pyOpenSSL<=21.0.0
+#
+# required by oslo.utils, packaging:
+#pyparsing<3
+#
+# required by gitpython, importlib-metadata:
+# fixed-requireements.txt:
+#     importlib-metadata requires typing-extensions but v4.2.0 requires py3.7+
+#typing-extensions<4.2

pants.toml

@@ -35,9 +35,9 @@ pants_ignore.add = [
   "st2common/tests/fixtures/requirements-used-for-tests.txt",
   "/fixed-requirements.txt",
   "/test-requirements.txt",
-  # keep requirements.txt for now. We might ignore it if we need an alternate interrim
-  # file that is decoupled from our legacy requirements files generation.
-  # "/requirements.txt",
+  # ignore requirements.txt for now, preferring interrim files that are decoupled from
+  # legacy requirements files generation: requirements-pants.txt & lockfiles/st2-constraints.txt
+  "/requirements.txt",
 ]
 
 [source]

requirements-pants.txt

@@ -0,0 +1,100 @@
+# Add/remove direct 3rd party dependencies here, with version constraints if necessary.
+# Then run `./pants generate-lockfiles --resolve=st2` to regenerate the lockfile.
+#
+# Please do not add transitive dependencies in this file (ie dependencies of our dependencies).
+# Use `lockfiles/st2-constraints.txt` to constrain the version of these transitive dependencies.
+#
+# Please keep this list alphabetical, with tooz backends in a separate list.
+
+apscheduler
+argcomplete
+ciso8601
+cryptography
+# eventlet 0.31+ and gunicorn 20.1.0 are not compatible
+eventlet<0.31
+# flex parses the openapi 2 spec in our router
+flex
+# gitpython & gitdb are used for pack management
+gitdb
+gitpython
+gunicorn
+jinja2
+jsonpath-rw
+jsonschema
+kombu
+lockfile
+mock
+mongoengine
+# Note: networkx v2.6 dropped support for Python3.6
+# networkx version is constrained in orquesta.
+networkx
+orjson
+orquesta @ git+https://github.com/StackStorm/[email protected]
+# NOTE: Recent version substantially affect the performance and add big import time overhead
+# See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details
+oslo.config>=1.12.1,<1.13
+paramiko
+# prance is used by st2-validate-api-spec to validate the openapi spec
+# prance needs flex, but do not use the extra as that gets an old version.
+prance
+prettytable
+# For st2client: prompt-toolkit v2+ does not have prompt_toolkit.token.Token
+prompt-toolkit<2
+psutil
+pymongo
+# pyrabbit used in an integration test
+pyrabbit
+pytest
+python-dateutil
+python-editor
+# pythonjsonlogger referenced in st2actions/conf/logging.conf
+python-json-logger
+python-statsd
+pytz
+PyYAML
+# RandomWords used in some tests
+RandomWords
+requests[security]
+retrying
+routes
+semver
+# setuptools provides pkg_resources
+setuptools
+simplejson
+six
+# NOTE: we use sseclient-py instead of sseclient because sseclient
+# has various issues which sometimes hang the connection for a long time, etc.
+sseclient-py
+# bandit doesn't work w/ stevedore 3+
+stevedore<3
+# For backward compatibility reasons, flat file backend is installed by default
+st2-auth-backend-flat-file @ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
+st2-auth-ldap @ git+https://github.com/StackStorm/st2-auth-ldap.git@master
+st2-rbac-backend @ git+https://github.com/StackStorm/st2-rbac-backend.git@master
+# tabulate used by tools/log_watcher.py
+tabulate
+tooz
+udatetime
+ujson
+unittest2
+virtualenv
+webob
+webtest
+# zstandard is used for micro benchmarks
+zstandard
+
+# tooz backends
+redis
+zake
+
+# was in fixed-requirements.txt, but not in requirements-pants.txt
+# keyczar is used by a python2-only test.
+#python-keyczar
+
+###########
+
+# not needed with switch to pytest
+#nose
+#nose-timer
+#nose-parallel
+#rednose