RHIROS-987 - upgrade flask,werkzeug to newer versions

Pipfile

@@ -11,27 +11,25 @@ pytest-mock = "*"
 pytest-cov = "*"
 
 [packages]
-Flask = "~=1.1.2"
+Flask = "~=2.2.3"
 gunicorn = "~=20.0.4"
 flask-restful = "~=0.3.8"
-Flask-SQLAlchemy = "~=2.4.4"
-sqlalchemy = "<1.4"
+Flask-SQLAlchemy = "~=3.0.3"
+sqlalchemy = "~=2.0"
 psycopg2-binary = "~=2.8.6"
-flask-migrate = "~=2.5.3"
-flask-script = "~=2.0.6"
+flask-migrate = "~=4.0.4"
 flask-cors = "~=3.0.9"
 confluent-kafka = "~=1.5.0"
 requests = "~=2.31.0"
 insights-core = "~=3.0.265"
 pydash = "~=4.9.2"
 app-common-python = "~=0.2.5"
 prometheus-client = "~=0.11.0"
-prometheus-flask-exporter = "~=0.18.2"
+prometheus-flask-exporter = "~=0.22.3"
 pyyaml = "*"
 watchtower = ">=1.0.0"
 boto3 = ">=1.16.13"
 botocore = ">=1.19.13"
-markupsafe = "==2.0.1"  # Necessary locking, issue: https://github.com/aws/aws-sam-cli/issues/3661
 flask-caching = "*"
 
 [requires]

README.md

@@ -41,8 +41,9 @@ README.md file in scripts directory.
 #### Initialize the database
 Run the following commands to excute the db migration scripts.
 ```bash
-python manage.py db upgrade
-python manage.py seed 
+export FLASK_APP=manage.py
+flask db upgrade
+flask seed
 ```
 
 #### Running the processor locally

clowdapp.yaml

@@ -24,7 +24,7 @@ objects:
       podSpec:
         image: ${IMAGE}:${IMAGE_TAG}
         command: ["sh"]
-        args: ["-c", "python -m manage db upgrade && python -m manage seed && gunicorn ros.api.main:app"]
+        args: ["-c", "export FLASK_APP=manage.py && flask db upgrade && flask seed && gunicorn ros.api.main:app"]
         resources:
           requests:
             cpu: ${CPU_REQUEST}

manage.py

@@ -1,14 +1,13 @@
-from flask_migrate import Migrate
-from flask_migrate import MigrateCommand
-from flask_script import Manager
-
-from ros.lib.app import app, db
+from ros.lib.app import app
+import click
+from flask.cli import with_appcontext
 from seed import Seed
 
-migrate = Migrate(app, db)
-manager = Manager(app)
-manager.add_command("db", MigrateCommand)
-manager.add_command('seed', Seed())
 
-if __name__ == "__main__":
-    manager.run()
+@click.command(name='seed')
+@with_appcontext
+def seed():
+    Seed().run()
+
+
+app.cli.add_command(seed)

migrations/env.py

@@ -1,10 +1,7 @@
-from __future__ import with_statement
-
 import logging
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
+from flask import current_app
 
 from alembic import context
 
@@ -17,22 +14,43 @@
 fileConfig(config.config_file_name)
 logger = logging.getLogger('alembic.env')
 
+
+def get_engine():
+    try:
+        # this works with Flask-SQLAlchemy<3 and Alchemical
+        return current_app.extensions['migrate'].db.get_engine()
+    except TypeError:
+        # this works with Flask-SQLAlchemy>=3
+        return current_app.extensions['migrate'].db.engine
+
+
+def get_engine_url():
+    try:
+        return get_engine().url.render_as_string(hide_password=False).replace(
+            '%', '%%')
+    except AttributeError:
+        return str(get_engine().url).replace('%', '%%')
+
+
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from flask import current_app
-config.set_main_option(
-    'sqlalchemy.url',
-    str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%'))
-target_metadata = current_app.extensions['migrate'].db.metadata
+config.set_main_option('sqlalchemy.url', get_engine_url())
+target_db = current_app.extensions['migrate'].db
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
 
+def get_metadata():
+    if hasattr(target_db, 'metadatas'):
+        return target_db.metadatas[None]
+    return target_db.metadata
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -47,7 +65,7 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True
+        url=url, target_metadata=get_metadata(), literal_binds=True
     )
 
     with context.begin_transaction():
@@ -72,16 +90,12 @@ def process_revision_directives(context, revision, directives):
                 directives[:] = []
                 logger.info('No changes in schema detected.')
 
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
-        poolclass=pool.NullPool,
-    )
+    connectable = get_engine()
 
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata,
+            target_metadata=get_metadata(),
             process_revision_directives=process_revision_directives,
             **current_app.extensions['migrate'].configure_args
         )

ros/api/common/utils.py

@@ -9,7 +9,8 @@
     is_valid_uuid,
     system_ids_by_org_id
 )
-
+from ros.extensions import db
+from sqlalchemy import exc
 
 LOG = logging.getLogger(__name__)
 prefix = "VALIDATE REQUEST"
@@ -52,13 +53,13 @@ def check_for_user():
         return ident, username
 
     def check_for_system(ident, inventory_id):
+        system_id = None
         if not is_valid_uuid(inventory_id):
             abort(404, message='Invalid inventory_id, Id should be in form of UUID4')
-
-        system_id = system_ids_by_org_id(ident['org_id']).filter(
-            System.inventory_id == inventory_id).first()
-
-        if system_id is None:
+        systems = system_ids_by_org_id(ident['org_id']).filter(System.inventory_id == inventory_id)
+        try:
+            system_id = db.session.execute(systems).scalar_one()
+        except exc.NoResultFound:
             abort(404, message=f"System {inventory_id} doesn't exist.")
 
         return system_id
@@ -77,6 +78,7 @@ def validate_request(*args, **kwargs):
 
         inventory_id = data['inventory_id']
         system_id = check_for_system(ident, inventory_id)
+
         rating = check_for_rating(data)
         new_kwargs = {
             'rating': rating, 'username': username,

ros/api/main.py

@@ -1,7 +1,9 @@
 from flask_restful import Api
 from flask_cors import CORS
+from ros.lib.app import app
 from ros.api.routes import initialize_routes
-from ros.lib.app import app, metrics, cache
+from ros.extensions import metrics
+from ros.extensions import cache
 from ros.lib.cw_logging import commence_cw_log_streaming
 from ros.lib.config import ROS_API_PORT
 

ros/api/v1/call_to_action.py

@@ -4,7 +4,7 @@
     org_id_from_identity_header, systems_ids_for_existing_profiles)
 from ros.lib.models import (
     PerformanceProfile)
-from ros.lib.app import cache
+from ros.extensions import cache
 
 
 class CallToActionApi(Resource):

ros/api/v1/hosts.py

@@ -133,9 +133,10 @@ def get(self):
             db.session.query(PerformanceProfile, System, RhAccount)
             .join(System, System.id == PerformanceProfile.system_id)
             .join(RhAccount, RhAccount.id == System.tenant_id)
-            .filter(PerformanceProfile.system_id.in_(system_query.subquery()))
+            .filter(PerformanceProfile.system_id.in_(system_query))
             .order_by(*sort_expression)
         )
+
         count = query.count()
         # NOTE: Override limit value to get all the systems when it is -1
         if limit == -1:
@@ -283,7 +284,7 @@ def get(self, host_id):
         username = user['username'] if 'username' in user else None
         org_id = org_id_from_identity_header(request)
 
-        system_query = system_ids_by_org_id(org_id).filter(System.inventory_id == host_id).subquery()
+        system_query = system_ids_by_org_id(org_id).filter(System.inventory_id == host_id)
 
         profile = PerformanceProfile.query.filter(
             PerformanceProfile.system_id.in_(system_query)).first()
@@ -293,7 +294,7 @@ def get(self, host_id):
             RecommendationRating.rated_by == username
         ).first()
 
-        system = db.session.query(System).filter(System.inventory_id == host_id).first()
+        system = db.session.scalar(db.select(System).filter(System.inventory_id == host_id))
 
         record = None
         if profile:
@@ -347,7 +348,7 @@ def get(self, host_id):
 
         org_id = org_id_from_identity_header(request)
 
-        system_query = system_ids_by_org_id(org_id).filter(System.inventory_id == host_id).subquery()
+        system_query = system_ids_by_org_id(org_id).filter(System.inventory_id == host_id)
 
         query = PerformanceProfileHistory.query.filter(
             PerformanceProfileHistory.system_id.in_(system_query)

ros/api/v1/recommendations.py

@@ -3,12 +3,12 @@
 from ros.api.modules.recommendations import Recommendation
 from flask_restful import Resource, abort, fields, marshal_with
 from flask import request
+from sqlalchemy import exc
 
 
 class RecommendationsApi(Resource):
-
     recommendation_fields = {
-        'rule_id':  fields.String,
+        'rule_id': fields.String,
         'description': fields.String,
         'reason': fields.String,
         'resolution': fields.String,
@@ -31,18 +31,19 @@ class RecommendationsApi(Resource):
 
     @marshal_with(data_fields)
     def get(self, host_id):
+        system = None
         if not is_valid_uuid(host_id):
             abort(404, message='Invalid host_id, Id should be in form of UUID4')
 
         ident = identity(request)['identity']
 
         filter_description = request.args.get('description')
 
-        system = system_ids_by_org_id(ident['org_id'], True).filter(System.inventory_id == host_id).first()
-
-        if not system:
-            abort(404, message="host with id {} doesn't exist"
-                  .format(host_id))
+        system_query = system_ids_by_org_id(ident['org_id'], True).filter(System.inventory_id == host_id)
+        try:
+            system = db.session.execute(system_query).scalar_one()
+        except exc.NoResultFound:
+            abort(404, message=f"System {host_id} doesn't exist.")
 
         profile = PerformanceProfile.query.filter_by(system_id=system.id).first()
         if not profile:
@@ -56,19 +57,18 @@ def get(self, host_id):
         if rule_hits:
             for rule_hit in rule_hits:
                 if filter_description:
-                    rule_data = db.session.query(Rule).filter(Rule.rule_id == rule_hit['rule_id'])\
-                                .filter(Rule.description.ilike(f'%{filter_description}%')).first()
+                    rule_data = db.session.scalar(db.select(Rule).filter(Rule.rule_id == rule_hit['rule_id'])
+                                                  .filter(Rule.description.ilike(f'%{filter_description}%')))
                 else:
-                    rule_data = db.session.query(Rule).filter(
-                        Rule.rule_id == rule_hit['rule_id']).first()
+                    rule_data = db.session.scalar(db.select(Rule).filter(Rule.rule_id == rule_hit['rule_id']))
 
                 if rule_data:
                     recommendation = Recommendation(
                         rule_data, rule_hit, system, psi_enabled
                     ).__dict__
                     recommendations_list.append(recommendation)
         return {
-                  'inventory_id': system.inventory_id,
-                  'data': recommendations_list,
-                  'meta': {'count': len(recommendations_list)}
-            }
+            'inventory_id': system.inventory_id,
+            'data': recommendations_list,
+            'meta': {'count': len(recommendations_list)}
+        }

ros/api/v1/status.py

@@ -1,5 +1,5 @@
 from flask_restful import Resource
-from ros.lib.app import metrics
+from ros.extensions import metrics
 
 
 class Status(Resource):

ros/extensions.py

@@ -0,0 +1,9 @@
+from flask_caching import Cache
+from ros.lib.config import REDIS_URL
+from prometheus_flask_exporter import RESTfulPrometheusMetrics
+from flask_sqlalchemy import SQLAlchemy
+
+
+cache = Cache(config={'CACHE_TYPE': 'RedisCache', 'CACHE_REDIS_URL': REDIS_URL, 'CACHE_DEFAULT_TIMEOUT': 300})
+metrics = RESTfulPrometheusMetrics.for_app_factory(defaults_prefix='ros', group_by='url_rule')
+db = SQLAlchemy()

ros/lib/app.py

@@ -1,24 +1,29 @@
 from flask import Flask
-from .models import db
+from ros.extensions import db
 from .config import DB_URI, DB_POOL_SIZE, DB_MAX_OVERFLOW
 from flask import request
 from .rbac_interface import ensure_has_permission
-from .config import get_logger, REDIS_URL
-from prometheus_flask_exporter import RESTfulPrometheusMetrics
-from flask_caching import Cache
-
+from .config import get_logger
+from flask_migrate import Migrate
 # Since we're using flask_sqlalchemy, we must create the flask app in both processor and web api
-app = Flask(__name__)
-# Initalize database connection
-app.config['SQLALCHEMY_DATABASE_URI'] = DB_URI
-app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
-    'pool_size': DB_POOL_SIZE,
-    'max_overflow': DB_MAX_OVERFLOW
-}
-app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-metrics = RESTfulPrometheusMetrics.for_app_factory(defaults_prefix='ros', group_by='url_rule')
-cache = Cache(config={'CACHE_TYPE': 'RedisCache', 'CACHE_REDIS_URL': REDIS_URL, 'CACHE_DEFAULT_TIMEOUT': 300})
-db.init_app(app)
+
+
+def create_app():
+    app = Flask(__name__)
+    # Initalize database connection
+    app.config['SQLALCHEMY_DATABASE_URI'] = DB_URI
+    app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
+        'pool_size': DB_POOL_SIZE,
+        'max_overflow': DB_MAX_OVERFLOW
+    }
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    db.init_app(app)
+    migrate = Migrate()
+    migrate.init_app(app, db)
+    return app
+
+
+app = create_app()
 
 
 @app.before_request