pthon/aiohttp correct which randomNumber is returned by 'update' (#2555)

* correct with randomNumber is returned by 'update'

* Aiohttp faster (#1)

marginal performance improvements and cleanup

[ci fw-only Python/aiohttp]

Author: samuelcolvin

frameworks/Python/aiohttp/app/main.py

@@ -22,6 +22,8 @@
     updates_raw,
 )
 
+CONNECTION_ORM = os.getenv('CONNECTION', 'ORM').upper() == 'ORM'
+
 THIS_DIR = Path(__file__).parent
 
 
@@ -41,30 +43,34 @@ def pg_dsn() -> str:
 
 async def startup(app: web.Application):
     dsn = pg_dsn()
-    app.update(
-        aiopg_engine=await aiopg.sa.create_engine(dsn=dsn, minsize=10, maxsize=20, loop=app.loop),
-        asyncpg_pool=await asyncpg.create_pool(dsn=dsn, min_size=10, max_size=20, loop=app.loop),
-    )
+    min_size, max_size = 50, 100
+    if CONNECTION_ORM:
+        app['pg'] = await aiopg.sa.create_engine(dsn=dsn, minsize=min_size, maxsize=max_size, loop=app.loop)
+    else:
+        app['pg'] = await asyncpg.create_pool(dsn=dsn, min_size=min_size, max_size=max_size, loop=app.loop)
 
 
 async def cleanup(app: web.Application):
-    app['aiopg_engine'].close()
-    await app['aiopg_engine'].wait_closed()
-    await app['asyncpg_pool'].close()
+    if CONNECTION_ORM:
+        app['pg'].close()
+        await app['pg'].wait_closed()
+    else:
+        await app['pg'].close()
 
 
 def setup_routes(app):
-    app.router.add_get('/json', json)
-    app.router.add_get('/db', single_database_query_orm)
-    app.router.add_get('/queries', multiple_database_queries_orm)
-    app.router.add_get('/fortunes', fortunes)
-    app.router.add_get('/updates', updates)
-    app.router.add_get('/plaintext', plaintext)
-
-    app.router.add_get('/raw/db', single_database_query_raw)
-    app.router.add_get('/raw/queries', multiple_database_queries_raw)
-    app.router.add_get('/raw/fortunes', fortunes_raw)
-    app.router.add_get('/raw/updates', updates_raw)
+    if CONNECTION_ORM:
+        app.router.add_get('/json', json)
+        app.router.add_get('/db', single_database_query_orm)
+        app.router.add_get('/queries/{queries:.*}', multiple_database_queries_orm)
+        app.router.add_get('/fortunes', fortunes)
+        app.router.add_get('/updates/{queries:.*}', updates)
+        app.router.add_get('/plaintext', plaintext)
+    else:
+        app.router.add_get('/db', single_database_query_raw)
+        app.router.add_get('/queries/{queries:.*}', multiple_database_queries_raw)
+        app.router.add_get('/fortunes', fortunes_raw)
+        app.router.add_get('/updates/{queries:.*}', updates_raw)
 
 
 def create_app(loop):

frameworks/Python/aiohttp/app/views.py

@@ -17,7 +17,7 @@ def json_response(data):
 
 def get_num_queries(request):
     try:
-        num_queries = int(request.GET.get('queries', 1))
+        num_queries = int(request.match_info.get('queries', 1))
     except ValueError:
         return 1
     if num_queries < 1:
@@ -39,7 +39,7 @@ async def single_database_query_orm(request):
     Test 2 ORM
     """
     id_ = randint(1, 10000)
-    async with request.app['aiopg_engine'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         cur = await conn.execute(select([sa_worlds.c.randomnumber]).where(sa_worlds.c.id == id_))
         r = await cur.first()
     return json_response({'id': id_, 'randomNumber': r[0]})
@@ -51,7 +51,7 @@ async def single_database_query_raw(request):
     """
     id_ = randint(1, 10000)
 
-    async with request.app['asyncpg_pool'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         r = await conn.fetchval('SELECT randomnumber FROM world WHERE id = $1', id_)
     return json_response({'id': id_, 'randomNumber': r})
 
@@ -66,7 +66,7 @@ async def multiple_database_queries_orm(request):
     ids.sort()
 
     result = []
-    async with request.app['aiopg_engine'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         for id_ in ids:
             cur = await conn.execute(select([sa_worlds.c.randomnumber]).where(sa_worlds.c.id == id_))
             r = await cur.first()
@@ -84,7 +84,7 @@ async def multiple_database_queries_raw(request):
     ids.sort()
 
     result = []
-    async with request.app['asyncpg_pool'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         stmt = await conn.prepare('SELECT randomnumber FROM world WHERE id = $1')
         for id_ in ids:
             result.append({
@@ -99,7 +99,7 @@ async def fortunes(request):
     """
     Test 4 ORM
     """
-    async with request.app['aiopg_engine'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         cur = await conn.execute(select([sa_fortunes.c.id, sa_fortunes.c.message]))
         fortunes = list(await cur.fetchall())
     fortunes.append(Fortune(id=0, message='Additional fortune added at request time.'))
@@ -112,7 +112,7 @@ async def fortunes_raw(request):
     """
     Test 4 RAW
     """
-    async with request.app['asyncpg_pool'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         fortunes = await conn.fetch('SELECT * FROM Fortune')
     fortunes.append(dict(id=0, message='Additional fortune added at request time.'))
     fortunes.sort(key=itemgetter('message'))
@@ -129,21 +129,21 @@ async def updates(request):
     ids = [randint(1, 10000) for _ in range(num_queries)]
     ids.sort()
 
-    async with request.app['aiopg_engine'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         for id_ in ids:
             cur = await conn.execute(
                 select([sa_worlds.c.randomnumber])
                 .where(sa_worlds.c.id == id_)
             )
-            r = await cur.first()
-            r = dict(r)
-            r['randomnumber'] = randint(1, 10000)
+            # the result of this is a dict with the previous random number `randomnumber` which we don't actually use
+            await cur.first()
+            rand_new = randint(1, 10000)
             await conn.execute(
                 sa_worlds.update()
                 .where(sa_worlds.c.id == id_)
-                .values(randomnumber=r['randomnumber'])
+                .values(randomnumber=rand_new)
             )
-            result.append({'id': id_, 'randomNumber': r['randomnumber']})
+            result.append({'id': id_, 'randomNumber': rand_new})
     return json_response(result)
 
 async def updates_raw(request):
@@ -157,16 +157,14 @@ async def updates_raw(request):
 
     result = []
     updates = []
-    async with request.app['asyncpg_pool'].acquire() as conn:
+    async with request.app['pg'].acquire() as conn:
         stmt = await conn.prepare('SELECT randomnumber FROM world WHERE id = $1')
         for id_ in ids:
-            r = {
-                'id': id_,
-                'randomNumber': await stmt.fetchval(id_)
-            }
-            r['randomNumber'] = randint(1, 10000)
-            result.append(r)
-            updates.append((r['randomNumber'], id_))
+            # the result of this is the int previous random number which we don't actually use
+            await stmt.fetchval(id_)
+            rand_new = randint(1, 10000)
+            result.append({'id': id_, 'randomNumber': rand_new})
+            updates.append((rand_new, id_))
         await conn.executemany('UPDATE world SET randomnumber=$1 WHERE id=$2', updates)
 
     return json_response(result)

frameworks/Python/aiohttp/benchmark_config.json

@@ -5,9 +5,9 @@
       "setup_file": "setup",
       "json_url": "/json",
       "db_url": "/db",
-      "query_url": "/queries?queries=",
+      "query_url": "/queries/",
       "fortune_url": "/fortunes",
-      "update_url": "/updates?queries=",
+      "update_url": "/updates/",
       "plaintext_url": "/plaintext",
       "port": 8080,
       "approach": "Realistic",
@@ -25,11 +25,11 @@
       "notes": "uses aiopg with sqlalchemy for database access"
     },
     "pg-raw": {
-      "setup_file": "setup",
-      "db_url": "/raw/db",
-      "query_url": "/raw/queries?queries=",
-      "fortune_url": "/raw/fortunes",
-      "update_url": "/raw/updates?queries=",
+      "setup_file": "setup_raw",
+      "db_url": "/db",
+      "query_url": "/queries/",
+      "fortune_url": "/fortunes",
+      "update_url": "/updates/",
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",

frameworks/Python/aiohttp/setup_raw.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export CONNECTION=RAW
+
+source $TROOT/setup.sh