diff --git a/.gitignore b/.gitignore index f9675679..3a2b94ee 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ htmlcov/ # Weaviate weaviate_data/ + +# Codegate Dashboard DB +codegate.db diff --git a/poetry.lock b/poetry.lock index 1c924a7d..0b7886bf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -122,6 +122,21 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "aiosqlite" +version = "0.19.0" +description = "asyncio bridge to the standard sqlite3 module" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosqlite-0.19.0-py3-none-any.whl", hash = "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"}, + {file = "aiosqlite-0.19.0.tar.gz", hash = "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d"}, +] + +[package.extras] +dev = ["aiounittest (==1.4.1)", "attribution (==1.6.2)", "black (==23.3.0)", "coverage[toml] (==7.2.3)", "flake8 (==5.0.4)", "flake8-bugbear (==23.3.12)", "flit (==3.7.1)", "mypy (==1.2.0)", "ufmt (==2.1.0)", "usort (==1.0.6)"] +docs = ["sphinx (==6.1.3)", "sphinx-mdinclude (==0.5.3)"] + [[package]] name = "annotated-types" version = "0.7.0" @@ -829,6 +844,92 @@ test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe, test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] tqdm = ["tqdm"] +[[package]] +name = "greenlet" +version = "3.1.1" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, + {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, + {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, + {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, + {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, + {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, + {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, + {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"}, + {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"}, + {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"}, + {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, + {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, + {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, + {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, + {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, + {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, + {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + [[package]] name = "grpcio" version = "1.68.0" @@ -2737,6 +2838,101 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "sqlalchemy" +version = "2.0.36" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-win32.whl", hash = "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-win_amd64.whl", hash = "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-win32.whl", hash = "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-win_amd64.whl", hash = "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-win32.whl", hash = "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-win_amd64.whl", hash = "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-win32.whl", hash = "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-win_amd64.whl", hash = "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:be9812b766cad94a25bc63bec11f88c4ad3629a0cec1cd5d4ba48dc23860486b"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aae840ebbd6cdd41af1c14590e5741665e5272d2fee999306673a1bb1fdb4d"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4557e1f11c5f653ebfdd924f3f9d5ebfc718283b0b9beebaa5dd6b77ec290971"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07b441f7d03b9a66299ce7ccf3ef2900abc81c0db434f42a5694a37bd73870f2"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:28120ef39c92c2dd60f2721af9328479516844c6b550b077ca450c7d7dc68575"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-win32.whl", hash = "sha256:b81ee3d84803fd42d0b154cb6892ae57ea6b7c55d8359a02379965706c7efe6c"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-win_amd64.whl", hash = "sha256:f942a799516184c855e1a32fbc7b29d7e571b52612647866d4ec1c3242578fcb"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3d6718667da04294d7df1670d70eeddd414f313738d20a6f1d1f379e3139a545"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:72c28b84b174ce8af8504ca28ae9347d317f9dba3999e5981a3cd441f3712e24"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-win32.whl", hash = "sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-win_amd64.whl", hash = "sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc022184d3e5cacc9579e41805a681187650e170eb2fd70e28b86192a479dcaa"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b817d41d692bf286abc181f8af476c4fbef3fd05e798777492618378448ee689"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-win32.whl", hash = "sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-win_amd64.whl", hash = "sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a"}, + {file = "SQLAlchemy-2.0.36-py3-none-any.whl", hash = "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e"}, + {file = "sqlalchemy-2.0.36.tar.gz", hash = "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + [[package]] name = "starlette" version = "0.41.3" @@ -3366,4 +3562,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.11" -content-hash = "91b84718c2ece34bca2a3437b0215f5010d0bda333cf827253e1435d9f93af30" +content-hash = "492586df32fd1993c640964e06bac0dfe20c85d2d38ee89178594e3c66f894e9" diff --git a/pyproject.toml b/pyproject.toml index 5d6ed018..a96cdf4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,11 @@ transformers = ">=4.46.3" structlog = ">=24.4.0" litellm = "^1.52.16" llama_cpp_python = ">=0.3.2" - cryptography = "^44.0.0" +sqlalchemy = "^2.0.28" +greenlet = "^3.0.3" +aiosqlite = "^0.19.0" + [tool.poetry.group.dev.dependencies] pytest = ">=7.4.0" pytest-cov = ">=4.1.0" diff --git a/sql/queries/queries.sql b/sql/queries/queries.sql new file mode 100644 index 00000000..9c1e319e --- /dev/null +++ b/sql/queries/queries.sql @@ -0,0 +1,89 @@ +-- name: CreatePrompt :one +INSERT INTO prompts ( + id, + timestamp, + provider, + system_prompt, + user_prompt, + type +) VALUES (?, ?, ?, ?, ?, ?) RETURNING *; + +-- name: GetPrompt :one +SELECT * FROM prompts WHERE id = ?; + +-- name: ListPrompts :many +SELECT * FROM prompts +ORDER BY timestamp DESC +LIMIT ? OFFSET ?; + +-- name: CreateOutput :one +INSERT INTO outputs ( + id, + prompt_id, + timestamp, + output +) VALUES (?, ?, ?, ?) RETURNING *; + +-- name: GetOutput :one +SELECT * FROM outputs WHERE id = ?; + +-- name: GetOutputsByPromptId :many +SELECT * FROM outputs +WHERE prompt_id = ? +ORDER BY timestamp DESC; + +-- name: CreateAlert :one +INSERT INTO alerts ( + id, + prompt_id, + output_id, + code_snippet, + trigger_string, + trigger_type, + trigger_category, + timestamp +) VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING *; + +-- name: GetAlert :one +SELECT * FROM alerts WHERE id = ?; + +-- name: ListAlertsByPrompt :many +SELECT * FROM alerts +WHERE prompt_id = ? +ORDER BY timestamp DESC; + +-- name: GetSettings :one +SELECT * FROM settings ORDER BY id LIMIT 1; + +-- name: UpsertSettings :one +INSERT INTO settings ( + id, + ip, + port, + llm_model, + system_prompt, + other_settings +) VALUES (?, ?, ?, ?, ?, ?) +ON CONFLICT(id) DO UPDATE SET + ip = excluded.ip, + port = excluded.port, + llm_model = excluded.llm_model, + system_prompt = excluded.system_prompt, + other_settings = excluded.other_settings +RETURNING *; + +-- name: GetPromptWithOutputsAndAlerts :many +SELECT + p.*, + o.id as output_id, + o.output, + a.id as alert_id, + a.code_snippet, + a.trigger_string, + a.trigger_type, + a.trigger_category +FROM prompts p +LEFT JOIN outputs o ON p.id = o.prompt_id +LEFT JOIN alerts a ON p.id = a.prompt_id +WHERE p.id = ? +ORDER BY o.timestamp DESC, a.timestamp DESC; diff --git a/sql/schema/schema.sql b/sql/schema/schema.sql new file mode 100644 index 00000000..059e8ef2 --- /dev/null +++ b/sql/schema/schema.sql @@ -0,0 +1,52 @@ +-- Schema for codegate database using SQLite + +-- Prompts table +CREATE TABLE prompts ( + id TEXT PRIMARY KEY, -- UUID stored as TEXT + timestamp DATETIME NOT NULL, + provider TEXT, -- VARCHAR(255) + system_prompt TEXT, + user_prompt TEXT NOT NULL, + type TEXT NOT NULL -- VARCHAR(50) (e.g. "fim", "chat") +); + +-- Outputs table +CREATE TABLE outputs ( + id TEXT PRIMARY KEY, -- UUID stored as TEXT + prompt_id TEXT NOT NULL, + timestamp DATETIME NOT NULL, + output TEXT NOT NULL, + FOREIGN KEY (prompt_id) REFERENCES prompts(id) +); + +-- Alerts table +CREATE TABLE alerts ( + id TEXT PRIMARY KEY, -- UUID stored as TEXT + prompt_id TEXT NOT NULL, + output_id TEXT NOT NULL, + code_snippet TEXT NOT NULL, -- VARCHAR(255) + trigger_string TEXT NOT NULL, -- VARCHAR(255) + trigger_type TEXT NOT NULL, -- VARCHAR(50) + trigger_category TEXT, + timestamp DATETIME NOT NULL, + FOREIGN KEY (prompt_id) REFERENCES prompts(id), + FOREIGN KEY (output_id) REFERENCES outputs(id) +); + +-- Settings table +CREATE TABLE settings ( + id TEXT PRIMARY KEY, -- UUID stored as TEXT + ip TEXT, -- VARCHAR(45) + port INTEGER, + llm_model TEXT, -- VARCHAR(255) + system_prompt TEXT, + other_settings TEXT -- JSON stored as TEXT +); + +-- Create indexes for foreign keys and frequently queried columns +CREATE INDEX idx_outputs_prompt_id ON outputs(prompt_id); +CREATE INDEX idx_alerts_prompt_id ON alerts(prompt_id); +CREATE INDEX idx_alerts_output_id ON alerts(output_id); +CREATE INDEX idx_prompts_timestamp ON prompts(timestamp); +CREATE INDEX idx_outputs_timestamp ON outputs(timestamp); +CREATE INDEX idx_alerts_timestamp ON alerts(timestamp); diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 00000000..563a57a4 --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,20 @@ +version: "2" +plugins: + - name: "python" + wasm: + url: "https://downloads.sqlc.dev/plugin/sqlc-gen-python_1.2.0.wasm" + sha256: "a6c5d174c407007c3717eea36ff0882744346e6ba991f92f71d6ab2895204c0e" + +sql: + - engine: "sqlite" + schema: "sql/schema" + queries: "sql/queries" + codegen: + - plugin: "python" + out: "src/codegate/db" + options: + package: "codegate.db" + emit_sync_querier: true + emit_async_querier: true + query_parameter_limit: 5 + emit_pydantic_models: true diff --git a/src/codegate/db/connection.py b/src/codegate/db/connection.py new file mode 100644 index 00000000..0e8d5aad --- /dev/null +++ b/src/codegate/db/connection.py @@ -0,0 +1,137 @@ +import asyncio +import datetime +import uuid +from pathlib import Path +from typing import Optional + +import structlog +from litellm import ChatCompletionRequest +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import create_async_engine + +from codegate.db.models import Prompt + +logger = structlog.get_logger("codegate") + + +class DbRecorder: + + def __init__(self, sqlite_path: Optional[str] = None): + # Initialize SQLite database engine with proper async URL + if not sqlite_path: + current_dir = Path(__file__).parent + self._db_path = (current_dir.parent.parent.parent / "codegate.db").absolute() + else: + self._db_path = Path(sqlite_path).absolute() + + logger.debug(f"Initializing DB from path: {self._db_path}") + engine_dict = { + "url": f"sqlite+aiosqlite:///{self._db_path}", + "echo": True, # Set to False in production + "isolation_level": "AUTOCOMMIT", # Required for SQLite + } + self._async_db_engine = create_async_engine(**engine_dict) + self._db_engine = create_engine(**engine_dict) + + if not self.does_db_exist(): + logger.info(f"Database does not exist at {self._db_path}. Creating..") + asyncio.run(self.init_db()) + + def does_db_exist(self): + return self._db_path.is_file() + + async def init_db(self): + """Initialize the database with the schema.""" + if self.does_db_exist(): + logger.info("Database already exists. Skipping initialization.") + return + + # Get the absolute path to the schema file + current_dir = Path(__file__).parent + schema_path = current_dir.parent.parent.parent / "sql" / "schema" / "schema.sql" + + if not schema_path.exists(): + raise FileNotFoundError(f"Schema file not found at {schema_path}") + + # Read the schema + with open(schema_path, "r") as f: + schema = f.read() + + try: + # Execute the schema + async with self._async_db_engine.begin() as conn: + # Split the schema into individual statements and execute each one + statements = [stmt.strip() for stmt in schema.split(";") if stmt.strip()] + for statement in statements: + # Use SQLAlchemy text() to create executable SQL statements + await conn.execute(text(statement)) + finally: + await self._async_db_engine.dispose() + + async def record_request( + self, normalized_request: ChatCompletionRequest, is_fim_request: bool, provider_str: str + ) -> Optional[Prompt]: + # Extract system prompt and user prompt from the messages + messages = normalized_request.get("messages", []) + system_prompt = [] + user_prompt = [] + + for msg in messages: + if msg.get("role") == "system": + system_prompt.append(msg.get("content")) + elif msg.get("role") == "user": + user_prompt.append(msg.get("content")) + + # If no user prompt found in messages, try to get from the prompt field + # (for non-chat completions) + if not user_prompt: + prompt = normalized_request.get("prompt") + if prompt: + user_prompt.append(prompt) + + if not user_prompt: + logger.warning("No user prompt found in request.") + return None + + # Create a new prompt record + prompt_params = Prompt( + id=str(uuid.uuid4()), # Generate a new UUID for the prompt + timestamp=datetime.datetime.now(datetime.timezone.utc), + provider=provider_str, + type="fim" if is_fim_request else "chat", + user_prompt="<|>".join(user_prompt), + system_prompt="<|>".join(system_prompt), + ) + # There is a `create_prompt` method in queries.py automatically generated by sqlc + # However, the method is is buggy and doesn't work as expected. + # Manually writing the SQL query to insert the prompt record. + async with self._async_db_engine.begin() as conn: + sql = text( + """ + INSERT INTO prompts (id, timestamp, provider, system_prompt, user_prompt, type) + VALUES (:id, :timestamp, :provider, :system_prompt, :user_prompt, :type) + RETURNING * + """ + ) + result = await conn.execute(sql, prompt_params.model_dump()) + row = result.first() + if row is None: + return None + + return Prompt( + id=row.id, + timestamp=row.timestamp, + provider=row.provider, + system_prompt=row.system_prompt, + user_prompt=row.user_prompt, + type=row.type, + ) + + +def init_db_sync(): + """DB will be initialized in the constructor in case it doesn't exist.""" + DbRecorder() + + +if __name__ == "__main__": + init_db_sync() diff --git a/src/codegate/db/models.py b/src/codegate/db/models.py new file mode 100644 index 00000000..b5edc0d6 --- /dev/null +++ b/src/codegate/db/models.py @@ -0,0 +1,42 @@ +# Code generated by sqlc. DO NOT EDIT. +# versions: +# sqlc v1.27.0 +from typing import Any, Optional + +import pydantic + + +class Alert(pydantic.BaseModel): + id: Any + prompt_id: Any + output_id: Any + code_snippet: Any + trigger_string: Any + trigger_type: Any + trigger_category: Optional[Any] + timestamp: Any + + +class Output(pydantic.BaseModel): + id: Any + prompt_id: Any + timestamp: Any + output: Any + + +class Prompt(pydantic.BaseModel): + id: Any + timestamp: Any + provider: Optional[Any] + system_prompt: Optional[Any] + user_prompt: Any + type: Any + + +class Setting(pydantic.BaseModel): + id: Any + ip: Optional[Any] + port: Optional[Any] + llm_model: Optional[Any] + system_prompt: Optional[Any] + other_settings: Optional[Any] diff --git a/src/codegate/db/queries.py b/src/codegate/db/queries.py new file mode 100644 index 00000000..fdfd0693 --- /dev/null +++ b/src/codegate/db/queries.py @@ -0,0 +1,600 @@ +# Code generated by sqlc. DO NOT EDIT. +# versions: +# sqlc v1.27.0 +# source: queries.sql +from typing import Any, AsyncIterator, Iterator, Optional + +import pydantic +import sqlalchemy +import sqlalchemy.ext.asyncio + +from codegate.db import models + +CREATE_ALERT = """-- name: create_alert \\:one +INSERT INTO alerts ( + id, + prompt_id, + output_id, + code_snippet, + trigger_string, + trigger_type, + trigger_category, + timestamp +) VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id, prompt_id, output_id, code_snippet, +trigger_string, trigger_type, trigger_category, timestamp +""" + + +class CreateAlertParams(pydantic.BaseModel): + id: Any + prompt_id: Any + output_id: Any + code_snippet: Any + trigger_string: Any + trigger_type: Any + trigger_category: Optional[Any] + timestamp: Any + + +CREATE_OUTPUT = """-- name: create_output \\:one +INSERT INTO outputs ( + id, + prompt_id, + timestamp, + output +) VALUES (?, ?, ?, ?) RETURNING id, prompt_id, timestamp, output +""" + + +CREATE_PROMPT = """-- name: create_prompt \\:one +INSERT INTO prompts ( + id, + timestamp, + provider, + system_prompt, + user_prompt, + type +) VALUES (?, ?, ?, ?, ?, ?) RETURNING id, timestamp, provider, system_prompt, user_prompt, type +""" + + +class CreatePromptParams(pydantic.BaseModel): + id: Any + timestamp: Any + provider: Optional[Any] + system_prompt: Optional[Any] + user_prompt: Any + type: Any + + +GET_ALERT = """-- name: get_alert \\:one +SELECT id, prompt_id, output_id, code_snippet, trigger_string, trigger_type, trigger_category, +timestamp FROM alerts WHERE id = ? +""" + + +GET_OUTPUT = """-- name: get_output \\:one +SELECT id, prompt_id, timestamp, output FROM outputs WHERE id = ? +""" + + +GET_OUTPUTS_BY_PROMPT_ID = """-- name: get_outputs_by_prompt_id \\:many +SELECT id, prompt_id, timestamp, output FROM outputs +WHERE prompt_id = ? +ORDER BY timestamp DESC +""" + + +GET_PROMPT = """-- name: get_prompt \\:one +SELECT id, timestamp, provider, system_prompt, user_prompt, type FROM prompts WHERE id = ? +""" + + +GET_PROMPT_WITH_OUTPUTS_AND_ALERTS = """-- name: get_prompt_with_outputs_and_alerts \\:many +SELECT + p.id, p.timestamp, p.provider, p.system_prompt, p.user_prompt, p.type, + o.id as output_id, + o.output, + a.id as alert_id, + a.code_snippet, + a.trigger_string, + a.trigger_type, + a.trigger_category +FROM prompts p +LEFT JOIN outputs o ON p.id = o.prompt_id +LEFT JOIN alerts a ON p.id = a.prompt_id +WHERE p.id = ? +ORDER BY o.timestamp DESC, a.timestamp DESC +""" + + +class GetPromptWithOutputsAndAlertsRow(pydantic.BaseModel): + id: Any + timestamp: Any + provider: Optional[Any] + system_prompt: Optional[Any] + user_prompt: Any + type: Any + output_id: Optional[Any] + output: Optional[Any] + alert_id: Optional[Any] + code_snippet: Optional[Any] + trigger_string: Optional[Any] + trigger_type: Optional[Any] + trigger_category: Optional[Any] + + +GET_SETTINGS = """-- name: get_settings \\:one +SELECT id, ip, port, llm_model, system_prompt, other_settings FROM settings ORDER BY id LIMIT 1 +""" + + +LIST_ALERTS_BY_PROMPT = """-- name: list_alerts_by_prompt \\:many +SELECT id, prompt_id, output_id, code_snippet, trigger_string, trigger_type, trigger_category, +timestamp FROM alerts +WHERE prompt_id = ? +ORDER BY timestamp DESC +""" + + +LIST_PROMPTS = """-- name: list_prompts \\:many +SELECT id, timestamp, provider, system_prompt, user_prompt, type FROM prompts +ORDER BY timestamp DESC +LIMIT ? OFFSET ? +""" + + +UPSERT_SETTINGS = """-- name: upsert_settings \\:one +INSERT INTO settings ( + id, + ip, + port, + llm_model, + system_prompt, + other_settings +) VALUES (?, ?, ?, ?, ?, ?) +ON CONFLICT(id) DO UPDATE SET + ip = excluded.ip, + port = excluded.port, + llm_model = excluded.llm_model, + system_prompt = excluded.system_prompt, + other_settings = excluded.other_settings +RETURNING id, ip, port, llm_model, system_prompt, other_settings +""" + + +class UpsertSettingsParams(pydantic.BaseModel): + id: Any + ip: Optional[Any] + port: Optional[Any] + llm_model: Optional[Any] + system_prompt: Optional[Any] + other_settings: Optional[Any] + + +class Querier: + def __init__(self, conn: sqlalchemy.engine.Connection): + self._conn = conn + + def create_alert(self, arg: CreateAlertParams) -> Optional[models.Alert]: + row = self._conn.execute( + sqlalchemy.text(CREATE_ALERT), + { + "p1": arg.id, + "p2": arg.prompt_id, + "p3": arg.output_id, + "p4": arg.code_snippet, + "p5": arg.trigger_string, + "p6": arg.trigger_type, + "p7": arg.trigger_category, + "p8": arg.timestamp, + }, + ).first() + if row is None: + return None + return models.Alert( + id=row[0], + prompt_id=row[1], + output_id=row[2], + code_snippet=row[3], + trigger_string=row[4], + trigger_type=row[5], + trigger_category=row[6], + timestamp=row[7], + ) + + def create_output( + self, *, id: Any, prompt_id: Any, timestamp: Any, output: Any + ) -> Optional[models.Output]: + row = self._conn.execute( + sqlalchemy.text(CREATE_OUTPUT), + { + "p1": id, + "p2": prompt_id, + "p3": timestamp, + "p4": output, + }, + ).first() + if row is None: + return None + return models.Output( + id=row[0], + prompt_id=row[1], + timestamp=row[2], + output=row[3], + ) + + def create_prompt(self, arg: CreatePromptParams) -> Optional[models.Prompt]: + row = self._conn.execute( + sqlalchemy.text(CREATE_PROMPT), + { + "p1": arg.id, + "p2": arg.timestamp, + "p3": arg.provider, + "p4": arg.system_prompt, + "p5": arg.user_prompt, + "p6": arg.type, + }, + ).first() + if row is None: + return None + return models.Prompt( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + ) + + def get_alert(self, *, id: Any) -> Optional[models.Alert]: + row = self._conn.execute(sqlalchemy.text(GET_ALERT), {"p1": id}).first() + if row is None: + return None + return models.Alert( + id=row[0], + prompt_id=row[1], + output_id=row[2], + code_snippet=row[3], + trigger_string=row[4], + trigger_type=row[5], + trigger_category=row[6], + timestamp=row[7], + ) + + def get_output(self, *, id: Any) -> Optional[models.Output]: + row = self._conn.execute(sqlalchemy.text(GET_OUTPUT), {"p1": id}).first() + if row is None: + return None + return models.Output( + id=row[0], + prompt_id=row[1], + timestamp=row[2], + output=row[3], + ) + + def get_outputs_by_prompt_id(self, *, prompt_id: Any) -> Iterator[models.Output]: + result = self._conn.execute(sqlalchemy.text(GET_OUTPUTS_BY_PROMPT_ID), {"p1": prompt_id}) + for row in result: + yield models.Output( + id=row[0], + prompt_id=row[1], + timestamp=row[2], + output=row[3], + ) + + def get_prompt(self, *, id: Any) -> Optional[models.Prompt]: + row = self._conn.execute(sqlalchemy.text(GET_PROMPT), {"p1": id}).first() + if row is None: + return None + return models.Prompt( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + ) + + def get_prompt_with_outputs_and_alerts( + self, *, id: Any + ) -> Iterator[GetPromptWithOutputsAndAlertsRow]: + result = self._conn.execute(sqlalchemy.text(GET_PROMPT_WITH_OUTPUTS_AND_ALERTS), {"p1": id}) + for row in result: + yield GetPromptWithOutputsAndAlertsRow( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + output_id=row[6], + output=row[7], + alert_id=row[8], + code_snippet=row[9], + trigger_string=row[10], + trigger_type=row[11], + trigger_category=row[12], + ) + + def get_settings(self) -> Optional[models.Setting]: + row = self._conn.execute(sqlalchemy.text(GET_SETTINGS)).first() + if row is None: + return None + return models.Setting( + id=row[0], + ip=row[1], + port=row[2], + llm_model=row[3], + system_prompt=row[4], + other_settings=row[5], + ) + + def list_alerts_by_prompt(self, *, prompt_id: Any) -> Iterator[models.Alert]: + result = self._conn.execute(sqlalchemy.text(LIST_ALERTS_BY_PROMPT), {"p1": prompt_id}) + for row in result: + yield models.Alert( + id=row[0], + prompt_id=row[1], + output_id=row[2], + code_snippet=row[3], + trigger_string=row[4], + trigger_type=row[5], + trigger_category=row[6], + timestamp=row[7], + ) + + def list_prompts(self, *, limit: Any, offset: Any) -> Iterator[models.Prompt]: + result = self._conn.execute(sqlalchemy.text(LIST_PROMPTS), {"p1": limit, "p2": offset}) + for row in result: + yield models.Prompt( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + ) + + def upsert_settings(self, arg: UpsertSettingsParams) -> Optional[models.Setting]: + row = self._conn.execute( + sqlalchemy.text(UPSERT_SETTINGS), + { + "p1": arg.id, + "p2": arg.ip, + "p3": arg.port, + "p4": arg.llm_model, + "p5": arg.system_prompt, + "p6": arg.other_settings, + }, + ).first() + if row is None: + return None + return models.Setting( + id=row[0], + ip=row[1], + port=row[2], + llm_model=row[3], + system_prompt=row[4], + other_settings=row[5], + ) + + +class AsyncQuerier: + def __init__(self, conn: sqlalchemy.ext.asyncio.AsyncConnection): + self._conn = conn + + async def create_alert(self, arg: CreateAlertParams) -> Optional[models.Alert]: + row = ( + await self._conn.execute( + sqlalchemy.text(CREATE_ALERT), + { + "p1": arg.id, + "p2": arg.prompt_id, + "p3": arg.output_id, + "p4": arg.code_snippet, + "p5": arg.trigger_string, + "p6": arg.trigger_type, + "p7": arg.trigger_category, + "p8": arg.timestamp, + }, + ) + ).first() + if row is None: + return None + return models.Alert( + id=row[0], + prompt_id=row[1], + output_id=row[2], + code_snippet=row[3], + trigger_string=row[4], + trigger_type=row[5], + trigger_category=row[6], + timestamp=row[7], + ) + + async def create_output( + self, *, id: Any, prompt_id: Any, timestamp: Any, output: Any + ) -> Optional[models.Output]: + row = ( + await self._conn.execute( + sqlalchemy.text(CREATE_OUTPUT), + { + "p1": id, + "p2": prompt_id, + "p3": timestamp, + "p4": output, + }, + ) + ).first() + if row is None: + return None + return models.Output( + id=row[0], + prompt_id=row[1], + timestamp=row[2], + output=row[3], + ) + + async def create_prompt(self, arg: CreatePromptParams) -> Optional[models.Prompt]: + row = ( + await self._conn.execute( + sqlalchemy.text(CREATE_PROMPT), + { + "p1": arg.id, + "p2": arg.timestamp, + "p3": arg.provider, + "p4": arg.system_prompt, + "p5": arg.user_prompt, + "p6": arg.type, + }, + ) + ).first() + if row is None: + return None + return models.Prompt( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + ) + + async def get_alert(self, *, id: Any) -> Optional[models.Alert]: + row = (await self._conn.execute(sqlalchemy.text(GET_ALERT), {"p1": id})).first() + if row is None: + return None + return models.Alert( + id=row[0], + prompt_id=row[1], + output_id=row[2], + code_snippet=row[3], + trigger_string=row[4], + trigger_type=row[5], + trigger_category=row[6], + timestamp=row[7], + ) + + async def get_output(self, *, id: Any) -> Optional[models.Output]: + row = (await self._conn.execute(sqlalchemy.text(GET_OUTPUT), {"p1": id})).first() + if row is None: + return None + return models.Output( + id=row[0], + prompt_id=row[1], + timestamp=row[2], + output=row[3], + ) + + async def get_outputs_by_prompt_id(self, *, prompt_id: Any) -> AsyncIterator[models.Output]: + result = await self._conn.stream( + sqlalchemy.text(GET_OUTPUTS_BY_PROMPT_ID), {"p1": prompt_id} + ) + async for row in result: + yield models.Output( + id=row[0], + prompt_id=row[1], + timestamp=row[2], + output=row[3], + ) + + async def get_prompt(self, *, id: Any) -> Optional[models.Prompt]: + row = (await self._conn.execute(sqlalchemy.text(GET_PROMPT), {"p1": id})).first() + if row is None: + return None + return models.Prompt( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + ) + + async def get_prompt_with_outputs_and_alerts( + self, *, id: Any + ) -> AsyncIterator[GetPromptWithOutputsAndAlertsRow]: + result = await self._conn.stream( + sqlalchemy.text(GET_PROMPT_WITH_OUTPUTS_AND_ALERTS), {"p1": id} + ) + async for row in result: + yield GetPromptWithOutputsAndAlertsRow( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + output_id=row[6], + output=row[7], + alert_id=row[8], + code_snippet=row[9], + trigger_string=row[10], + trigger_type=row[11], + trigger_category=row[12], + ) + + async def get_settings(self) -> Optional[models.Setting]: + row = (await self._conn.execute(sqlalchemy.text(GET_SETTINGS))).first() + if row is None: + return None + return models.Setting( + id=row[0], + ip=row[1], + port=row[2], + llm_model=row[3], + system_prompt=row[4], + other_settings=row[5], + ) + + async def list_alerts_by_prompt(self, *, prompt_id: Any) -> AsyncIterator[models.Alert]: + result = await self._conn.stream(sqlalchemy.text(LIST_ALERTS_BY_PROMPT), {"p1": prompt_id}) + async for row in result: + yield models.Alert( + id=row[0], + prompt_id=row[1], + output_id=row[2], + code_snippet=row[3], + trigger_string=row[4], + trigger_type=row[5], + trigger_category=row[6], + timestamp=row[7], + ) + + async def list_prompts(self, *, limit: Any, offset: Any) -> AsyncIterator[models.Prompt]: + result = await self._conn.stream(sqlalchemy.text(LIST_PROMPTS), {"p1": limit, "p2": offset}) + async for row in result: + yield models.Prompt( + id=row[0], + timestamp=row[1], + provider=row[2], + system_prompt=row[3], + user_prompt=row[4], + type=row[5], + ) + + async def upsert_settings(self, arg: UpsertSettingsParams) -> Optional[models.Setting]: + row = ( + await self._conn.execute( + sqlalchemy.text(UPSERT_SETTINGS), + { + "p1": arg.id, + "p2": arg.ip, + "p3": arg.port, + "p4": arg.llm_model, + "p5": arg.system_prompt, + "p6": arg.other_settings, + }, + ) + ).first() + if row is None: + return None + return models.Setting( + id=row[0], + ip=row[1], + port=row[2], + llm_model=row[3], + system_prompt=row[4], + other_settings=row[5], + ) diff --git a/src/codegate/providers/base.py b/src/codegate/providers/base.py index 6ffcf4dd..585c1d9a 100644 --- a/src/codegate/providers/base.py +++ b/src/codegate/providers/base.py @@ -6,6 +6,7 @@ from litellm import ModelResponse from litellm.types.llms.openai import ChatCompletionRequest +from codegate.db.connection import DbRecorder from codegate.pipeline.base import PipelineResult, SequentialPipelineProcessor from codegate.providers.completion.base import BaseCompletionHandler from codegate.providers.formatting.input_pipeline import PipelineResponseFormatter @@ -36,8 +37,8 @@ def __init__( self._output_normalizer = output_normalizer self._pipeline_processor = pipeline_processor self._fim_pipelin_processor = fim_pipeline_processor - self._pipeline_response_formatter = PipelineResponseFormatter(output_normalizer) + self.db_recorder = DbRecorder() self._setup_routes() @@ -126,7 +127,7 @@ def _is_fim_request_body(self, data: Dict) -> bool: def _is_fim_request(self, request: Request, data: Dict) -> bool: """ - Determin if the request is FIM by the URL or the data of the request. + Determine if the request is FIM by the URL or the data of the request. """ # Avoid more expensive inspection of body by just checking the URL. if self._is_fim_request_url(request): @@ -150,6 +151,10 @@ async def complete( """ normalized_request = self._input_normalizer.normalize(data) streaming = data.get("stream", False) + await self.db_recorder.record_request( + normalized_request, is_fim_request, self.provider_route_name + ) + input_pipeline_result = await self._run_input_pipeline(normalized_request, is_fim_request) if input_pipeline_result.response: return self._pipeline_response_formatter.handle_pipeline_response( diff --git a/src/codegate/server.py b/src/codegate/server.py index ec5093ba..f8a953f4 100644 --- a/src/codegate/server.py +++ b/src/codegate/server.py @@ -8,8 +8,8 @@ from codegate.pipeline.codegate_context_retriever.codegate import CodegateContextRetriever from codegate.pipeline.codegate_system_prompt.codegate import CodegateSystemPrompt from codegate.pipeline.extract_snippets.extract_snippets import CodeSnippetExtractor -from codegate.pipeline.secrets.signatures import CodegateSignatures from codegate.pipeline.secrets.secrets import CodegateSecrets +from codegate.pipeline.secrets.signatures import CodegateSignatures from codegate.pipeline.version.version import CodegateVersion from codegate.providers.anthropic.provider import AnthropicProvider from codegate.providers.llamacpp.provider import LlamaCppProvider @@ -20,6 +20,7 @@ def init_app() -> FastAPI: + """Create the FastAPI application.""" app = FastAPI( title="CodeGate", description=__description__,