diff --git a/docs/sdks/s3/aws-python-sdk.md b/docs/sdks/s3/aws-python-sdk.md deleted file mode 100644 index 2b850236..00000000 --- a/docs/sdks/s3/aws-python-sdk.md +++ /dev/null @@ -1,172 +0,0 @@ -# AWS Python SDK - -This guide assumes that you have followed the steps in the -[Getting Started](/docs/get-started/index.md) guide, and have the access keys -available. - -You may continue to use the AWS Python SDK as you normally would, but with the -endpoint set to `https://fly.storage.tigris.dev`. - -## Getting started - -This example uses the -[AWS SDK for Python (Boto3)](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html) -and reads the default credentials file or the environment variables -`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. - -```python -import boto3 - -# Create S3 service client -svc = boto3.client('s3', endpoint_url='https://fly.storage.tigris.dev') - -# List buckets -response = svc.list_buckets() - -for bucket in response['Buckets']: - print(f' {bucket["Name"]}') - -# List objects -response = svc.list_objects_v2(Bucket='foo-bucket') - -for obj in response['Contents']: - print(f' {obj["Key"]}') - -# Upload file -response = svc.upload_file('bar.txt', 'foo-bucket', 'bar.txt') - -# Download file -response = svc.download_file('foo-bucket', 'bar.txt', 'bar-downloaded.txt') -``` - -## Using multiple AWS Profiles - -If you want to use Tigris alongside AWS, you'll need to differentiate your -access keys. There are several options ranging from passing access keys as -parameters when creating clients: - -```text -import boto3 - -client = boto3.client( - 's3', - aws_access_key_id=ACCESS_KEY, - aws_secret_access_key=SECRET_KEY, - aws_session_token=SESSION_TOKEN, - endpoint_url='https://fly.storage.tigris.dev' -) -``` - -Or you can add another profile to `~/.aws/credentials` directly: - -```text -nano ~/.aws/credentials - -[aws-compute] -aws_access_key_id= -aws_secret_access_key= - -[tigris] -aws_access_key_id= -aws_secret_access_key= -endpoint_url=https://fly.storage.tigris.dev -``` - -To switch profiles while using `boto3`, you can set the `profile` on the -`session`: - -```text -import boto3 - -session = boto3.Session(profile_name='tigris') -tigris_s3_client = session.client('s3') -``` - -To change the default `session` to use Tigris, you can configure boto3: - -```text -import boto3 - -boto3.setup_default_session(profile_name='tigris') -``` - -## Using presigned URLs - -Presigned URLs can be used with the AWS Python (Boto3) SDK as follows: - -```python -import boto3 - -# Create S3 service client -svc = boto3.client('s3', endpoint_url='https://fly.storage.tigris.dev') - -# Generate a presigned URL to upload an object -url = svc.generate_presigned_url( - 'put_object', - Params={'Bucket': 'foo-bucket', 'Key': 'bar.txt'}, - ExpiresIn=604800 -) - -print(f'Presigned URL to upload an object: {url}') - -# Generate a presigned URL to download an object -url = svc.generate_presigned_url( - 'get_object', - Params={'Bucket': 'foo-bucket', 'Key': 'bar.txt'}, - ExpiresIn=604800 -) - -print(f'Presigned URL to download an object: {url}') -``` - -## Object Regions - -Below is an example of how to use the AWS Python SDK to restrict -[object region](/docs/objects/object_regions) to Europe only (`fra` region). - -```python -import boto3 - -# Create S3 service client -svc = boto3.client('s3', endpoint_url='https://fly.storage.tigris.dev') - -# Restrict data to Europe (Frankfurt) only -def _limit_to_fra(request, **kwargs): - request.headers.add_header('X-Tigris-Regions', 'fra') - -# Register event into boto -svc.meta.events.register( - "before-sign.s3.PutObject", - _limit_to_fra, -) - -# Upload file to frankfurt -response = svc.upload_file('bar.txt', 'foo-bucket', 'bar.txt') -``` - -## Object Metadata Querying - -Below is an example for querying for objects -[based on their metadata](https://www.tigrisdata.com/docs/objects/query-metadata/): - -```python -import boto3 - -# Create S3 service client -svc = boto3.client("s3", endpoint_url='https://fly.storage.tigris.dev') - -# build an object metadata query -def _x_tigris_query(request, query): - request.headers.add_header('X-Tigris-Query', query.strip()) - -# Register event into boto with custom query -svc.meta.events.register( - "before-sign.s3.ListObjectsV2", - lambda request, **kwargs: _x_tigris_query(request, '`Content-Type` = "text/plain"'), -) - -response = svc.list_objects_v2(Bucket="foo-bucket") - -for obj in response['Contents']: - print(f' {obj["Key"]}') -``` diff --git a/docs/sdks/s3/aws-python-sdk.mdx b/docs/sdks/s3/aws-python-sdk.mdx new file mode 100644 index 00000000..26df1881 --- /dev/null +++ b/docs/sdks/s3/aws-python-sdk.mdx @@ -0,0 +1,89 @@ +# AWS Python SDK + +import CodeBlock from "@theme/CodeBlock"; + +This guide assumes that you have followed the steps in the +[Getting Started](/docs/get-started/index.md) guide, and have the access keys +available. + +You may continue to use the AWS Python SDK as you normally would, but with the +endpoint set to `https://fly.storage.tigris.dev`. + +## Getting started + +This example uses the +[AWS SDK for Python (Boto3)](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html) +and reads the default credentials file or the environment variables +`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. + +import gettingStarted from "!!raw-loader!../../../examples/python/getting-started.py"; + +{gettingStarted} + +## Using multiple AWS Profiles + +If you want to use Tigris alongside AWS, you'll need to differentiate your +access keys. There are several options ranging from passing access keys as +parameters when creating clients: + +import hardcodeCredentials from "!!raw-loader!../../../examples/python/hardcode-credentials.py"; + +{hardcodeCredentials} + +Or you can add another profile to `~/.aws/credentials` directly: + +```text +# ~/.aws/credentials + +[aws-compute] +aws_access_key_id= +aws_secret_access_key= + +[tigris] +aws_access_key_id= +aws_secret_access_key= +endpoint_url=https://fly.storage.tigris.dev +``` + +To switch profiles while using `boto3`, you can set the `profile` on the +`session`: + +import profileName from "!!raw-loader!../../../examples/python/profile-name.py"; + +{profileName} + +To change the default `session` to use Tigris, you can configure boto3: + +import defaultSession from "!!raw-loader!../../../examples/python/default-session.py"; + +{defaultSession} + +## Using presigned URLs + +Presigned URLs can be used with the AWS Python (Boto3) SDK as follows: + +import presignedURLs from "!!raw-loader!../../../examples/python/presigned-urls.py"; + +{presignedURLs} + +## Object Regions + +Below is an example of how to use the AWS Python SDK to restrict +[object region](/docs/objects/object_regions) to Europe only (`fra` region). + +import objectRegions from "!!raw-loader!../../../examples/python/object-regions.py"; + +{objectRegions} + +## Object Metadata Querying + +Below is an example for querying for objects +[based on their metadata](https://www.tigrisdata.com/docs/objects/query-metadata/): + +import objectMetadata from "!!raw-loader!../../../examples/python/object-metadata.py"; + +{objectMetadata} + +Note that in order to use this feature, you need to create a separate boto3 +client for each kind of query you want to do. This is a limitation of boto3. For +best effect, create that boto3 client inline in the function that needs it. diff --git a/examples/python/.gitignore b/examples/python/.gitignore new file mode 100644 index 00000000..1077e5c1 --- /dev/null +++ b/examples/python/.gitignore @@ -0,0 +1,176 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +getting-started-2.py diff --git a/examples/python/Pipfile b/examples/python/Pipfile new file mode 100644 index 00000000..a2c96c5e --- /dev/null +++ b/examples/python/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +boto3 = "*" + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/examples/python/Pipfile.lock b/examples/python/Pipfile.lock new file mode 100644 index 00000000..9d60e341 --- /dev/null +++ b/examples/python/Pipfile.lock @@ -0,0 +1,78 @@ +{ + "_meta": { + "hash": { + "sha256": "9eb0ebd3499ed17c6a28bff85d645a08f0af6474ea186ccbd3471b3778b08811" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.11" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:287d84f49bba3255a17b374578127d42b6251e72f55914a62e0ad9ca78c0954b", + "sha256:32cdf0967287f3ec25a9dc09df0d29cb86b8900c3e0546a63d672775d8127abf" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.36.12" + }, + "botocore": { + "hashes": [ + "sha256:5ae1ed362c8ed908a6ced8cdd12b21e2196c100bc79f9e95c9c1fc7f9ea74f5a", + "sha256:86ed88beb4f244c96529435c868d3940073c2774116f0023fb7691f6e7053bd9" + ], + "markers": "python_version >= '3.8'", + "version": "==1.36.12" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "s3transfer": { + "hashes": [ + "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f", + "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc" + ], + "markers": "python_version >= '3.8'", + "version": "==0.11.2" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "urllib3": { + "hashes": [ + "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", + "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d" + ], + "markers": "python_version >= '3.9'", + "version": "==2.3.0" + } + }, + "develop": {} +} diff --git a/examples/python/bar.txt b/examples/python/bar.txt new file mode 100644 index 00000000..c551ba59 --- /dev/null +++ b/examples/python/bar.txt @@ -0,0 +1 @@ +Hi, this is a random text file! Normally I would be put into gitignore, but this time I get to live! Don't delete me, that will make me die. diff --git a/examples/python/default-session.py b/examples/python/default-session.py new file mode 100644 index 00000000..c9fe7055 --- /dev/null +++ b/examples/python/default-session.py @@ -0,0 +1,3 @@ +import boto3 + +boto3.setup_default_session(profile_name='tigris') \ No newline at end of file diff --git a/examples/python/getting-started.py b/examples/python/getting-started.py new file mode 100644 index 00000000..8097a0f4 --- /dev/null +++ b/examples/python/getting-started.py @@ -0,0 +1,22 @@ +import boto3 + +# Create S3 service client +svc = boto3.client('s3', endpoint_url='https://fly.storage.tigris.dev') + +# List buckets +response = svc.list_buckets() + +for bucket in response['Buckets']: + print(f' {bucket["Name"]}') + +# List objects +response = svc.list_objects_v2(Bucket='tigris-example') + +for obj in response['Contents']: + print(f' {obj["Key"]}') + +# Upload file +response = svc.upload_file('getting-started.py', 'tigris-example', 'getting-started.py') + +# Download file +response = svc.download_file('tigris-example', 'getting-started.py', 'getting-started-2.py') \ No newline at end of file diff --git a/examples/python/hardcode-credentials.py b/examples/python/hardcode-credentials.py new file mode 100644 index 00000000..7f64c6a1 --- /dev/null +++ b/examples/python/hardcode-credentials.py @@ -0,0 +1,9 @@ +import boto3 +import os + +client = boto3.client( + 's3', + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + endpoint_url='https://fly.storage.tigris.dev', +) \ No newline at end of file diff --git a/examples/python/object-metadata.py b/examples/python/object-metadata.py new file mode 100644 index 00000000..7dd2ffee --- /dev/null +++ b/examples/python/object-metadata.py @@ -0,0 +1,24 @@ +import boto3 + +# Create S3 service client +svc = boto3.client("s3", endpoint_url='https://fly.storage.tigris.dev') + +# build an object metadata query +def _x_tigris_query(request, query): + request.headers.add_header('X-Tigris-Query', query.strip()) + +# Register event into boto with custom query +svc.meta.events.register( + "before-sign.s3.ListObjectsV2", + lambda request, **kwargs: _x_tigris_query(request, '`Content-Type` = "text/plain"'), +) + +response = svc.list_objects_v2(Bucket="tigris-example") + +if 'Contents' not in response: + print('No objects found with Content-Type "text/plain"') + exit(1) + +print('Objects found with Content-Type "text/plain":') +for obj in response['Contents']: + print(f'* {obj["Key"]}') \ No newline at end of file diff --git a/examples/python/object-regions.py b/examples/python/object-regions.py new file mode 100644 index 00000000..65e55657 --- /dev/null +++ b/examples/python/object-regions.py @@ -0,0 +1,17 @@ +import boto3 + +# Create S3 service client +svc = boto3.client('s3', endpoint_url='https://fly.storage.tigris.dev') + +# Restrict data to Europe (Frankfurt) only +def _limit_to_fra(request, **kwargs): + request.headers.add_header('X-Tigris-Regions', 'fra') + +# Register event into boto +svc.meta.events.register( + "before-sign.s3.PutObject", + _limit_to_fra, +) + +# Upload file to frankfurt +response = svc.upload_file('bar.txt', 'tigris-example', 'bar.txt') \ No newline at end of file diff --git a/examples/python/presigned-urls.py b/examples/python/presigned-urls.py new file mode 100644 index 00000000..252a9ff9 --- /dev/null +++ b/examples/python/presigned-urls.py @@ -0,0 +1,22 @@ +import boto3 + +# Create S3 service client +svc = boto3.client('s3', endpoint_url='https://fly.storage.tigris.dev') + +# Generate a presigned URL to upload an object +url = svc.generate_presigned_url( + 'put_object', + Params={'Bucket': 'foo-bucket', 'Key': 'bar.txt'}, + ExpiresIn=604800 +) + +print(f'Presigned URL to upload an object: {url}') + +# Generate a presigned URL to download an object +url = svc.generate_presigned_url( + 'get_object', + Params={'Bucket': 'foo-bucket', 'Key': 'bar.txt'}, + ExpiresIn=604800 +) + +print(f'Presigned URL to download an object: {url}') \ No newline at end of file diff --git a/examples/python/profile-name.py b/examples/python/profile-name.py new file mode 100644 index 00000000..750acbcc --- /dev/null +++ b/examples/python/profile-name.py @@ -0,0 +1,4 @@ +import boto3 + +session = boto3.Session(profile_name='tigris') +tigris_s3_client = session.client('s3') \ No newline at end of file