Skip to content

Commit 32f3462

Browse files
VincentAntoineVincentAntoine
committed
Group deployment files in a single folder
1 parent cc0f6af commit 32f3462

15 files changed

Lines changed: 88 additions & 26 deletions

‎Makefile‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,20 @@ dev-run-data-warehouse:
44
git clone --depth=1 --branch=main https://github.com/MTES-MCT/monitorenv.git ./forklift/tests/test_data/external/monitorenv || echo "Monitorenv repository already present - skipping git clone" && \
55
export DATA_WAREHOUSE_PASSWORD=password && \
66
export DATA_WAREHOUSE_USER=clickhouse_user && \
7-
docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml up -d --remove-orphans
7+
docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml up -d --remove-orphans
88

99
dev-run-metabase:
10-
docker compose -f ./infra/data_warehouse/docker-compose-dev-metabase.yml up -d && \
10+
docker compose -f ./infra/testing/docker-compose-dev-metabase.yml up -d && \
1111
docker cp ./infra/metabase_plugins/clickhouse.metabase-driver.jar metabase:/plugins && \
1212
docker restart metabase
1313

1414
dev-stop-data-warehouse:
15-
docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml down
15+
export DATA_WAREHOUSE_PASSWORD=password && \
16+
export DATA_WAREHOUSE_USER=clickhouse_user && \
17+
docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml down
1618

1719
dev-stop-metabase:
18-
docker compose -f ./infra/data_warehouse/docker-compose-dev-metabase.yml down
20+
docker compose -f ./infra/testing/docker-compose-dev-metabase.yml down
1921

2022
dev-erase-data-warehouse-databases:
2123
docker volume rm data_warehouse_data-warehouse-db data_warehouse_data-warehouse-logs data_warehouse_monitorfish-db-data data_warehouse_monitorenv-db-data || exit 0
@@ -31,17 +33,10 @@ docker-run-data-warehouse:
3133
git clone --depth=1 --branch=main https://github.com/MTES-MCT/monitorenv.git ./forklift/tests/test_data/external/monitorenv || echo "Monitorenv repository already present - skipping git clone" && \
3234
export DATA_WAREHOUSE_PASSWORD=password && \
3335
export DATA_WAREHOUSE_USER=clickhouse_user && \
34-
docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml up -d --remove-orphans
36+
docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml up -d --remove-orphans
3537
docker-test-forklift: docker-run-data-warehouse
3638
docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -u forklift:$(DOCKER_GROUP) --env-file forklift/.env.test forklift:$(VERSION) coverage run -m pytest --pdb tests
3739
docker-tag-forklift:
3840
docker tag forklift:$(VERSION) ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(VERSION)
3941
docker-push-forklift:
4042
docker push ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(VERSION)
41-
42-
# RUN commands
43-
run-datawarehouse:
44-
docker compose -f ./infra/data_warehouse/docker-compose.yml up -d
45-
register-forklift-flows:
46-
docker pull ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(FORKLIFT_VERSION) && \
47-
infra/forklift/register-flows.sh

infra/data_warehouse/.data-warehouse-template renamed to infra/deployment/.data-warehouse

File renamed without changes.

forklift/.env.template renamed to infra/deployment/.env.template

File renamed without changes.

‎infra/deployment/Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
run-datawarehouse:
2+
docker compose -f ./docker-compose.yml up -d
3+
register-forklift-flows:
4+
docker pull ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(FORKLIFT_VERSION) && \
5+
./register-flows.sh

‎infra/deployment/README.md‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
## Data Warehouse Deployment
2+
### 1. Purpose and Architecture
3+
4+
This Data Warehouse is meant to collect data from Monitorfish, Monitorenv and Rapportnav V2 applications and provide
5+
a centralized database for statistical analysis through data querying and visualization tools such as Metabase.
6+
7+
It is composed of 2 dockerized services :
8+
- a Clickhouse database
9+
- a set of python data flows orchestrated by Prefect, named `Forklift`
10+
11+
### 2. Deployment
12+
#### 2.1. Prerequisites
13+
14+
- Install docker : https://docs.docker.com/engine/install/debian/#install-using-the-repository
15+
- Install GNU make if not already installed :
16+
`sudo apt-get update`
17+
`sudo apt-get install make`
18+
- Install pyenv : https://github.com/pyenv/pyenv-installer
19+
- Install python : `pyenv install 3.12.3`
20+
- Copy the contents of the `infra/deployment` folder onto the host machine, in the `~` folder.
21+
- Create a virtual environment in the `~/prefect-agent` folder and install `prefect` in this virtual environment :
22+
- `cd prefect-agent`
23+
- `pyenv local 3.12.3`
24+
- `python -m venv .venv`
25+
- `source .venv/bin/activate`
26+
- `pip install prefect=1.3.0`
27+
- Run `prefect backend server` to switch Prefect into server mode (as opposed to cloud mode, the default). A file `backend.toml` should appear at `~/.prefect/backend.toml` containing `backend = "server"`.
28+
- Define and start a Prefect agent service:
29+
- Fill in the placeholders in `.prefect-agent` and `prefectdockeragent.service`.
30+
- Move `prefectdockeragent.service` into `/etc/systemd/system`.
31+
- Run `sudo systemctl enable prefectdockeragent.service` to enable the service.
32+
- Run `sudo systemctl start prefectdockeragent.service` to start the service.
33+
- Run `systemctl status prefectdockeragent.service` to check the service is running.
34+
- Rename `.env.template` as `.env` and fill in the environement variables : these are the environment variables that will be used by the flow runners, inside the forklift runner containers.
35+
36+
#### 2.2. Running the Data Warehouse
37+
38+
- Update the `DATA_WAREHOUSE_USER` and `DATA_WAREHOUSE_PASSWORD` environment variables in the `.data-warehouse` file.
39+
- Run `source .data-warehouse`.
40+
- Run `make run-datawarehouse`.
41+
42+
#### 2.3. Running or updating `Forklift`
43+
44+
- Update the `FORKLIFT_VERSION` variable in the `.data-warehouse` file.
45+
- Run `source .data-warehouse`.
46+
- Run `make register-forklift-flows`.

infra/forklift/backend.toml renamed to infra/deployment/backend.toml

File renamed without changes.

infra/data_warehouse/docker-compose.yml renamed to infra/deployment/docker-compose.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
version: '3'
21
services:
32
db:
43
image: clickhouse/clickhouse-server:23.11.4.24

‎infra/deployment/prefect-agent/.prefect-agent‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
PREFECT_SERVER_URL=

‎infra/deployment/prefect-agent/.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.12.3

‎infra/deployment/prefect-agent/prefectdockeragent.service‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[Unit]
2+
Description=Prefect Docker Agent
3+
4+
[Service]
5+
Type=simple
6+
7+
User=<USER-TO-CHANGE>
8+
ExecStart=<SCRIPT-LOCATION-TO-CHANGE>/prefectdockeragent.sh
9+
Restart=always
10+
RestartSec=30
11+
12+
[Install]
13+
WantedBy=multi-user.target

0 commit comments

Comments
 (0)