diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 2abc1ed..ce8ade4 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -56,10 +56,10 @@ jobs:
           echo "$HOME/bin" >> $GITHUB_PATH
 
       - name: Run caddy validate
-        run: caddy validate
+        run: caddy validate --config caddy/Caddyfile
 
       - name: Run caddy fmt
-        run: caddy fmt --overwrite
+        run: caddy fmt --overwrite caddy/Caddyfile
 
       - name: Run reviewdog
         uses: reviewdog/action-suggester@v1
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 86afdd5..a2bad21 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -19,9 +19,9 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -r dev-requirements.txt
+          pip install -r webhook/dev-requirements.txt
           git config --global user.name "GitHub CI Bot"
           git config --global user.email "noone@example.com"
 
       - name: Run tests
-        run: pytest
+        run: pytest webhook
diff --git a/Makefile b/Makefile
index 6100061..11db7b5 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ serve:
 	mkdir -p sites
 	$(RUNTIME) run --rm -it \
 	    -v $$PWD/sites:/srv:Z \
-	    -v $$PWD/Caddyfile:/etc/caddy/Caddyfile:ro,Z \
+	    -v $$PWD/caddy/Caddyfile:/etc/caddy/Caddyfile:ro,Z \
 	    -e SITE_DIR=/srv \
 	    -p 2015:2015 \
 	    $(CADDY_IMAGE):$(CADDY_VERSION) \
@@ -14,6 +14,6 @@ serve:
 
 fmt:
 	$(RUNTIME) run --rm -it \
-	    -v $$PWD/Caddyfile:/etc/caddy/Caddyfile:Z \
+	    -v $$PWD/caddy/Caddyfile:/etc/caddy/Caddyfile:Z \
 	    $(CADDY_IMAGE):$(CADDY_VERSION) \
 	    caddy fmt --overwrite /etc/caddy/Caddyfile
diff --git a/Caddyfile b/caddy/Caddyfile
similarity index 100%
rename from Caddyfile
rename to caddy/Caddyfile
diff --git a/dev-requirements.txt b/webhook/dev-requirements.txt
similarity index 100%
rename from dev-requirements.txt
rename to webhook/dev-requirements.txt
diff --git a/requirements.txt b/webhook/requirements.txt
similarity index 100%
rename from requirements.txt
rename to webhook/requirements.txt
diff --git a/test_webhook.py b/webhook/test_webhook.py
similarity index 100%
rename from test_webhook.py
rename to webhook/test_webhook.py
diff --git a/webhook.py b/webhook/webhook.py
similarity index 100%
rename from webhook.py
rename to webhook/webhook.py
diff --git a/webhook.service b/webhook/webhook.service
similarity index 100%
rename from webhook.service
rename to webhook/webhook.service