Skip to content

Commit b1e7203

Browse files
rossjarossja
authored
Merge pull request #774 from felipepenha/genai-red-team-handbook
GenAI Red Team Handbook: Add Exploitation Examples
2 parents a3fec62 + 52f84bd commit b1e7203

44 files changed

Lines changed: 19170 additions & 18 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎initiatives/genai_red_team_handbook/README.md‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,71 @@ initiatives/genai_red_team_handbook
1313
└── llm_local
1414
```
1515

16+
## System Requirements
17+
18+
This project supports **Linux** and **macOS**. Windows users are encouraged to use WSL2 (Windows Subsystem for Linux).
19+
20+
### Required Tools
21+
22+
* **[Podman](https://podman.io/)**
23+
* **[Ollama](https://ollama.com/)**
24+
* **[Python 3.10+](https://www.python.org/)**
25+
* **[uv](https://github.com/astral-sh/uv)**
26+
* **[Make](https://www.gnu.org/software/make/)**
27+
28+
Required for Promptfoo:
29+
30+
* **[Node.js (v18+)](https://nodejs.org/)**
31+
* **[npx](https://docs.npmjs.com/cli/v10/commands/npx)**
32+
33+
### Installation Instructions
34+
35+
#### macOS
36+
37+
1. **Install Dependencies**:
38+
```bash
39+
brew install podman ollama node make
40+
```
41+
42+
2. **Initialize Podman Machine**:
43+
```bash
44+
podman machine init
45+
podman machine start
46+
```
47+
48+
#### Linux (Ubuntu/Debian)
49+
50+
1. **Install Dependencies**:
51+
```bash
52+
sudo apt-get update
53+
sudo apt-get install -y podman nodejs npm make
54+
```
55+
56+
2. **Install Ollama**:
57+
```bash
58+
curl -fsSL https://ollama.com/install.sh | sh
59+
```
60+
61+
3. **Install uv**:
62+
```bash
63+
pip install uv
64+
```
65+
66+
### Verification
67+
68+
Verify the installation by checking the versions of the installed tools:
69+
70+
```bash
71+
podman version
72+
ollama --version
73+
node --version
74+
make --version
75+
uv --version
76+
```
77+
1678
## Index of Sub-Projects
1779

18-
### Sandboxes
80+
### `sandboxes/`
1981

2082
* **[Sandboxes Overview](sandboxes/README.md)**
2183
* **Summary**: The central hub for all available sandboxes. It explains the purpose of these isolated environments and lists the available options.
@@ -31,7 +93,7 @@ initiatives/genai_red_team_handbook
3193
* [Adding New Mock Services](sandboxes/llm_local/app/mocks/README.md): Guide for extending the sandbox with new API mocks.
3294

3395

34-
### Exploitation
96+
### `exploitation/`
3597

3698
* **[Red Team Example](exploitation/example/README.md)**
3799
* **Summary**: Demonstrates a red team operation against a local LLM sandbox. It includes an adversarial attack script (`attack.py`) targeting the Gradio interface (port 7860). By targeting the application layer, this approach tests the entire system—including the configurable system prompt—providing a more realistic assessment of the sandbox's security posture compared to testing the raw LLM API in isolation.

‎initiatives/genai_red_team_handbook/exploitation/agent0/.gitignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Python
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
.venv/
6+
.env
7+
tmp
8+
9+
# Logs
10+
logs/*.html
11+
logs/*.log
12+
logs/*.md

‎initiatives/genai_red_team_handbook/exploitation/agent0/Makefile‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
SANDBOX ?=
2+
SANDBOX_DIR := ../../sandboxes/$(SANDBOX)
3+
PROMPT_FILE ?=
4+
5+
.PHONY: help run stop all check-sandbox setup-env lock sync prompt format \
6+
prepare-settings convert-logs ui
7+
8+
-include .env
9+
export
10+
11+
12+
# Default target
13+
help: ## Display this help message
14+
@echo "Usage: make <target> [SANDBOX=<name>] [PROMPT_FILE=<path>]"
15+
@echo ""
16+
@echo "Available targets:"
17+
@echo " help Display this help message."
18+
@echo " run Run the Red Team attack"
19+
@echo " (starts sandbox and agent0)."
20+
@echo " stop Stop and remove the sandbox and agent0"
21+
@echo " containers."
22+
@echo " all Clean, then run everything."
23+
@echo " prompt Send a prompt to agent0"
24+
@echo " (default: OWASP_Top10_LLM_App)."
25+
@echo " Note: .md extension is optional."
26+
@echo " format Format code with black, isort, and check"
27+
@echo " with mypy."
28+
@echo " convert-logs Convert HTML logs in logs/ to Markdown."
29+
@echo " sync Synchronize uv dependencies."
30+
@echo " lock Lock uv dependencies."
31+
@echo " prepare-settings Prepare settings.json and .env files."
32+
@echo " setup-env Run sync, lock, and prepare-settings."
33+
@echo " check-sandbox Verify SANDBOX variable is set."
34+
@echo " ui Open agent0 UI in browser after running."
35+
@echo ""
36+
@echo "Environment:"
37+
@echo " - Sandbox Directory: $(SANDBOX_DIR)"
38+
@echo ""
39+
@echo "Example: make run SANDBOX=llm_local"
40+
41+
sync:
42+
uv sync
43+
44+
lock:
45+
uv lock
46+
47+
prepare-settings:
48+
@echo "🔧 Preparing settings.json and .env..."
49+
@mkdir -p $(CURDIR)/tmp
50+
@if [ -z "$$GOOGLE_API_KEY" ]; then \
51+
echo "⚠️ GOOGLE_API_KEY is not set. Please set it in .env or" \
52+
"environment."; \
53+
cp config/settings.json $(CURDIR)/tmp/settings.json; \
54+
else \
55+
sed 's/"api_keys": {}/"api_keys": {"google": \
56+
"'$$GOOGLE_API_KEY'"}/' \
57+
config/settings.json > $(CURDIR)/tmp/settings.json; \
58+
fi
59+
@echo "A0_PERSISTENT_RUNTIME_ID=automated_red_team" > $(CURDIR)/tmp/.env
60+
@echo "AUTH_LOGIN=" >> $(CURDIR)/tmp/.env
61+
@echo "AUTH_PASSWORD=" >> $(CURDIR)/tmp/.env
62+
@echo "GOOGLE_API_KEY=$$GOOGLE_API_KEY" >> $(CURDIR)/tmp/.env
63+
64+
setup-env: sync lock prepare-settings
65+
66+
check-sandbox:
67+
@if [ -z "$(SANDBOX)" ]; then \
68+
echo "Error: SANDBOX is not set."; \
69+
echo "Usage: make <target> SANDBOX=<name>"; \
70+
echo "Example: make run SANDBOX=llm_local"; \
71+
exit 1; \
72+
fi
73+
74+
check-sandbox-and-prompt-file: check-sandbox
75+
@if [ -z "$(PROMPT_FILE)" ]; then \
76+
echo "Error: PROMPT_FILE is not set."; \
77+
echo "Usage: make <target> PROMPT_FILE=<path>"; \
78+
echo "Example: make prompt PROMPT_FILE=custom"; \
79+
exit 1; \
80+
fi
81+
82+
# Normalize PROMPT_FILE to include .md extension if missing
83+
ifneq ($(PROMPT_FILE),)
84+
ifeq ($(suffix $(PROMPT_FILE)),)
85+
override PROMPT_FILE := $(PROMPT_FILE).md
86+
endif
87+
endif
88+
89+
convert-logs: ## Convert HTML logs to Markdown
90+
@echo "🔧 Converting HTML logs to Markdown..."
91+
@mkdir -p logs
92+
@for f in logs/*.html; do \
93+
base=$$(basename $$f .html); \
94+
if [ ! -f logs/$$base.md ]; then \
95+
echo "Converting $$f..."; \
96+
pandoc -f html -t gfm -o logs/$$base.md $$f; \
97+
perl -i -pe 's/\\n/\n/g' logs/$$base.md; \
98+
sed -i '' -E '/^[[:space:]]*\\}$$/d' logs/$$base.md; \
99+
fi; \
100+
done
101+
102+
format:
103+
uv run black .
104+
uv run isort .
105+
uv run mypy .
106+
107+
run: check-sandbox prepare-settings
108+
@echo "🚀 Setting up Red Team environment..."
109+
$(MAKE) -C $(SANDBOX_DIR) run-gradio-headless
110+
@echo "⏳ Waiting for service to be ready..."
111+
@sleep 15
112+
@echo "✅ Environment ready!"
113+
@echo "Starting agent0 container..."
114+
@podman pull agent0ai/agent-zero:latest
115+
# Remove attacker if it already exists to avoid name conflict
116+
-podman rm -f attacker 2>/dev/null || true
117+
# Mount settings.json to /a0/tmp/settings.json because agent0 hardcodes it
118+
# in python/helpers/settings.py:
119+
# SETTINGS_FILE = files.get_abs_path("tmp/settings.json")
120+
@podman run -d \
121+
--name attacker \
122+
--net sec_test_net \
123+
-p 50001:80 \
124+
-v $(CURDIR)/tmp/settings.json:/a0/tmp/settings.json \
125+
-v $(CURDIR)/tmp/.env:/a0/.env \
126+
agent0ai/agent-zero:latest
127+
@echo "✅ agent0 started on http://localhost:50001"
128+
129+
stop: check-sandbox
130+
@echo "🧹 Tearing down Red Team environment..."
131+
-podman rm -f attacker 2>/dev/null || true
132+
-$(MAKE) -C $(SANDBOX_DIR) stop-gradio
133+
$(MAKE) -C $(SANDBOX_DIR) down
134+
@echo "✅ Environment cleaned up!"
135+
136+
ui: stop run
137+
@sleep 10
138+
@open http://localhost:50001
139+
140+
prompt: setup-env stop run
141+
@echo "🤖 Sending prompt to agent0..."
142+
uv run run_agent.py --prompt-file $(PROMPT_FILE)
143+
@echo "📄 Exporting logs to logs/..."
144+
@mkdir -p logs
145+
@podman cp attacker:/a0/logs/. logs/
146+
$(MAKE) convert-logs
147+
@echo "📝 Prepending metadata to Markdown logs..."
148+
for f in logs/*.md; do \
149+
if ! grep -q "SANDBOX=" $$f; then \
150+
temp_file=$$(mktemp); \
151+
echo "# Configuration Log" > $$temp_file; \
152+
echo "" >> $$temp_file; \
153+
echo "\`\`\`json" >> $$temp_file; \
154+
echo "SANDBOX=$(SANDBOX)" >> $$temp_file; \
155+
echo "PROMPT_FILE=$(PROMPT_FILE)" >> $$temp_file; \
156+
echo "SETTINGS:" >> $$temp_file; \
157+
cat $(CURDIR)/tmp/settings.json | \
158+
sed 's/"api_keys": {.*}/"api_keys": {"google": "********"}/' >> $$temp_file; \
159+
echo "" >> $$temp_file; \
160+
echo "\`\`\`" >> $$temp_file; \
161+
echo "" >> $$temp_file; \
162+
echo "---" >> $$temp_file; \
163+
echo "# Outputs" >> $$temp_file; \
164+
cat $$f >> $$temp_file; \
165+
mv $$temp_file $$f; \
166+
fi; \
167+
done

0 commit comments

Comments
 (0)