Skip to content

Commit f887c6f

Browse files
odesenfansodesenfans
authored
Feature: use the Aleph.im P2P service (#328)
Problem: P2P communication with the JS P2P daemon is sometimes unstable. Solution: develop a new P2P service for Aleph.im nodes to replace this daemon. * Rewrote all the calls to the P2P daemon to use the new service instead * Adapted Docker Compose files * Decommissioned the P2P protocol implementation as streams are not supported by the new service for now * The key generation mechanism now writes the private key file in PKCS8 DER format, for compatibility with the P2P service * Added a migration script to migrate the private key of existing nodes to the new format
1 parent 1bc15fe commit f887c6f

35 files changed

+292
-657
lines changed

‎.github/workflows/pyaleph-ci.yml

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,6 @@ jobs:
2828
with:
2929
# Fetch the whole history for all tags and branches (required for aleph.__version__)
3030
fetch-depth: 0
31-
# Install nodejs for jsp2pd
32-
- uses: actions/setup-node@v2
33-
with:
34-
node-version: '16'
3531
- name: Set up Python 3.8
3632
id: setup-python
3733
uses: actions/setup-python@v2
@@ -45,8 +41,6 @@ jobs:
4541
with:
4642
path: ${{ steps.pip-cache.outputs.dir }}
4743
key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('setup.cfg') }}
48-
- name: Install jsp2pd
49-
run: npm install --global [email protected]
5044
- name: Install Python dependencies
5145
run: |
5246
pip install .[testing]

‎deployment/docker-build/config.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ p2p:
4343
http_port: 4024
4444
port: 4025
4545
control_port: 4030
46-
listen_port: 4031
4746
reconnect_delay: 60
4847

4948
sentry:

‎deployment/docker-build/docker-compose.yml

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,48 @@ volumes:
88
pyaleph-mongodb:
99

1010
services:
11-
p2pd:
11+
12+
rabbitmq:
1213
restart: always
13-
image: alephim/jsp2pd:0.10.2-1.0.0
14+
image: rabbitmq:3.10.7-management
1415
networks:
1516
- pyaleph
1617
environment:
17-
PRIVATE_KEY_FILE: "/etc/jsp2pd/keys/serialized-node-secret.key"
18-
LISTEN_MADDR: "/ip4/0.0.0.0/tcp/4030"
19-
HOST_MADDRS: "/ip4/0.0.0.0/tcp/4025"
20-
PUBSUB: "true"
21-
PUBSUB_ROUTER: "floodsub"
18+
RABBITMQ_DEFAULT_USER: guest
19+
RABBITMQ_DEFAULT_PASS: guest
2220
ports:
23-
- "4025:4025"
24-
- "4030:4030"
21+
- "127.0.0.1:5672:5672"
22+
- "127.0.0.1:15672:15672"
23+
24+
p2p-service:
25+
restart: always
26+
image: alephim/p2p-service:0.1.1
27+
networks:
28+
- pyaleph
2529
volumes:
26-
- ../../keys:/etc/jsp2pd/keys
30+
- ../../config.yml:/etc/p2p-service/config.yml
31+
- ../../keys/node-secret.pkcs8.der:/etc/p2p-service/node-secret.pkcs8.der
32+
depends_on:
33+
- rabbitmq
34+
environment:
35+
RUST_LOG: info
36+
ports:
37+
- "4025:4025"
38+
- "127.0.0.1:4030:4030"
39+
command:
40+
- "--config"
41+
- "/etc/p2p-service/config.yml"
42+
- "--private-key-file"
43+
- "/etc/p2p-service/node-secret.pkcs8.der"
2744

2845
ipfs:
2946
restart: always
3047
image: ipfs/kubo:v0.15.0
3148
ports:
3249
- "4001:4001"
3350
- "4001:4001/udp"
34-
- "5001:5001"
35-
- "8080:8080"
51+
- "127.0.0.1:5001:5001"
52+
- "127.0.0.1:8080:8080"
3653
volumes:
3754
- "pyaleph-ipfs:/data/ipfs"
3855
environment:
@@ -50,7 +67,7 @@ services:
5067
networks:
5168
- pyaleph
5269
ports:
53-
- "27017:27017"
70+
- "127.0.0.1:27017:27017"
5471

5572
networks:
5673
pyaleph:

‎deployment/docker-build/jsp2pd/build_jsp2pd.sh

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎deployment/docker-build/jsp2pd/jsp2pd.dockerfile

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎deployment/migrations/scripts/0003-convert-key-to-pkcs8-der.py

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
"""
2+
This migration converts the PEM private key file to PKCS8 DER for compatibility
3+
with the new Aleph.im P2P service. The Rust implementation of libp2p can only load
4+
RSA keys in that format.
5+
"""
6+
7+
8+
import logging
9+
import os
10+
from pathlib import Path
11+
from typing import Optional
12+
13+
import yaml
14+
from Crypto.PublicKey import RSA
15+
from p2pclient.libp2p_stubs.crypto.rsa import RSAPrivateKey
16+
17+
from aleph.exceptions import InvalidKeyDirException
18+
19+
LOGGER = logging.getLogger(os.path.basename(__file__))
20+
21+
22+
PKCS8_DER_KEY_FILE = "node-secret.pkcs8.der"
23+
24+
25+
def convert_pem_key_file_to_pkcs8_der(
26+
pem_key_file: Path, pkcs8_der_key_file: Path
27+
) -> None:
28+
with pem_key_file.open() as pem:
29+
private_key = RSAPrivateKey(RSA.import_key(pem.read()))
30+
31+
with pkcs8_der_key_file.open("wb") as der:
32+
der.write(private_key.impl.export_key(format="DER", pkcs=8))
33+
34+
35+
def get_key_from_config(config_file: Path) -> Optional[str]:
36+
"""
37+
In previous versions of the CCN, it was possible to set the key value directly
38+
in the config file. This function tries to find it in the config or returns None.
39+
40+
:param config_file: Path to the CCN configuration file.
41+
:return: The private key used to identify the node on the P2P network, or None
42+
if the key is not provided in the config file.
43+
"""
44+
with open(config_file) as f:
45+
config = yaml.safe_load(f)
46+
47+
try:
48+
return config["p2p"]["key"]
49+
except KeyError:
50+
return None
51+
52+
53+
def upgrade(**kwargs):
54+
key_dir = Path(kwargs["key_dir"])
55+
pem_key_file = key_dir / "node-secret.key"
56+
57+
# Nothing to do if the PKCS8 DER key file already exists
58+
pkcs8_der_key_file = key_dir / PKCS8_DER_KEY_FILE
59+
if pkcs8_der_key_file.is_file():
60+
LOGGER.info(
61+
"Key file %s already exists, nothing to do",
62+
pkcs8_der_key_file,
63+
)
64+
return
65+
66+
if not key_dir.is_dir():
67+
raise InvalidKeyDirException(
68+
f"The specified key directory ('{key_dir}') is not a directory."
69+
)
70+
71+
LOGGER.info("Converting the private key file to PKCS8 DER format...")
72+
convert_pem_key_file_to_pkcs8_der(pem_key_file, pkcs8_der_key_file)
73+
LOGGER.info("Successfully created %s.", pkcs8_der_key_file)
74+
75+
76+
def downgrade(**kwargs):
77+
# Nothing to do, the key file is still present in the key directory
78+
pass

‎deployment/samples/docker-compose/docker-compose.yml

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,28 +18,45 @@ services:
1818
depends_on:
1919
- mongodb
2020
- ipfs
21-
- p2pd
21+
- p2p-service
2222
networks:
2323
- pyaleph
2424
logging:
2525
options:
2626
max-size: 50m
2727

28-
p2pd:
28+
p2p-service:
2929
restart: always
30-
image: alephim/jsp2pd:0.10.2-1.0.0
30+
image: alephim/p2p-service:0.1.1
3131
networks:
3232
- pyaleph
33+
volumes:
34+
- ./config.yml:/etc/p2p-service/config.yml
35+
- ./keys/node-secret.pkcs8.der:/etc/p2p-service/node-secret.pkcs8.der
36+
depends_on:
37+
- rabbitmq
3338
environment:
34-
PRIVATE_KEY_FILE: "/etc/jsp2pd/keys/serialized-node-secret.key"
35-
LISTEN_MADDR: "/ip4/0.0.0.0/tcp/4030"
36-
HOST_MADDRS: "/ip4/0.0.0.0/tcp/4025"
37-
PUBSUB: "true"
38-
PUBSUB_ROUTER: "floodsub"
39+
RUST_LOG: info
3940
ports:
4041
- "4025:4025"
41-
volumes:
42-
- ./keys:/etc/jsp2pd/keys
42+
- "127.0.0.1:4030:4030"
43+
command:
44+
- "--config"
45+
- "/etc/p2p-service/config.yml"
46+
- "--private-key-file"
47+
- "/etc/p2p-service/node-secret.pkcs8.der"
48+
49+
rabbitmq:
50+
restart: always
51+
image: rabbitmq:3.10.7-management
52+
networks:
53+
- pyaleph
54+
environment:
55+
RABBITMQ_DEFAULT_USER: guest
56+
RABBITMQ_DEFAULT_PASS: guest
57+
ports:
58+
- "127.0.0.1:5672:5672"
59+
- "127.0.0.1:15672:15672"
4360

4461
ipfs:
4562
restart: always

‎deployment/samples/docker-compose/sample-config.yml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,15 @@ aleph:
3535
queue_topic: ALEPH-TEST
3636

3737
p2p:
38-
daemon_host: p2pd
38+
daemon_host: p2p-service
3939
http_port: 4024
4040
port: 4025
4141
control_port: 4030
42-
listen_port: 4031
4342
reconnect_delay: 60
4443

44+
rabbitmq:
45+
host: rabbitmq
46+
port: 5672
47+
4548
sentry:
4649
dsn: ""

‎deployment/samples/docker-monitoring/docker-compose.yml

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,28 +20,45 @@ services:
2020
depends_on:
2121
- mongodb
2222
- ipfs
23-
- p2pd
23+
- p2p-service
2424
networks:
2525
- pyaleph
2626
logging:
2727
options:
2828
max-size: 50m
2929

30-
p2pd:
30+
p2p-service:
3131
restart: always
32-
image: alephim/jsp2pd:0.10.2-1.0.0
32+
image: alephim/p2p-service:0.1.1
3333
networks:
3434
- pyaleph
35+
volumes:
36+
- ./config.yml:/etc/p2p-service/config.yml
37+
- ./keys/node-secret.pkcs8.der:/etc/p2p-service/node-secret.pkcs8.der
38+
depends_on:
39+
- rabbitmq
3540
environment:
36-
PRIVATE_KEY_FILE: "/etc/jsp2pd/keys/serialized-node-secret.key"
37-
LISTEN_MADDR: "/ip4/0.0.0.0/tcp/4030"
38-
HOST_MADDRS: "/ip4/0.0.0.0/tcp/4025"
39-
PUBSUB: "true"
40-
PUBSUB_ROUTER: "floodsub"
41+
RUST_LOG: info
4142
ports:
4243
- "4025:4025"
43-
volumes:
44-
- ./keys:/etc/jsp2pd/keys
44+
- "127.0.0.1:4030:4030"
45+
command:
46+
- "--config"
47+
- "/etc/p2p-service/config.yml"
48+
- "--private-key-file"
49+
- "/etc/p2p-service/node-secret.pkcs8.der"
50+
51+
rabbitmq:
52+
restart: always
53+
image: rabbitmq:3.10.7-management
54+
networks:
55+
- pyaleph
56+
environment:
57+
RABBITMQ_DEFAULT_USER: guest
58+
RABBITMQ_DEFAULT_PASS: guest
59+
ports:
60+
- "127.0.0.1:5672:5672"
61+
- "127.0.0.1:15672:15672"
4562

4663
ipfs:
4764
restart: always

‎deployment/samples/docker-monitoring/sample-config.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,10 @@ aleph:
3535
queue_topic: ALEPH-TEST
3636

3737
p2p:
38-
daemon_host: p2pd
38+
daemon_host: p2p-service
3939
http_port: 4024
4040
port: 4025
4141
control_port: 4030
42-
listen_port: 4031
4342
reconnect_delay: 60
4443

4544
sentry:

‎deployment/scripts/run_aleph_ccn.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,11 @@ function wait_for_it()
4141
DB_URI=$(get_config mongodb.uri | sed "s-mongodb://--")
4242
IPFS_HOST=$(get_config ipfs.host)
4343
IPFS_PORT=$(get_config ipfs.port)
44+
RABBITMQ_HOST=$(get_config rabbitmq.host)
45+
RABBITMQ_PORT=$(get_config rabbitmq.port)
4446

4547
wait_for_it "${DB_URI}"
4648
wait_for_it -h "${IPFS_HOST}" -p "${IPFS_PORT}"
49+
wait_for_it -h "${RABBITMQ_HOST}" -p "${RABBITMQ_PORT}"
4750

4851
exec pyaleph "${PYALEPH_ARGS[@]}"

‎docs/guides/private_net.rst

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ Example of config file at this point (I disabled IPFS but you can leave it enabl
5555
p2p:
5656
host: 0.0.0.0
5757
control_port: 4030
58-
listen_port: 4031
5958
http_port: 4024
6059
port: 4025
6160
peers: []

‎setup.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ install_requires =
4141
aioipfs@git+https://github.com/aleph-im/aioipfs.git@76d5624661e879a13b70f3ea87dc9c9604c7eda7
4242
aleph-client==0.4.6
4343
aleph-message==0.2.2
44+
aleph-p2p-client@git+https://github.com/aleph-im/p2p-service-client-python@f9fc6057be5bf1712180129c831116a740441434
4445
aleph-pytezos@git+https://github.com/aleph-im/aleph-pytezos.git@97fe92ffa6e21ef5ec17ef4fa16c86022b30044c
4546
coincurve==15.0.1
4647
configmanager==1.35.1

0 commit comments

Comments
 (0)