diff --git a/contrib/pyln-client/pyln/client/__init__.py b/contrib/pyln-client/pyln/client/__init__.py
index da040efbd4a1..9a506fd70567 100644
--- a/contrib/pyln-client/pyln/client/__init__.py
+++ b/contrib/pyln-client/pyln/client/__init__.py
@@ -1,6 +1,6 @@
from .lightning import LightningRpc, RpcError, Millisatoshi
from .plugin import Plugin, monkey_patch, RpcException
-
+from .gossmap import Gossmap, GossmapNode, GossmapChannel, GossmapNodeId
__version__ = "0.10.1"
@@ -12,5 +12,9 @@
"RpcException",
"Millisatoshi",
"__version__",
- "monkey_patch"
+ "monkey_patch",
+ "Gossmap",
+ "GossmapNode",
+ "GossmapChannel",
+ "GossmapNodeId",
]
diff --git a/contrib/pyln-client/pyln/client/gossmap.py b/contrib/pyln-client/pyln/client/gossmap.py
new file mode 100755
index 000000000000..71379d438804
--- /dev/null
+++ b/contrib/pyln-client/pyln/client/gossmap.py
@@ -0,0 +1,312 @@
+#! /usr/bin/python3
+
+from pyln.spec.bolt7 import (channel_announcement, channel_update,
+ node_announcement)
+from pyln.proto import ShortChannelId, PublicKey
+from typing import Any, Dict, List, Optional, Union, cast
+
+import io
+import struct
+
+# These duplicate constants in lightning/common/gossip_store.h
+GOSSIP_STORE_VERSION = 9
+GOSSIP_STORE_LEN_DELETED_BIT = 0x80000000
+GOSSIP_STORE_LEN_PUSH_BIT = 0x40000000
+GOSSIP_STORE_LEN_MASK = (~(GOSSIP_STORE_LEN_PUSH_BIT
+ | GOSSIP_STORE_LEN_DELETED_BIT))
+
+# These duplicate constants in lightning/gossipd/gossip_store_wiregen.h
+WIRE_GOSSIP_STORE_PRIVATE_CHANNEL = 4104
+WIRE_GOSSIP_STORE_PRIVATE_UPDATE = 4102
+WIRE_GOSSIP_STORE_DELETE_CHAN = 4103
+WIRE_GOSSIP_STORE_ENDED = 4105
+WIRE_GOSSIP_STORE_CHANNEL_AMOUNT = 4101
+
+
+class GossipStoreHeader(object):
+ def __init__(self, buf: bytes):
+ length, self.crc, self.timestamp = struct.unpack('>III', buf)
+ self.deleted = (length & GOSSIP_STORE_LEN_DELETED_BIT) != 0
+ self.length = (length & GOSSIP_STORE_LEN_MASK)
+
+
+class GossmapHalfchannel(object):
+ """One direction of a GossmapChannel."""
+ def __init__(self, channel: 'GossmapChannel', direction: int,
+ timestamp: int, cltv_expiry_delta: int,
+ htlc_minimum_msat: int, htlc_maximum_msat: int,
+ fee_base_msat: int, fee_proportional_millionths: int):
+
+ self.channel = channel
+ self.direction = direction
+ self.source = channel.node1 if direction == 0 else channel.node2
+ self.destination = channel.node2 if direction == 0 else channel.node1
+
+ self.timestamp: int = timestamp
+ self.cltv_expiry_delta: int = cltv_expiry_delta
+ self.htlc_minimum_msat: int = htlc_minimum_msat
+ self.htlc_maximum_msat: Optional[int] = htlc_maximum_msat
+ self.fee_base_msat: int = fee_base_msat
+ self.fee_proportional_millionths: int = fee_proportional_millionths
+
+ def __repr__(self):
+ return "GossmapHalfchannel[{}x{}]".format(str(self.channel.scid), self.direction)
+
+
+class GossmapNodeId(object):
+ def __init__(self, buf: Union[bytes, str]):
+ if isinstance(buf, str):
+ buf = bytes.fromhex(buf)
+ if len(buf) != 33 or (buf[0] != 2 and buf[0] != 3):
+ raise ValueError("{} is not a valid node_id".format(buf.hex()))
+ self.nodeid = buf
+
+ def to_pubkey(self) -> PublicKey:
+ return PublicKey(self.nodeid)
+
+ def __eq__(self, other):
+ if not isinstance(other, GossmapNodeId):
+ return False
+ return self.nodeid.__eq__(other.nodeid)
+
+ def __lt__(self, other):
+ if not isinstance(other, GossmapNodeId):
+ raise ValueError(f"Cannot compare GossmapNodeId with {type(other)}")
+ return self.nodeid.__lt__(other.nodeid) # yes, that works
+
+ def __hash__(self):
+ return self.nodeid.__hash__()
+
+ def __repr__(self):
+ return "GossmapNodeId[{}]".format(self.nodeid.hex())
+
+ @classmethod
+ def from_str(cls, s: str):
+ if s.startswith('0x'):
+ s = s[2:]
+ if len(s) != 66:
+ raise ValueError(f"{s} is not a valid hexstring of a node_id")
+ return cls(bytes.fromhex(s))
+
+
+class GossmapChannel(object):
+ """A channel: fields of channel_announcement are in .fields, optional updates are in .updates_fields, which can be None if there has been no channel update."""
+ def __init__(self,
+ fields: Dict[str, Any],
+ announce_offset: int,
+ scid,
+ node1: 'GossmapNode',
+ node2: 'GossmapNode',
+ is_private: bool):
+ self.fields = fields
+ self.announce_offset = announce_offset
+ self.is_private = is_private
+ self.scid = scid
+ self.node1 = node1
+ self.node2 = node2
+ self.updates_fields: List[Optional[Dict[str, Any]]] = [None, None]
+ self.updates_offset: List[Optional[int]] = [None, None]
+ self.satoshis = None
+ self.half_channels: List[Optional[GossmapHalfchannel]] = [None, None]
+
+ def _update_channel(self,
+ direction: int,
+ fields: Dict[str, Any],
+ off: int):
+ self.updates_fields[direction] = fields
+ self.updates_offset[direction] = off
+
+ half = GossmapHalfchannel(self, direction,
+ fields['timestamp'],
+ fields['cltv_expiry_delta'],
+ fields['htlc_minimum_msat'],
+ fields.get('htlc_maximum_msat', None),
+ fields['fee_base_msat'],
+ fields['fee_proportional_millionths'])
+ self.half_channels[direction] = half
+
+ def get_direction(self, direction: int):
+ """ returns the GossmapHalfchannel if known by channel_update """
+ if not 0 <= direction <= 1:
+ raise ValueError("direction can only be 0 or 1")
+ return self.half_channels[direction]
+
+ def __repr__(self):
+ return "GossmapChannel[{}]".format(str(self.scid))
+
+
+class GossmapNode(object):
+ """A node: fields of node_announcement are in .announce_fields, which can be None of there has been no node announcement.
+
+.channels is a list of the GossmapChannels attached to this node.
+"""
+ def __init__(self, node_id: Union[GossmapNodeId, bytes, str]):
+ if isinstance(node_id, bytes) or isinstance(node_id, str):
+ node_id = GossmapNodeId(node_id)
+ self.announce_fields: Optional[Dict[str, Any]] = None
+ self.announce_offset: Optional[int] = None
+ self.channels: List[GossmapChannel] = []
+ self.node_id = node_id
+
+ def __repr__(self):
+ return "GossmapNode[{}]".format(self.node_id.nodeid.hex())
+
+ def __eq__(self, other):
+ if not isinstance(other, GossmapNode):
+ return False
+ return self.node_id.__eq__(other.node_id)
+
+ def __lt__(self, other):
+ if not isinstance(other, GossmapNode):
+ raise ValueError(f"Cannot compare GossmapNode with {type(other)}")
+ return self.node_id.__lt__(other.node_id)
+
+
+class Gossmap(object):
+ """Class to represent the gossip map of the network"""
+ def __init__(self, store_filename: str = "gossip_store"):
+ self.store_filename = store_filename
+ self.store_file = open(store_filename, "rb")
+ self.store_buf = bytes()
+ self.nodes: Dict[GossmapNodeId, GossmapNode] = {}
+ self.channels: Dict[ShortChannelId, GossmapChannel] = {}
+ self._last_scid: Optional[str] = None
+ version = self.store_file.read(1)
+ if version[0] != GOSSIP_STORE_VERSION:
+ raise ValueError("Invalid gossip store version {}".format(int(version)))
+ self.bytes_read = 1
+ self.refresh()
+
+ def _new_channel(self,
+ fields: Dict[str, Any],
+ announce_offset: int,
+ scid: ShortChannelId,
+ node1: GossmapNode,
+ node2: GossmapNode,
+ is_private: bool):
+ c = GossmapChannel(fields, announce_offset,
+ scid, node1, node2,
+ is_private)
+ self._last_scid = scid
+ self.channels[scid] = c
+ node1.channels.append(c)
+ node2.channels.append(c)
+
+ def _del_channel(self, scid: ShortChannelId):
+ c = self.channels[scid]
+ del self.channels[scid]
+ c.node1.channels.remove(c)
+ c.node2.channels.remove(c)
+ # Beware self-channels n1-n1!
+ if len(c.node1.channels) == 0 and c.node1 != c.node2:
+ del self.nodes[c.node1.node_id]
+ if len(c.node2.channels) == 0:
+ del self.nodes[c.node2.node_id]
+
+ def _add_channel(self, rec: bytes, off: int, is_private: bool):
+ fields = channel_announcement.read(io.BytesIO(rec[2:]), {})
+ # Add nodes one the fly
+ node1_id = GossmapNodeId(fields['node_id_1'])
+ node2_id = GossmapNodeId(fields['node_id_2'])
+ if node1_id not in self.nodes:
+ self.nodes[node1_id] = GossmapNode(node1_id)
+ if node2_id not in self.nodes:
+ self.nodes[node2_id] = GossmapNode(node2_id)
+ self._new_channel(fields, off,
+ ShortChannelId.from_int(fields['short_channel_id']),
+ self.get_node(node1_id), self.get_node(node2_id),
+ is_private)
+
+ def _set_channel_amount(self, rec: bytes):
+ """ Sets channel capacity of last added channel """
+ sats, = struct.unpack(">Q", rec[2:])
+ self.channels[self._last_scid].satoshis = sats
+
+ def get_channel(self, short_channel_id: ShortChannelId):
+ """ Resolves a channel by its short channel id """
+ if isinstance(short_channel_id, str):
+ short_channel_id = ShortChannelId.from_str(short_channel_id)
+ return self.channels.get(short_channel_id)
+
+ def get_node(self, node_id: Union[GossmapNodeId, str]):
+ """ Resolves a node by its public key node_id """
+ if isinstance(node_id, str):
+ node_id = GossmapNodeId.from_str(node_id)
+ return self.nodes.get(cast(GossmapNodeId, node_id))
+
+ def _update_channel(self, rec: bytes, off: int):
+ fields = channel_update.read(io.BytesIO(rec[2:]), {})
+ direction = fields['channel_flags'] & 1
+ c = self.channels[ShortChannelId.from_int(fields['short_channel_id'])]
+ c._update_channel(direction, fields, off)
+
+ def _add_node_announcement(self, rec: bytes, off: int):
+ fields = node_announcement.read(io.BytesIO(rec[2:]), {})
+ node_id = GossmapNodeId(fields['node_id'])
+ self.nodes[node_id].announce_fields = fields
+ self.nodes[node_id].announce_offset = off
+
+ def reopen_store(self):
+ """FIXME: Implement!"""
+ assert False
+
+ def _remove_channel_by_deletemsg(self, rec: bytes):
+ scidint, = struct.unpack(">Q", rec[2:])
+ scid = ShortChannelId.from_int(scidint)
+ # It might have already been deleted when we skipped it.
+ if scid in self.channels:
+ self._del_channel(scid)
+
+ def _pull_bytes(self, length: int) -> bool:
+ """Pull bytes from file into our internal buffer"""
+ if len(self.store_buf) < length:
+ self.store_buf += self.store_file.read(length
+ - len(self.store_buf))
+ return len(self.store_buf) >= length
+
+ def _read_record(self) -> Optional[bytes]:
+ """If a whole record is not in the file, returns None.
+ If deleted, returns empty."""
+ if not self._pull_bytes(12):
+ return None
+ hdr = GossipStoreHeader(self.store_buf[:12])
+ if not self._pull_bytes(12 + hdr.length):
+ return None
+ self.bytes_read += len(self.store_buf)
+ ret = self.store_buf[12:]
+ self.store_buf = bytes()
+ if hdr.deleted:
+ ret = bytes()
+ return ret
+
+ def refresh(self):
+ """Catch up with any changes to the gossip store"""
+ while True:
+ off = self.bytes_read
+ rec = self._read_record()
+ # EOF?
+ if rec is None:
+ break
+ # Deleted?
+ if len(rec) == 0:
+ continue
+
+ rectype, = struct.unpack(">H", rec[:2])
+ if rectype == channel_announcement.number:
+ self._add_channel(rec, off, False)
+ elif rectype == WIRE_GOSSIP_STORE_PRIVATE_CHANNEL:
+ self._add_channel(rec[2 + 8 + 2:], off + 2 + 8 + 2, True)
+ elif rectype == WIRE_GOSSIP_STORE_CHANNEL_AMOUNT:
+ self._set_channel_amount(rec)
+ elif rectype == channel_update.number:
+ self._update_channel(rec, off)
+ elif rectype == WIRE_GOSSIP_STORE_PRIVATE_UPDATE:
+ self._update_channel(rec[2 + 2:], off + 2 + 2)
+ elif rectype == WIRE_GOSSIP_STORE_DELETE_CHAN:
+ self._remove_channel_by_deletemsg(rec)
+ elif rectype == node_announcement.number:
+ self._add_node_announcement(rec, off)
+ elif rectype == WIRE_GOSSIP_STORE_ENDED:
+ self.reopen_store()
+ else:
+ continue
diff --git a/contrib/pyln-client/requirements.txt b/contrib/pyln-client/requirements.txt
index 2b99830cb4b7..b661f4d0bb2a 100644
--- a/contrib/pyln-client/requirements.txt
+++ b/contrib/pyln-client/requirements.txt
@@ -1 +1,2 @@
recommonmark>=0.7.*
+pyln-bolt7
diff --git a/contrib/pyln-client/setup.py b/contrib/pyln-client/setup.py
index 8650962ea600..5c4380317a29 100644
--- a/contrib/pyln-client/setup.py
+++ b/contrib/pyln-client/setup.py
@@ -1,5 +1,4 @@
from setuptools import setup
-from pyln import client
import io
@@ -9,8 +8,14 @@
with io.open('requirements.txt', encoding='utf-8') as f:
requirements = [r for r in f.read().split('\n') if len(r)]
+# setup shouldn't try to load module, so we hack-parse __init__.py
+with io.open('pyln/client/__init__.py', encoding='utf-8') as f:
+ for line in f.read().split('\n'):
+ if line.startswith('__version__ = "'):
+ version = line.split('"')[1]
+
setup(name='pyln-client',
- version=client.__version__,
+ version=version,
description='Client library for lightningd',
long_description=long_description,
long_description_content_type='text/markdown',
diff --git a/contrib/pyln-client/tests/data/gossip_store-part1.xz b/contrib/pyln-client/tests/data/gossip_store-part1.xz
new file mode 100644
index 000000000000..73bd769f96bf
Binary files /dev/null and b/contrib/pyln-client/tests/data/gossip_store-part1.xz differ
diff --git a/contrib/pyln-client/tests/data/gossip_store-part2.xz b/contrib/pyln-client/tests/data/gossip_store-part2.xz
new file mode 100644
index 000000000000..7bf94f3c0510
Binary files /dev/null and b/contrib/pyln-client/tests/data/gossip_store-part2.xz differ
diff --git a/contrib/pyln-client/tests/data/gossip_store.simple.xz b/contrib/pyln-client/tests/data/gossip_store.simple.xz
new file mode 100644
index 000000000000..a8be99deb6ed
Binary files /dev/null and b/contrib/pyln-client/tests/data/gossip_store.simple.xz differ
diff --git a/contrib/pyln-client/tests/test_gossmap.py b/contrib/pyln-client/tests/test_gossmap.py
new file mode 100644
index 000000000000..e834003206ff
--- /dev/null
+++ b/contrib/pyln-client/tests/test_gossmap.py
@@ -0,0 +1,121 @@
+from pyln.client import Gossmap, GossmapNode, GossmapNodeId
+
+import os.path
+import lzma
+
+
+def unxz_data_tmp(src, tmp_path, dst, wmode):
+ fulldst = os.path.join(tmp_path, dst)
+ with open(fulldst, wmode) as out:
+ with lzma.open(os.path.join(os.path.dirname(__file__), "data", src), "rb")as f:
+ out.write(f.read())
+ return fulldst
+
+
+def test_gossmap(tmp_path):
+ sfile = unxz_data_tmp("gossip_store-part1.xz", tmp_path, "gossip_store", "xb")
+ g = Gossmap(sfile)
+
+ chans = len(g.channels)
+ nodes = len(g.nodes)
+
+ g.refresh()
+ assert chans == len(g.channels)
+ assert nodes == len(g.nodes)
+
+ # Now append.
+ unxz_data_tmp("gossip_store-part2.xz", tmp_path, "gossip_store", "ab")
+
+ g.refresh()
+
+ # This actually deletes a channel, which deletes a node.
+ assert g.get_channel("686386x1093x1") is None
+ assert g.get_node('029deaf9d2fba868fe0a124050f0a13e021519a12f41bea34f391fe7533fb3166d') is None
+ # The other node is untouched
+ assert g.get_node('02e0af3c70bf42343316513e54683b10c01d906c04a05dfcd9479b90d7beed9129')
+
+ # It will notice the new ones.
+ assert chans < len(g.channels)
+ assert nodes < len(g.nodes)
+
+ # Whole load at the same time gives the same results.
+ g2 = Gossmap(sfile)
+ assert set(g.channels.keys()) == set(g2.channels.keys())
+ assert set(g.nodes.keys()) == set(g2.nodes.keys())
+
+ # Check some details
+ channel2 = g.get_channel("686200x1137x0")
+ assert g.get_channel("686386x1093x1") is None
+ assert channel2.satoshis == 3000000
+
+
+def test_gossmap_halfchannel(tmp_path):
+ """ this test a simple [l1->l2] gossip store that was created by the pyln-testing framework """
+ sfile = unxz_data_tmp("gossip_store.simple.xz", tmp_path, "gossip_store", "xb")
+ g = Gossmap(sfile)
+
+ l1id = "022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59"
+ l2id = "0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518"
+
+ # check structure parsed correctly
+ assert(len(g.nodes) == 2)
+ n1 = g.get_node(l1id)
+ n2 = g.get_node(l2id)
+ assert n1
+ assert n2
+
+ chan = g.get_channel("103x1x1")
+ assert chan
+ assert chan.node1 == n1
+ assert chan.node2 == n2
+
+ half0 = chan.get_direction(0)
+ half1 = chan.get_direction(1)
+ assert half0
+ assert half1
+ assert half0.direction == 0
+ assert half1.direction == 1
+ assert half0.channel == chan
+ assert half1.channel == chan
+ assert half0.source == n1
+ assert half0.destination == n2
+ assert half1.source == n2
+ assert half1.destination == n1
+
+ # check metadata
+ assert half0.timestamp == 1631005020
+ assert half1.timestamp == 1631005020
+ assert half0.cltv_expiry_delta == 6
+ assert half1.cltv_expiry_delta == 6
+ assert half0.htlc_minimum_msat == 0
+ assert half1.htlc_minimum_msat == 0
+ assert half0.htlc_maximum_msat == 990000000
+ assert half1.htlc_maximum_msat == 990000000
+ assert half0.fee_base_msat == 1
+ assert half1.fee_base_msat == 1
+ assert half0.fee_proportional_millionths == 10
+
+
+def test_objects():
+ boltz = "026165850492521f4ac8abd9bd8088123446d126f648ca35e60f88177dc149ceb2"
+ acinq = "03864ef025fde8fb587d989186ce6a4a186895ee44a926bfc370e2c366597a3f8f"
+
+ boltz_id = GossmapNodeId(bytes.fromhex(boltz))
+ acinq_id = GossmapNodeId(bytes.fromhex(acinq))
+ assert boltz_id == GossmapNodeId(boltz)
+
+ assert boltz_id < acinq_id
+ assert acinq_id > boltz_id
+ assert boltz_id != acinq_id
+ assert acinq_id != boltz_id
+ assert not boltz_id > acinq_id
+ assert not acinq_id < boltz_id
+ assert not boltz_id == acinq_id
+ assert not acinq_id == boltz_id
+
+ boltz_node = GossmapNode(boltz_id)
+ acinq_node = GossmapNode(acinq_id)
+ assert boltz_node == GossmapNode(boltz)
+ assert boltz_node < acinq_node
+ assert acinq_node > boltz_node
+ assert boltz_node != acinq_node
diff --git a/contrib/pyln-proto/pyln/proto/__init__.py b/contrib/pyln-proto/pyln/proto/__init__.py
index a90ac9940186..ce4d2abead00 100644
--- a/contrib/pyln-proto/pyln/proto/__init__.py
+++ b/contrib/pyln-proto/pyln/proto/__init__.py
@@ -1,4 +1,5 @@
from .bech32 import bech32_decode
+from .primitives import ShortChannelId, PublicKey
from .invoice import Invoice
from .onion import OnionPayload, TlvPayload, LegacyOnionPayload
from .wire import LightningConnection, LightningServerSocket
@@ -13,4 +14,6 @@
"LegacyOnionPayload",
"TlvPayload",
"bech32_decode",
+ "ShortChannelId",
+ "PublicKey",
]
diff --git a/contrib/pyln-proto/pyln/proto/message/message.py b/contrib/pyln-proto/pyln/proto/message/message.py
index eb3d7ec556e5..127755a993f2 100644
--- a/contrib/pyln-proto/pyln/proto/message/message.py
+++ b/contrib/pyln-proto/pyln/proto/message/message.py
@@ -310,7 +310,7 @@ def write(self, io_out: BufferedIOBase, v: Dict[str, Any], otherfields: Dict[str
f.fieldtype.write(io_out, val, otherfields)
def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> Optional[Dict[str, Any]]:
- vals = {}
+ vals: Dict[str, Any] = {}
for field in self.fields:
val = field.fieldtype.read(io_in, vals)
if val is None:
diff --git a/contrib/pyln-proto/pyln/proto/primitives.py b/contrib/pyln-proto/pyln/proto/primitives.py
index 418f522121aa..d9deb6a9c248 100644
--- a/contrib/pyln-proto/pyln/proto/primitives.py
+++ b/contrib/pyln-proto/pyln/proto/primitives.py
@@ -85,6 +85,12 @@ def __eq__(self, other: object) -> bool:
and self.outnum == other.outnum
)
+ def __hash__(self):
+ return self.to_int().__hash__()
+
+ def __repr__(self):
+ return "ShortChannelId[{}]".format(str(self))
+
class Secret(object):
def __init__(self, data: bytes) -> None:
@@ -147,6 +153,15 @@ def __str__(self):
self.serializeCompressed().hex()
)
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, PublicKey):
+ return False
+
+ return self.key == other.key
+
+ def __hash__(self):
+ return self.to_bytes().__hash__()
+
def Keypair(object):
def __init__(self, priv, pub):
diff --git a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py
index 6bc370acc905..cb11f894225b 100644
--- a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py
+++ b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py
@@ -1,3 +1,3 @@
__base_version__ = "1.0"
-__post_version__ = "137"
-__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56"
+__post_version__ = "186"
+__gitversion__ = "38abac62065172c00722dca10e7d3fc3049afd72"
diff --git a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/text.py b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/text.py
index d24de3564f60..d5a2e99efae2 100644
--- a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/text.py
+++ b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/text.py
@@ -136,7 +136,8 @@
### Requirements
The sending node:
- - MUST order `tlv_record`s in a `tlv_stream` by monotonically-increasing `type`.
+ - MUST order `tlv_record`s in a `tlv_stream` by strictly-increasing `type`,
+ hence MUST not produce more than a single TLV record with the same `type`
- MUST minimally encode `type` and `length`.
- When defining custom record `type` identifiers:
- SHOULD pick random `type` identifiers to avoid collision with other
@@ -152,7 +153,8 @@
- MUST stop parsing the `tlv_stream`.
- if a `type` or `length` is not minimally encoded:
- MUST fail to parse the `tlv_stream`.
- - if decoded `type`s are not monotonically-increasing:
+ - if decoded `type`s are not strictly-increasing (including situations when
+ two or more occurrences of the same `type` are met):
- MUST fail to parse the `tlv_stream`.
- if `length` exceeds the number of bytes remaining in the message:
- MUST fail to parse the `tlv_stream`.
@@ -176,8 +178,8 @@
field, the node is forced to add parsing logic for that field in order to
determine the offset of any fields that follow.
-The monotonicity constraint ensures that all `type`s are unique and can appear
-at most once. Fields that map to complex objects, e.g. vectors, maps, or
+The strict monotonicity constraint ensures that all `type`s are unique and can
+appear at most once. Fields that map to complex objects, e.g. vectors, maps, or
structs, should do so by defining the encoding such that the object is
serialized within a single `tlv_record`. The uniqueness constraint, among other
things, enables the following optimizations:
@@ -252,7 +254,7 @@
* [`flen*byte`:`features`]
* [`init_tlvs`:`tlvs`]
-1. tlvs: `init_tlvs`
+1. `tlv_stream`: `init_tlvs`
2. types:
1. type: 1 (`networks`)
2. data:
@@ -671,7 +673,7 @@
The n1 namespace supports the following TLV types:
-1. tlvs: `n1`
+1. `tlv_stream`: `n1`
2. types:
1. type: 1 (`tlv1`)
2. data:
@@ -690,7 +692,7 @@
The n2 namespace supports the following TLV types:
-1. tlvs: `n2`
+1. `tlv_stream`: `n2`
2. types:
1. type: 0 (`tlv1`)
2. data:
diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/csv.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/csv.py
index f43d75bbee3d..b24b32454e4e 100644
--- a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/csv.py
+++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/csv.py
@@ -21,6 +21,8 @@
"msgdata,open_channel,tlvs,open_channel_tlvs,",
"tlvtype,open_channel_tlvs,upfront_shutdown_script,0",
"tlvdata,open_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...",
+ "tlvtype,open_channel_tlvs,channel_type,1",
+ "tlvdata,open_channel_tlvs,channel_type,type,byte,...",
"msgtype,accept_channel,33",
"msgdata,accept_channel,temporary_channel_id,byte,32",
"msgdata,accept_channel,dust_limit_satoshis,u64,",
@@ -39,6 +41,8 @@
"msgdata,accept_channel,tlvs,accept_channel_tlvs,",
"tlvtype,accept_channel_tlvs,upfront_shutdown_script,0",
"tlvdata,accept_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...",
+ "tlvtype,accept_channel_tlvs,channel_type,1",
+ "tlvdata,accept_channel_tlvs,channel_type,type,byte,...",
"msgtype,funding_created,34",
"msgdata,funding_created,temporary_channel_id,byte,32",
"msgdata,funding_created,funding_txid,sha256,",
@@ -58,6 +62,10 @@
"msgdata,closing_signed,channel_id,channel_id,",
"msgdata,closing_signed,fee_satoshis,u64,",
"msgdata,closing_signed,signature,signature,",
+ "msgdata,closing_signed,tlvs,closing_signed_tlvs,",
+ "tlvtype,closing_signed_tlvs,fee_range,1",
+ "tlvdata,closing_signed_tlvs,fee_range,min_fee_satoshis,u64,",
+ "tlvdata,closing_signed_tlvs,fee_range,max_fee_satoshis,u64,",
"msgtype,update_add_htlc,128",
"msgdata,update_add_htlc,channel_id,channel_id,",
"msgdata,update_add_htlc,id,u64,",
diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_csv_version.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_csv_version.py
index 0741f08250cc..2140fc8212ff 100644
--- a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_csv_version.py
+++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_csv_version.py
@@ -1 +1 @@
-__csv_version__ = "1"
+__csv_version__ = "2"
diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py
index 6bc370acc905..cb11f894225b 100644
--- a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py
+++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py
@@ -1,3 +1,3 @@
__base_version__ = "1.0"
-__post_version__ = "137"
-__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56"
+__post_version__ = "186"
+__gitversion__ = "38abac62065172c00722dca10e7d3fc3049afd72"
diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/text.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/text.py
index 7ffe333386c5..b6a9305891eb 100644
--- a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/text.py
+++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/text.py
@@ -126,11 +126,14 @@
* [`byte`:`channel_flags`]
* [`open_channel_tlvs`:`tlvs`]
-1. tlvs: `open_channel_tlvs`
+1. `tlv_stream`: `open_channel_tlvs`
2. types:
1. type: 0 (`upfront_shutdown_script`)
2. data:
* [`...*byte`:`shutdown_scriptpubkey`]
+ 1. type: 1 (`channel_type`)
+ 2. data:
+ * [`...*byte`:`type`]
The `chain_hash` value denotes the exact blockchain that the opened channel will
reside within. This is usually the genesis hash of the respective blockchain.
@@ -198,6 +201,19 @@
Since it's broadcast in the `node_announcement` message other nodes can use it to identify peers
willing to accept large channel even before exchanging the `init` message with them.
+#### Defined Channel Types
+
+Channel types are an explicit enumeration: for convenience of future
+definitions they reuse even feature bits, but they are not an
+arbitrary combination (they represent the persistent features which
+affect the channel operation).
+
+The currently defined types are:
+ - no features (no bits set)
+ - `option_static_remotekey` (bit 12)
+ - `option_anchor_outputs` and `option_static_remotekey` (bits 20 and 12)
+ - `option_anchors_zero_fee_htlc_tx` and `option_static_remotekey` (bits 22 and 12)
+
#### Requirements
The sending node:
@@ -218,6 +234,10 @@
- MAY include `upfront_shutdown_script`.
- if it includes `open_channel_tlvs`:
- MUST include `upfront_shutdown_script`.
+ - if it includes `channel_type`:
+ - MUST set it to a defined type representing the type it wants.
+ - MUST use the smallest bitmap possible to represent the channel type.
+ - SHOULD NOT set it to a type containing a feature which was not negotiated.
The sending node SHOULD:
- set `to_self_delay` sufficient to ensure the sender can irreversibly spend a commitment transaction output, in case of misbehavior by the receiver.
@@ -253,6 +273,7 @@
- the funder's amount for the initial commitment transaction is not sufficient for full [fee payment](03-transactions.md#fee-payment).
- both `to_local` and `to_remote` amounts for the initial commitment transaction are less than or equal to `channel_reserve_satoshis` (see [BOLT 3](03-transactions.md#commitment-transaction-outputs)).
- `funding_satoshis` is greater than or equal to 2^24 and the receiver does not support `option_support_large_channel`.
+ - It supports `channel_type`, `channel_type` was set, and the `type` is not suitable.
The receiving node MUST NOT:
- consider funds received, using `push_msat`, to be received until the funding transaction has reached sufficient depth.
@@ -301,11 +322,14 @@
* [`point`:`first_per_commitment_point`]
* [`accept_channel_tlvs`:`tlvs`]
-1. tlvs: `accept_channel_tlvs`
+1. `tlv_stream`: `accept_channel_tlvs`
2. types:
1. type: 0 (`upfront_shutdown_script`)
2. data:
* [`...*byte`:`shutdown_scriptpubkey`]
+ 1. type: 1 (`channel_type`)
+ 2. data:
+ * [`...*byte`:`type`]
#### Requirements
@@ -317,14 +341,20 @@
avoid double-spending of the funding transaction.
- MUST set `channel_reserve_satoshis` greater than or equal to `dust_limit_satoshis` from the `open_channel` message.
- MUST set `dust_limit_satoshis` less than or equal to `channel_reserve_satoshis` from the `open_channel` message.
+ - if it sets `channel_type`:
+ - MUST set it to the `channel_type` from `open_channel`
The receiver:
- if `minimum_depth` is unreasonably large:
- MAY reject the channel.
- if `channel_reserve_satoshis` is less than `dust_limit_satoshis` within the `open_channel` message:
- - MUST reject the channel.
+ - MUST reject the channel.
- if `channel_reserve_satoshis` from the `open_channel` message is less than `dust_limit_satoshis`:
- - MUST reject the channel.
+ - MUST reject the channel.
+ - if `channel_type` is set, and `channel_type` was set in `open_channel`, and they are not equal types:
+ - MUST reject the channel.
+
+
Other fields have the same requirements as their counterparts in `open_channel`.
### The `funding_created` Message
@@ -352,9 +382,10 @@
The sender:
- when creating the funding transaction:
- SHOULD use only BIP141 (Segregated Witness) inputs.
+ - SHOULD ensure the funding transaction confirms in the next 2016 blocks.
The recipient:
- - if `signature` is incorrect:
+ - if `signature` is incorrect OR non-compliant with LOW-S-standard rule[LOWS](https://github.com/bitcoin/bitcoin/pull/6769):
- MUST fail the channel.
#### Rationale
@@ -363,6 +394,9 @@
A transaction with all Segregated Witness inputs is not malleable, hence the funding transaction recommendation.
+The funder may use CPFP on a change output to ensure that the funding transaction confirms before 2016 blocks,
+otherwise the fundee may forget that channel.
+
### The `funding_signed` Message
This message gives the funder the signature it needs for the first
@@ -379,17 +413,25 @@
#### Requirements
Both peers:
- - if `option_static_remotekey` was negotiated:
- - `option_static_remotekey` applies to all commitment transactions
+ - if `channel_type` was present in both `open_channel` and `accept_channel`:
+ - This is the `channel_type` (they must be equal, required above)
- otherwise:
- - `option_static_remotekey` does not apply to any commitment transactions
+ - if `option_anchors_zero_fee_htlc_tx` was negotiated:
+ - the `channel_type` is `option_anchors_zero_fee_htlc_tx` and `option_static_remotekey` (bits 22 and 12)
+ - otherwise, if `option_anchor_outputs` was negotiated:
+ - the `channel_type` is `option_anchor_outputs` and `option_static_remotekey` (bits 20 and 12)
+ - otherwise, if `option_static_remotekey` was negotiated:
+ - the `channel_type` is `option_static_remotekey` (bit 12)
+ - otherwise:
+ - the `channel_type` is empty
+ - MUST use that `channel_type` for all commitment transactions.
The sender MUST set:
- `channel_id` by exclusive-OR of the `funding_txid` and the `funding_output_index` from the `funding_created` message.
- `signature` to the valid signature, using its `funding_pubkey` for the initial commitment transaction, as defined in [BOLT #3](03-transactions.md#commitment-transaction).
The recipient:
- - if `signature` is incorrect:
+ - if `signature` is incorrect OR non-compliant with LOW-S-standard rule[LOWS](https://github.com/bitcoin/bitcoin/pull/6769):
- MUST fail the channel.
- MUST NOT broadcast the funding transaction before receipt of a valid `funding_signed`.
- on receipt of a valid `funding_signed`:
@@ -397,9 +439,19 @@
#### Rationale
-We decide on `option_static_remotekey` at this point when we first have to generate the commitment
-transaction. Even if a later reconnection does not negotiate this parameter, this channel will continue to use `option_static_remotekey`; we don't support "downgrading".
-This simplifies channel state, particularly penalty transaction handling.
+We decide on `option_static_remotekey`, `option_anchor_outputs` or
+`option_anchors_zero_fee_htlc_tx` at this point when we first have to generate
+the commitment transaction. The feature bits that were communicated in the
+`init` message exchange for the current connection determine the channel
+commitment format for the total lifetime of the channel. Even if a later
+reconnection does not negotiate this parameter, this channel will continue to
+use `option_static_remotekey`, `option_anchor_outputs` or
+`option_anchors_zero_fee_htlc_tx`; we don't support "downgrading".
+
+`option_anchors_zero_fee_htlc_tx` is considered superior to
+`option_anchor_outputs`, which again is considered superior to
+`option_static_remotekey`, and the superior one is is favored if more than one
+is negotiated.
### The `funding_locked` Message
@@ -415,16 +467,15 @@
The sender MUST:
- NOT send `funding_locked` unless outpoint of given by `funding_txid` and
`funding_output_index` in the `funding_created` message pays exactly `funding_satoshis` to the scriptpubkey specified in [BOLT #3](03-transactions.md#funding-transaction-output).
- - wait until the funding transaction has reached
-`minimum_depth` before sending this message.
- - set `next_per_commitment_point` to the
-per-commitment point to be used for the following commitment
-transaction, derived as specified in
-[BOLT #3](03-transactions.md#per-commitment-secret-requirements).
+ - wait until the funding transaction has reached `minimum_depth` before
+ sending this message.
+ - set `next_per_commitment_point` to the per-commitment point to be used
+ for the following commitment transaction, derived as specified in
+ [BOLT #3](03-transactions.md#per-commitment-secret-requirements).
A non-funding node (fundee):
- - SHOULD forget the channel if it does not see the correct
-funding transaction after a reasonable timeout.
+ - SHOULD forget the channel if it does not see the correct funding
+ transaction after a timeout of 2016 blocks.
From the point of waiting for `funding_locked` onward, either node MAY
fail the channel if it does not receive a required response from the
@@ -437,6 +488,11 @@
would create a Denial of Service risk; therefore, forgetting it is recommended
(even if the promise of `push_msat` is significant).
+If the fundee forgets the channel before it was confirmed, the funder will need
+to broadcast the commitment transaction to get his funds back and open a new
+channel. To avoid this, the funder should ensure the funding transaction
+confirms in the next 2016 blocks.
+
## Channel Close
Nodes can negotiate a mutual close of the connection, which unlike a
@@ -482,7 +538,7 @@
- MUST NOT send a `shutdown`.
- MUST NOT send an `update_add_htlc` after a `shutdown`.
- if no HTLCs remain in either commitment transaction:
- - MUST NOT send any `update` message after a `shutdown`.
+ - MUST NOT send any `update` message after a `shutdown`.
- SHOULD fail to route any HTLC added after it has sent `shutdown`.
- if it sent a non-zero-length `shutdown_scriptpubkey` in `open_channel` or `accept_channel`:
- MUST send the same value in `scriptpubkey`.
@@ -491,8 +547,11 @@
1. `OP_DUP` `OP_HASH160` `20` 20-bytes `OP_EQUALVERIFY` `OP_CHECKSIG`
(pay to pubkey hash), OR
2. `OP_HASH160` `20` 20-bytes `OP_EQUAL` (pay to script hash), OR
- 3. `OP_0` `20` 20-bytes (version 0 pay to witness pubkey), OR
- 4. `OP_0` `32` 32-bytes (version 0 pay to witness script hash)
+ 3. `OP_0` `20` 20-bytes (version 0 pay to witness pubkey hash), OR
+ 4. `OP_0` `32` 32-bytes (version 0 pay to witness script hash), OR
+ 5. if (and only if) `option_shutdown_anysegwit` is negotiated:
+ * `OP_1` through `OP_16` inclusive, followed by a single push of 2 to 40 bytes
+ (witness program versions 1 through 16)
A receiving node:
- if it hasn't received a `funding_signed` (if it is a funder) or a `funding_created` (if it is a fundee):
@@ -542,11 +601,24 @@
exchange continues until both agree on the same fee or when one side fails
the channel.
+In the modern method, the funder sends its permissable fee range, and the
+non-funder has to pick a fee in this range. If the non-funder chooses the same
+value, negotiation is complete after two messages, otherwise the funder will
+reply with the same value (completing after three messages).
+
1. type: 39 (`closing_signed`)
2. data:
* [`channel_id`:`channel_id`]
* [`u64`:`fee_satoshis`]
* [`signature`:`signature`]
+ * [`closing_signed_tlvs`:`tlvs`]
+
+1. `tlv_stream`: `closing_signed_tlvs`
+2. types:
+ 1. type: 1 (`fee_range`)
+ 2. data:
+ * [`u64`:`min_fee_satoshis`]
+ * [`u64`:`max_fee_satoshis`]
#### Requirements
@@ -555,45 +627,70 @@
- SHOULD send a `closing_signed` message.
The sending node:
- - MUST set `fee_satoshis` less than or equal to the
- base fee of the final commitment transaction, as calculated in [BOLT #3](03-transactions.md#fee-calculation).
- - SHOULD set the initial `fee_satoshis` according to its
- estimate of cost of inclusion in a block.
- - MUST set `signature` to the Bitcoin signature of the close
- transaction, as specified in [BOLT #3](03-transactions.md#closing-transaction).
+ - SHOULD set the initial `fee_satoshis` according to its estimate of cost of
+ inclusion in a block.
+ - SHOULD set `fee_range` according to the minimum and maximum fees it is
+ prepared to pay for a close transaction.
+ - if it doesn't receive a `closing_signed` response after a reasonable amount of time:
+ - MUST fail the channel
+ - if it is not the funder:
+ - SHOULD set `max_fee_satoshis` to at least the `max_fee_satoshis` received
+ - SHOULD set `min_fee_satoshis` to a fairly low value
+ - MUST set `signature` to the Bitcoin signature of the close transaction,
+ as specified in [BOLT #3](03-transactions.md#closing-transaction).
The receiving node:
- if the `signature` is not valid for either variant of closing transaction
- specified in [BOLT #3](03-transactions.md#closing-transaction):
+ specified in [BOLT #3](03-transactions.md#closing-transaction) OR non-compliant with LOW-S-standard rule[LOWS](https://github.com/bitcoin/bitcoin/pull/6769):
- MUST fail the connection.
- if `fee_satoshis` is equal to its previously sent `fee_satoshis`:
- SHOULD sign and broadcast the final closing transaction.
- MAY close the connection.
- - otherwise, if `fee_satoshis` is greater than
-the base fee of the final commitment transaction as calculated in
-[BOLT #3](03-transactions.md#fee-calculation):
- - MUST fail the connection.
- - if `fee_satoshis` is not strictly
-between its last-sent `fee_satoshis` and its previously-received
-`fee_satoshis`, UNLESS it has since reconnected:
+ - if `fee_satoshis` matches its previously sent `fee_range`:
+ - SHOULD use `fee_satoshis` to sign and broadcast the final closing transaction
+ - SHOULD reply with a `closing_signed` with the same `fee_satoshis` value if it is different from its previously sent `fee_satoshis`
+ - MAY close the connection.
+ - if the message contains a `fee_range`:
+ - if there is no overlap between that and its own `fee_range`:
+ - SHOULD fail the connection
+ - MUST fail the channel if it doesn't receive a satisfying `fee_range` after a reasonable amount of time
+ - otherwise:
+ - if it is the funder:
+ - if `fee_satoshis` is not in the overlap between the sent and received `fee_range`:
+ - MUST fail the channel
+ - otherwise:
+ - MUST reply with the same `fee_satoshis`.
+ - otherwise (it is not the funder):
+ - if it has already sent a `closing_signed`:
+ - if `fee_satoshis` is not the same as the value it sent:
+ - MUST fail the channel
+ - otherwise:
+ - MUST propose a `fee_satoshis` in the overlap between received and (about-to-be) sent `fee_range`.
+ - otherwise, if `fee_satoshis` is not strictly between its last-sent `fee_satoshis`
+ and its previously-received `fee_satoshis`, UNLESS it has since reconnected:
- SHOULD fail the connection.
- - if the receiver agrees with the fee:
+ - otherwise, if the receiver agrees with the fee:
- SHOULD reply with a `closing_signed` with the same `fee_satoshis` value.
- otherwise:
- MUST propose a value "strictly between" the received `fee_satoshis`
- and its previously-sent `fee_satoshis`.
+ and its previously-sent `fee_satoshis`.
#### Rationale
-The "strictly between" requirement ensures that forward
-progress is made, even if only by a single satoshi at a time. To avoid
-keeping state and to handle the corner case, where fees have shifted
+When `fee_range` is not provided, the "strictly between" requirement ensures
+that forward progress is made, even if only by a single satoshi at a time.
+To avoid keeping state and to handle the corner case, where fees have shifted
between disconnection and reconnection, negotiation restarts on reconnection.
Note there is limited risk if the closing transaction is
delayed, but it will be broadcast very soon; so there is usually no
reason to pay a premium for rapid processing.
+Note that the non-funder is not paying the fee, so there is no reason for it
+to have a maximum feerate. It may want a minimum feerate, however, to ensure
+that the transaction propagates. It can always use CPFP later to speed up
+confirmation if necessary, so that minimum should be low.
+
## Normal Operation
Once both nodes have exchanged `funding_locked` (and optionally [`announcement_signatures`](07-routing-gossip.md#the-announcement_signatures-message)), the channel can be used to make payments via Hashed Time Locked Contracts.
@@ -741,31 +838,33 @@
Thus, the worst case is `3R+2G+2S`, assuming `R` is at least 1. Note that the
chances of three reorganizations in which the other node wins all of them is
low for `R` of 2 or more. Since high fees are used (and HTLC spends can use
-almost arbitrary fees), `S` should be small; although, given that block times are
-irregular and empty blocks still occur, `S=2` should be considered a
-minimum. Similarly, the grace period `G` can be low (1 or 2), as nodes are
-required to timeout or fulfill as soon as possible; but if `G` is too low it increases the
-risk of unnecessary channel closure due to networking delays.
+almost arbitrary fees), `S` should be small during normal operation; although,
+given that block times are irregular, empty blocks still occur, fees may vary
+greatly, and the fees cannot be bumped on HTLC transactions, `S=12` should be
+considered a minimum. `S` is also the parameter that may vary the most under
+attack, so a higher value may be desirable when non-negligible amounts are at
+risk. The grace period `G` can be low (1 or 2), as nodes are required to timeout
+or fulfill as soon as possible; but if `G` is too low it increases the risk of
+unnecessary channel closure due to networking delays.
There are four values that need be derived:
1. the `cltv_expiry_delta` for channels, `3R+2G+2S`: if in doubt, a
- `cltv_expiry_delta` of 12 is reasonable (R=2, G=1, S=2).
+ `cltv_expiry_delta` of at least 34 is reasonable (R=2, G=2, S=12).
-2. the deadline for offered HTLCs: the deadline after which the channel has to be failed
- and timed out on-chain. This is `G` blocks after the HTLC's
- `cltv_expiry`: 1 block is reasonable.
+2. the deadline for offered HTLCs: the deadline after which the channel has to
+ be failed and timed out on-chain. This is `G` blocks after the HTLC's
+ `cltv_expiry`: 1 or 2 blocks is reasonable.
-3. the deadline for received HTLCs this node has fulfilled: the deadline after which
-the channel has to be failed and the HTLC fulfilled on-chain before its
- `cltv_expiry`. See steps 4-7 above, which imply a deadline of `2R+G+S`
- blocks before `cltv_expiry`: 7 blocks is reasonable.
+3. the deadline for received HTLCs this node has fulfilled: the deadline after
+ which the channel has to be failed and the HTLC fulfilled on-chain before
+ its `cltv_expiry`. See steps 4-7 above, which imply a deadline of `2R+G+S`
+ blocks before `cltv_expiry`: 18 blocks is reasonable.
4. the minimum `cltv_expiry` accepted for terminal payments: the
worst case for the terminal node C is `2R+G+S` blocks (as, again, steps
- 1-3 above don't apply). The default in
- [BOLT #11](11-payment-encoding.md) is 9, which is slightly more
- conservative than the 7 that this calculation suggests.
+ 1-3 above don't apply). The default in [BOLT #11](11-payment-encoding.md) is
+ 18, which matches this calculation.
#### Requirements
@@ -812,6 +911,10 @@
transaction, it cannot pay the fee for either the local or remote commitment
transaction at the current `feerate_per_kw` while maintaining its channel
reserve (see [Updating Fees](#updating-fees-update_fee)).
+ - if `option_anchors` applies to this commitment transaction and the sending
+ node is the funder:
+ - MUST be able to additionally pay for `to_local_anchor` and
+ `to_remote_anchor` above its reserve.
- SHOULD NOT offer `amount_msat` if, after adding that HTLC to its commitment
transaction, its remaining balance doesn't allow it to pay the commitment
transaction fee when receiving or sending a future additional non-dust HTLC
@@ -826,8 +929,6 @@
- MUST offer `amount_msat` greater than 0.
- MUST NOT offer `amount_msat` below the receiving node's `htlc_minimum_msat`
- MUST set `cltv_expiry` less than 500000000.
- - for channels with `chain_hash` identifying the Bitcoin blockchain:
- - MUST set the four most significant bytes of `amount_msat` to 0.
- if result would be offering more than the remote's
`max_accepted_htlcs` HTLCs, in the remote commitment transaction:
- MUST NOT add an HTLC.
@@ -844,15 +945,13 @@
A receiving node:
- receiving an `amount_msat` equal to 0, OR less than its own `htlc_minimum_msat`:
- SHOULD fail the channel.
- - receiving an `amount_msat` that the sending node cannot afford at the current `feerate_per_kw` (while maintaining its channel reserve):
+ - receiving an `amount_msat` that the sending node cannot afford at the current `feerate_per_kw` (while maintaining its channel reserve and any `to_local_anchor` and `to_remote_anchor` costs):
- SHOULD fail the channel.
- if a sending node adds more than receiver `max_accepted_htlcs` HTLCs to
its local commitment transaction, OR adds more than receiver `max_htlc_value_in_flight_msat` worth of offered HTLCs to its local commitment transaction:
- SHOULD fail the channel.
- if sending node sets `cltv_expiry` to greater or equal to 500000000:
- SHOULD fail the channel.
- - for channels with `chain_hash` identifying the Bitcoin blockchain, if the four most significant bytes of `amount_msat` are not 0:
- - MUST fail the channel.
- MUST allow multiple HTLCs with the same `payment_hash`.
- if the sender did not previously acknowledge the commitment of that HTLC:
- MUST ignore a repeated `id` value after a reconnection.
@@ -885,10 +984,6 @@
`cltv_expiry` values equal to or greater than 500000000 would indicate a time in
seconds, and the protocol only supports an expiry in blocks.
-`amount_msat` is deliberately limited for this version of the
-specification; larger amounts are not necessary, nor wise, during the
-bootstrap phase of the network.
-
The node _responsible_ for paying the Bitcoin fee should maintain a "fee
spike buffer" on top of its reserve to accommodate a future fee increase.
Without this buffer, the node _responsible_ for paying the Bitcoin fee may
@@ -1011,12 +1106,12 @@
A receiving node:
- once all pending updates are applied:
- - if `signature` is not valid for its local commitment transaction:
+ - if `signature` is not valid for its local commitment transaction OR non-compliant with LOW-S-standard rule [LOWS](https://github.com/bitcoin/bitcoin/pull/6769):
- MUST fail the channel.
- if `num_htlcs` is not equal to the number of HTLC outputs in the local
commitment transaction:
- MUST fail the channel.
- - if any `htlc_signature` is not valid for the corresponding HTLC transaction:
+ - if any `htlc_signature` is not valid for the corresponding HTLC transaction OR non-compliant with LOW-S-standard rule [LOWS](https://github.com/bitcoin/bitcoin/pull/6769):
- MUST fail the channel.
- MUST respond with a `revoke_and_ack` message.
@@ -1033,7 +1128,13 @@
those HTLCs, and cannot fail the related incoming HTLCs until the
output HTLCs are fully resolved.
-Note that the `htlc_signature` implicitly enforces the time-lock mechanism in the case of offered HTLCs being timed out or received HTLCs being spent. This is done to reduce fees by creating smaller scripts compared to explicitly stating time-locks on HTLC outputs.
+Note that the `htlc_signature` implicitly enforces the time-lock mechanism in
+the case of offered HTLCs being timed out or received HTLCs being spent. This
+is done to reduce fees by creating smaller scripts compared to explicitly
+stating time-locks on HTLC outputs.
+
+The `option_anchors` allows HTLC transactions to "bring their own fees" by
+attaching other inputs and outputs, hence the modified signature flags.
### Completing the Transition to the Updated State: `revoke_and_ack`
@@ -1064,7 +1165,7 @@
transaction.
A receiving node:
- - if `per_commitment_secret` does not generate the previous `per_commitment_point`:
+ - if `per_commitment_secret` is not a valid secret key or does not generate the previous `per_commitment_point`:
- MUST fail the channel.
- if the `per_commitment_secret` was not generated by the protocol in [BOLT #3](03-transactions.md#per-commitment-secret-requirements):
- MAY fail the channel.
@@ -1119,13 +1220,18 @@
#### Rationale
-Bitcoin fees are required for unilateral closes to be effective —
-particularly since there is no general method for the broadcasting node to use
-child-pays-for-parent to increase its effective fee.
+Bitcoin fees are required for unilateral closes to be effective.
+With `option_anchors`, `feerate_per_kw` is not as critical anymore to guarantee
+confirmation as it was in the legacy commitment format, but it still needs to
+be enough to be able to enter the mempool (satisfy min relay fee and mempool
+min fee).
+
+For the legacy commitment format, there is no general method for the
+broadcasting node to use child-pays-for-parent to increase its effective fee.
Given the variance in fees, and the fact that the transaction may be
spent in the future, it's a good idea for the fee payer to keep a good
-margin (say 5x the expected fee requirement); but, due to differing methods of
+margin (say 5x the expected fee requirement) for legacy commitment txes; but, due to differing methods of
fee estimation, an exact value is not specified.
Since the fees are currently one-sided (the party which requested the
@@ -1209,7 +1315,8 @@
next `commitment_signed` it expects to receive.
- MUST set `next_revocation_number` to the commitment number of the
next `revoke_and_ack` message it expects to receive.
- - if `option_static_remotekey` applies to the commitment transaction:
+ - if `option_static_remotekey` or `option_anchors` applies to the commitment
+ transaction:
- MUST set `my_current_per_commitment_point` to a valid point.
- otherwise:
- MUST set `my_current_per_commitment_point` to its commitment point for
@@ -1243,6 +1350,10 @@
the last `revoke_and_ack` the receiving node sent, AND the receiving node
hasn't already received a `closing_signed`:
- MUST re-send the `revoke_and_ack`.
+ - if it has previously sent a `commitment_signed` that needs to be
+ retransmitted:
+ - MUST retransmit `revoke_and_ack` and `commitment_signed` in the same
+ relative order as initially transmitted.
- otherwise:
- if `next_revocation_number` is not equal to 1 greater than the
commitment number of the last `revoke_and_ack` the receiving node has sent:
@@ -1252,7 +1363,8 @@
- SHOULD fail the channel.
A receiving node:
- - if `option_static_remotekey` applies to the commitment transaction:
+ - if `option_static_remotekey` or `option_anchors` applies to the commitment
+ transaction:
- if `next_revocation_number` is greater than expected above, AND
`your_last_per_commitment_secret` is correct for that
`next_revocation_number` minus 1:
@@ -1312,7 +1424,9 @@
to be added. Requiring they be identical would effectively mean a
write to disk by the sender upon each transmission, whereas the scheme
here encourages a single persistent write to disk for each
-`commitment_signed` sent or received.
+`commitment_signed` sent or received. But if you need to retransmit both a
+`commitment_signed` and a `revoke_and_ack`, the relative order of these two
+must be preserved, otherwise it will lead to a channel closure.
A re-transmittal of `revoke_and_ack` should never be asked for after a
`closing_signed` has been received, since that would imply a shutdown has been
diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/csv.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/csv.py
index 9f3a5eeca760..fbc4f6f7e04b 100644
--- a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/csv.py
+++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/csv.py
@@ -47,6 +47,9 @@
"msgtype,final_incorrect_htlc_amount,19",
"msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64,",
"msgtype,channel_disabled,UPDATE|20",
+ "msgdata,channel_disabled,flags,u16,",
+ "msgdata,channel_disabled,len,u16,",
+ "msgdata,channel_disabled,channel_update,byte,len",
"msgtype,expiry_too_far,21",
"msgtype,invalid_onion_payload,PERM|22",
"msgdata,invalid_onion_payload,type,bigsize,",
diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_csv_version.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_csv_version.py
index 0741f08250cc..2140fc8212ff 100644
--- a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_csv_version.py
+++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_csv_version.py
@@ -1 +1 @@
-__csv_version__ = "1"
+__csv_version__ = "2"
diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py
index 6bc370acc905..cb11f894225b 100644
--- a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py
+++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py
@@ -1,3 +1,3 @@
__base_version__ = "1.0"
-__post_version__ = "137"
-__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56"
+__post_version__ = "186"
+__gitversion__ = "38abac62065172c00722dca10e7d3fc3049afd72"
diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/text.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/text.py
index 3ed313a2a902..bfbde39c9e07 100644
--- a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/text.py
+++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/text.py
@@ -78,9 +78,9 @@
algorithm.
- Elliptic curve: for all computations involving elliptic curves, the Bitcoin
curve is used, as specified in [`secp256k1`][sec2]
- - Pseudo-random stream: [`ChaCha20`][rfc7539] is used to generate a
- pseudo-random byte stream. For its generation, a fixed null-nonce
- (`0x0000000000000000`) is used, along with a key derived from a shared
+ - Pseudo-random stream: [`ChaCha20`][rfc8439] is used to generate a
+ pseudo-random byte stream. For its generation, a fixed 96-bit null-nonce
+ (`0x000000000000000000000000`) is used, along with a key derived from a shared
secret and with a `0x00`-byte stream of the desired output size as the
message.
- The terms _origin node_ and _final node_ refer to the initial packet sender
@@ -128,7 +128,7 @@
path, so that each hop may only recover the address and HMAC of the next hop.
The pseudo-random byte stream is generated by encrypting (using `ChaCha20`) a
`0x00`-byte stream, of the required length, which is initialized with a key
-derived from the shared secret and a zero-nonce (`0x00000000000000`).
+derived from the shared secret and a 96-bit zero-nonce (`0x000000000000000000000000`).
The use of a fixed nonce is safe, since the keys are never reused.
@@ -247,8 +247,9 @@
### `tlv_payload` format
This is a more flexible format, which avoids the redundant `short_channel_id` field for the final node.
+It is formatted according to the Type-Length-Value format defined in [BOLT #1](01-messaging.md#type-length-value-format).
-1. tlvs: `tlv_payload`
+1. `tlv_stream`: `tlv_payload`
2. types:
1. type: 2 (`amt_to_forward`)
2. data:
@@ -989,7 +990,7 @@
1. type: UPDATE|20 (`channel_disabled`)
2. data:
- * [`u16`: `flags`]
+ * [`u16`:`flags`]
* [`u16`:`len`]
* [`len*byte`:`channel_update`]
@@ -1200,7 +1201,7 @@
[RFC2104]: https://tools.ietf.org/html/rfc2104
[fips198]: http://csrc.nist.gov/publications/fips/fips198-1/FIPS-198-1_final.pdf
[sec2]: http://www.secg.org/sec2-v2.pdf
-[rfc7539]: https://tools.ietf.org/html/rfc7539
+[rfc8439]: https://tools.ietf.org/html/rfc8439
# Authors
diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_csv_version.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_csv_version.py
index 0741f08250cc..2140fc8212ff 100644
--- a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_csv_version.py
+++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_csv_version.py
@@ -1 +1 @@
-__csv_version__ = "1"
+__csv_version__ = "2"
diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py
index 6bc370acc905..cb11f894225b 100644
--- a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py
+++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py
@@ -1,3 +1,3 @@
__base_version__ = "1.0"
-__post_version__ = "137"
-__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56"
+__post_version__ = "186"
+__gitversion__ = "38abac62065172c00722dca10e7d3fc3049afd72"
diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/text.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/text.py
index 94466844704d..96cbb4979ba8 100644
--- a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/text.py
+++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/text.py
@@ -463,8 +463,6 @@
- MUST set this to less than or equal to the channel capacity.
- MUST set this to less than or equal to `max_htlc_value_in_flight_msat`
it received from the peer.
- - for channels with `chain_hash` identifying the Bitcoin blockchain:
- - MUST set this to less than 2^32.
- otherwise:
- MUST set the `option_channel_htlc_max` bit of `message_flags` to 0.
- MUST set bits in `channel_flags` and `message_flags `that are not assigned a meaning to 0.
@@ -599,7 +597,7 @@
* [`len*byte`:`encoded_short_ids`]
* [`query_short_channel_ids_tlvs`:`tlvs`]
-1. tlvs: `query_short_channel_ids_tlvs`
+1. `tlv_stream`: `query_short_channel_ids_tlvs`
2. types:
1. type: 1 (`query_flags`)
2. data:
@@ -704,7 +702,7 @@
* [`u32`:`number_of_blocks`]
* [`query_channel_range_tlvs`:`tlvs`]
-1. tlvs: `query_channel_range_tlvs`
+1. `tlv_stream`: `query_channel_range_tlvs`
2. types:
1. type: 1 (`query_option`)
2. data:
@@ -724,12 +722,12 @@
* [`chain_hash`:`chain_hash`]
* [`u32`:`first_blocknum`]
* [`u32`:`number_of_blocks`]
- * [`byte`:`full_information`]
+ * [`byte`:`sync_complete`]
* [`u16`:`len`]
* [`len*byte`:`encoded_short_ids`]
* [`reply_channel_range_tlvs`:`tlvs`]
-1. tlvs: `reply_channel_range_tlvs`
+1. `tlv_stream`: `reply_channel_range_tlvs`
2. types:
1. type: 1 (`timestamps_tlv`)
2. data:
@@ -763,7 +761,7 @@
The checksum of a `channel_update` is the CRC32C checksum as specified in [RFC3720](https://tools.ietf.org/html/rfc3720#appendix-B.4) of this `channel_update` without its `signature` and `timestamp` fields.
-This allows to query for channels within specific blocks.
+This allows querying for channels within specific blocks.
#### Requirements
@@ -782,23 +780,21 @@
- MUST set with `chain_hash` equal to that of `query_channel_range`,
- MUST limit `number_of_blocks` to the maximum number of blocks whose
results could fit in `encoded_short_ids`
- - if does not maintain up-to-date channel information for `chain_hash`:
- - MUST set `full_information` to 0.
- - otherwise:
- - SHOULD set `full_information` to 1.
+ - MAY split block contents across multiple `reply_channel_range`.
- the first `reply_channel_range` message:
- MUST set `first_blocknum` less than or equal to the `first_blocknum` in `query_channel_range`
- MUST set `first_blocknum` plus `number_of_blocks` greater than `first_blocknum` in `query_channel_range`.
- successive `reply_channel_range` message:
- - MUST set `first_blocknum` to the previous `first_blocknum` plus `number_of_blocks`.
+ - MUST have `first_blocknum` equal or greater than the previous `first_blocknum`.
+ - MUST set `sync_complete` to `false` if this is not the final `reply_channel_range`.
- the final `reply_channel_range` message:
- MUST have `first_blocknum` plus `number_of_blocks` equal or greater than the `query_channel_range` `first_blocknum` plus `number_of_blocks`.
+ - MUST set `sync_complete` to `true`.
If the incoming message includes `query_option`, the receiver MAY append additional information to its reply:
- if bit 0 in `query_option_flags` is set, the receiver MAY append a `timestamps_tlv` that contains `channel_update` timestamps for all `short_chanel_id`s in `encoded_short_ids`
- if bit 1 in `query_option_flags` is set, the receiver MAY append a `checksums_tlv` that contains `channel_update` checksums for all `short_chanel_id`s in `encoded_short_ids`
-
#### Rationale
A single response might be too large for a single packet, so multiple replies
@@ -982,7 +978,7 @@
#### Requirements
A node:
- - if a channel's latest `channel_update`s `timestamp` is older than two weeks
+ - if a channel's oldest `channel_update`s `timestamp` is older than two weeks
(1209600 seconds):
- MAY prune the channel.
- MAY ignore the channel.
@@ -1000,6 +996,11 @@
from the rest of the network; however, they would remain in the local network
view would be forwarded to other peers indefinitely.
+The oldest `channel_update` is used to prune the channel since both sides need
+to be active in order for the channel to be usable. Doing so prunes channels
+even if one side continues to send fresh `channel_update`s but the other node
+has disappeared.
+
## Recommendations for Routing
When calculating a route for an HTLC, both the `cltv_expiry_delta` and the fee