Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions crates/perry-codegen/src/expr/env_clones.rs
Original file line number Diff line number Diff line change
Expand Up @@ -240,8 +240,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
"base64url" => 3,
"latin1" | "binary" => 4,
"ascii" => 5,
"utf16le" | "utf-16le" | "ucs2" | "ucs-2" => 6,
_ => bail!(
"perry-codegen: unknown Buffer encoding \"{}\": expected one of utf8, utf-8, hex, base64, base64url, ascii, latin1, binary",
"perry-codegen: unknown Buffer encoding \"{}\": expected one of utf8, utf-8, hex, base64, base64url, ascii, latin1, binary, utf16le, utf-16le, ucs2, ucs-2",
s
),
};
Expand Down Expand Up @@ -345,8 +346,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
"base64url" => 3,
"latin1" | "binary" => 4,
"ascii" => 5,
"utf16le" | "utf-16le" | "ucs2" | "ucs-2" => 6,
_ => bail!(
"perry-codegen: unknown Buffer encoding \"{}\": expected one of utf8, utf-8, hex, base64, base64url, ascii, latin1, binary",
"perry-codegen: unknown Buffer encoding \"{}\": expected one of utf8, utf-8, hex, base64, base64url, ascii, latin1, binary, utf16le, utf-16le, ucs2, ucs-2",
s
),
};
Expand Down
1 change: 1 addition & 0 deletions crates/perry-codegen/src/lower_call/property_get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ pub fn try_lower_property_get_method_call(
"base64url" => 3,
"latin1" | "binary" => 4,
"ascii" => 5,
"utf16le" | "utf-16le" | "ucs2" | "ucs-2" => 6,
_ => 0,
};
tag.to_string()
Expand Down
12 changes: 2 additions & 10 deletions crates/perry-runtime/src/buffer/cmp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,11 +224,7 @@ fn buffer_search_needle_with_encoding(
let len = (*str_ptr).byte_len as usize;
let data_ptr = (str_ptr as *const u8).add(std::mem::size_of::<StringHeader>());
let bytes = std::slice::from_raw_parts(data_ptr, len);
let decoded = match encoding {
1 => decode_hex(bytes),
2 | 3 => decode_base64(bytes),
_ => bytes.to_vec(),
};
let decoded = super::from::buffer_string_bytes_for_encoding(bytes, encoding);
Some(decoded)
};
}
Expand Down Expand Up @@ -290,11 +286,7 @@ pub extern "C" fn js_buffer_index_of_enc(
let len = (*str_ptr).byte_len as usize;
let data_ptr = (str_ptr as *const u8).add(std::mem::size_of::<StringHeader>());
let bytes = std::slice::from_raw_parts(data_ptr, len);
let decoded = match encoding {
1 => decode_hex(bytes),
2 | 3 => decode_base64(bytes),
_ => bytes.to_vec(),
};
let decoded = super::from::buffer_string_bytes_for_encoding(bytes, encoding);
return buffer_index_of_bytes(buf, &decoded, start);
}
}
Expand Down
22 changes: 18 additions & 4 deletions crates/perry-runtime/src/buffer/encode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ use super::*;
/// range (Node semantics: `start` clamped to `[0, len]`, `end` clamped to
/// `[start, len]`; defaults are `start=0, end=len`).
///
/// `encoding`: 0 = utf8 (default), 1 = hex, 2 = base64, 3 = base64url, 4 = latin1/binary, 5 = ascii.
/// `encoding`: 0 = utf8 (default), 1 = hex, 2 = base64, 3 = base64url,
/// 4 = latin1/binary, 5 = ascii, 6 = utf16le/ucs2.
#[no_mangle]
pub extern "C" fn js_buffer_to_string_range(
buf_ptr: *const BufferHeader,
Expand Down Expand Up @@ -42,13 +43,15 @@ pub extern "C" fn js_buffer_to_string_range(
3 => base64url_encode_into_string(bytes),
4 => latin1_bytes_to_string(bytes),
5 => ascii_bytes_to_string(bytes),
6 => utf16le_bytes_to_string(bytes),
_ => buf_bytes_to_utf8_string(bytes),
}
}
}

/// Convert a buffer to a string
/// encoding: 0 = utf8 (default), 1 = hex, 2 = base64, 3 = base64url, 4 = latin1/binary, 5 = ascii.
/// encoding: 0 = utf8 (default), 1 = hex, 2 = base64, 3 = base64url,
/// 4 = latin1/binary, 5 = ascii, 6 = utf16le/ucs2.
#[no_mangle]
pub extern "C" fn js_buffer_to_string(
buf_ptr: *const BufferHeader,
Expand Down Expand Up @@ -86,6 +89,7 @@ pub extern "C" fn js_buffer_to_string(
3 => base64url_encode_into_string(bytes),
4 => latin1_bytes_to_string(bytes),
5 => ascii_bytes_to_string(bytes),
6 => utf16le_bytes_to_string(bytes),
_ => {
// UTF-8 (default) — Node spec: invalid UTF-8 sequences are
// replaced with U+FFFD. Pre-fix this path passed the raw
Expand Down Expand Up @@ -120,6 +124,16 @@ fn ascii_bytes_to_string(bytes: &[u8]) -> *mut StringHeader {
js_string_from_ascii_bytes(out.as_ptr(), out.len() as u32)
}

/// Decode UTF-16LE byte pairs. Node drops an incomplete trailing byte.
fn utf16le_bytes_to_string(bytes: &[u8]) -> *mut StringHeader {
let mut units = Vec::with_capacity(bytes.len() / 2);
for pair in bytes.chunks_exact(2) {
units.push(u16::from_le_bytes([pair[0], pair[1]]));
}
let out = String::from_utf16_lossy(&units);
js_string_from_bytes(out.as_ptr(), out.len() as u32)
}

/// Build a Perry string (validated UTF-8) from a buffer's raw bytes.
/// Invalid UTF-8 sequences are replaced with U+FFFD per the WHATWG / Node
/// `Buffer.toString('utf8')` contract. Issue #609.
Expand Down Expand Up @@ -149,8 +163,8 @@ fn buf_bytes_to_utf8_string(bytes: &[u8]) -> *mut StringHeader {
/// - Otherwise fall through to `js_jsvalue_to_string` (encoding ignored,
/// matches Node behavior for non-Buffer values like numbers/objects).
///
/// `enc_tag` is the i32 produced by `js_encoding_tag_from_value` (or a
/// compile-time-folded literal): 0 = utf8, 1 = hex, 2 = base64.
/// `enc_tag` is the i32 produced by `js_encoding_tag_from_value` or a
/// compile-time-folded literal; see `js_buffer_to_string` for the tag table.
#[no_mangle]
pub extern "C" fn js_value_to_string_with_encoding(value: f64, enc_tag: i32) -> *mut StringHeader {
let bits = value.to_bits();
Expand Down
52 changes: 44 additions & 8 deletions crates/perry-runtime/src/buffer/from.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use super::*;

/// Create a Buffer from a string
/// encoding: 0 = utf8 (default), 1 = hex, 2 = base64, 3 = base64url, 4 = latin1/binary, 5 = ascii
/// encoding: 0 = utf8 (default), 1 = hex, 2 = base64, 3 = base64url,
/// 4 = latin1/binary, 5 = ascii, 6 = utf16le/ucs2.
#[no_mangle]
pub extern "C" fn js_buffer_from_string(
str_ptr: *const StringHeader,
Expand Down Expand Up @@ -34,6 +35,7 @@ fn buffer_from_str_bytes(str_bytes: &[u8], encoding: i32) -> *mut BufferHeader {
1 => hex_decode_into_buffer(str_bytes),
2 | 3 => base64_decode_into_buffer(str_bytes),
4 | 5 => latin1_string_to_buffer(str_bytes),
6 => buffer_from_vec(utf16le_string_bytes(str_bytes)),
_ => {
// UTF-8 (default)
let len = str_bytes.len();
Expand All @@ -46,6 +48,27 @@ fn buffer_from_str_bytes(str_bytes: &[u8], encoding: i32) -> *mut BufferHeader {
}
}

pub(crate) fn buffer_string_bytes_for_encoding(str_bytes: &[u8], encoding: i32) -> Vec<u8> {
match encoding {
1 => decode_hex(str_bytes),
2 | 3 => decode_base64(str_bytes),
4 | 5 => latin1_string_bytes(str_bytes),
6 => utf16le_string_bytes(str_bytes),
_ => str_bytes.to_vec(),
}
}

fn buffer_from_vec(out: Vec<u8>) -> *mut BufferHeader {
unsafe {
let buf = buffer_alloc(out.len() as u32);
(*buf).length = out.len() as u32;
if !out.is_empty() {
ptr::copy_nonoverlapping(out.as_ptr(), buffer_data_mut(buf), out.len());
}
buf
}
}

/// Encode a JS string as Node's `latin1`/`binary` Buffer input encoding.
///
/// Perry strings are stored as UTF-8, while Node's latin1 encoder writes the
Expand All @@ -54,19 +77,25 @@ fn buffer_from_str_bytes(str_bytes: &[u8], encoding: i32) -> *mut BufferHeader {
/// `ascii` uses the same input-encoding behavior in modern Node, so it
/// shares this path.
fn latin1_string_to_buffer(str_bytes: &[u8]) -> *mut BufferHeader {
buffer_from_vec(latin1_string_bytes(str_bytes))
}

fn latin1_string_bytes(str_bytes: &[u8]) -> Vec<u8> {
let decoded = String::from_utf8_lossy(str_bytes);
let mut out = Vec::with_capacity(decoded.chars().count());
for ch in decoded.chars() {
out.push((ch as u32 & 0xFF) as u8);
}
unsafe {
let buf = buffer_alloc(out.len() as u32);
(*buf).length = out.len() as u32;
if !out.is_empty() {
ptr::copy_nonoverlapping(out.as_ptr(), buffer_data_mut(buf), out.len());
}
buf
out
}

fn utf16le_string_bytes(str_bytes: &[u8]) -> Vec<u8> {
let decoded = String::from_utf8_lossy(str_bytes);
let mut out = Vec::with_capacity(decoded.len() * 2);
for unit in decoded.encode_utf16() {
out.extend_from_slice(&unit.to_le_bytes());
}
out
}

/// Map a JS string value (NaN-boxed, pointer-tagged, or raw `*const StringHeader`
Expand All @@ -78,6 +107,7 @@ fn latin1_string_to_buffer(str_bytes: &[u8]) -> *mut BufferHeader {
/// - 3 = base64url encode tag (decode uses the same permissive table)
/// - 4 = latin1 / binary
/// - 5 = ascii
/// - 6 = utf16le / utf-16le / ucs2 / ucs-2
///
/// Used by codegen for non-literal encoding arguments to `Buffer.from(str, enc)`
/// and `buf.toString(enc)` where the encoding expression cannot be statically
Expand Down Expand Up @@ -116,6 +146,12 @@ pub extern "C" fn js_encoding_tag_from_value(value: f64) -> i32 {
4
} else if eq_ascii_lower(bytes, b"ascii") {
5
} else if eq_ascii_lower(bytes, b"utf16le")
|| eq_ascii_lower(bytes, b"utf-16le")
|| eq_ascii_lower(bytes, b"ucs2")
|| eq_ascii_lower(bytes, b"ucs-2")
{
6
} else {
// utf8, utf-8, and unknown fall through to UTF-8.
// Matches the runtime's `_ =>` arm in js_buffer_from_string/js_buffer_to_string.
Expand Down