cut: support multibyte characters in non-UTF-8 locales

src/uu/cut/Cargo.toml

@@ -20,7 +20,7 @@ doctest = false
 
 [dependencies]
 clap = { workspace = true }
-uucore = { workspace = true, features = ["ranges"] }
+uucore = { workspace = true, features = ["ranges", "i18n-charmap"] }
 memchr = { workspace = true }
 bstr = { workspace = true }
 fluent = { workspace = true }

src/uu/cut/locales/en-US.ftl

@@ -101,6 +101,7 @@ cut-help-complement = invert the filter - instead of displaying only the filtere
 cut-help-only-delimited = in field mode, only print lines which contain the delimiter
 cut-help-zero-terminated = instead of filtering columns based on line, filter columns based on \\0 (NULL character)
 cut-help-output-delimiter = in field mode, replace the delimiter in output lines with this option's argument
+cut-help-no-split-multibyte = in byte mode, do not split multibyte characters
 
 # Error messages
 cut-error-is-directory = Is a directory

src/uu/cut/locales/fr-FR.ftl

@@ -101,6 +101,7 @@ cut-help-complement = inverser le filtre - au lieu d'afficher seulement les colo
 cut-help-only-delimited = en mode champ, afficher seulement les lignes qui contiennent le délimiteur
 cut-help-zero-terminated = au lieu de filtrer les colonnes basées sur la ligne, filtrer les colonnes basées sur \\0 (caractère NULL)
 cut-help-output-delimiter = en mode champ, remplacer le délimiteur dans les lignes de sortie avec l'argument de cette option
+cut-help-no-split-multibyte = en mode octet, ne pas couper les caractères multioctets
 
 # Messages d'erreur
 cut-error-is-directory = Est un répertoire

src/uu/cut/src/cut.rs

@@ -13,6 +13,7 @@ use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, std
 use std::path::Path;
 use uucore::display::Quotable;
 use uucore::error::{FromIo, UResult, USimpleError, set_exit_code};
+use uucore::i18n::charmap::mb_char_len;
 use uucore::line_ending::LineEnding;
 use uucore::os_str_as_bytes;
 
@@ -29,6 +30,8 @@ struct Options<'a> {
     out_delimiter: Option<&'a [u8]>,
     line_ending: LineEnding,
     field_opts: Option<FieldOptions<'a>>,
+    /// `-n`: in byte mode, do not split multi-byte characters.
+    byte_no_split: bool,
 }
 
 enum Delimiter<'a> {
@@ -104,6 +107,128 @@ fn cut_bytes<R: Read, W: Write>(
     Ok(())
 }
 
+/// Fill `spans` with the byte spans `[start, end)` of `line`'s characters, using
+/// the current locale's encoding. Invalid/incomplete sequences count as one
+/// byte. `spans` is cleared first; its capacity is reused across calls.
+fn char_spans_into(line: &[u8], spans: &mut Vec<(usize, usize)>) {
+    spans.clear();
+    let mut i = 0;
+    while i < line.len() {
+        let len = mb_char_len(&line[i..]).clamp(1, line.len() - i);
+        spans.push((i, i + len));
+        i += len;
+    }
+}
+
+/// Character mode (`-c`): ranges index whole (possibly multi-byte) characters.
+///
+/// Ranges are sorted and disjoint, so a single forward cursor over the line's
+/// characters serves every range in order. Characters are decoded on the fly
+/// (no per-line allocation) and only as far as the ranges actually reach, so a
+/// bounded selection like `-c 5-30` never decodes the tail of a long line.
+fn cut_characters<R: Read, W: Write>(
+    reader: R,
+    out: &mut W,
+    ranges: &[Range],
+    opts: &Options,
+) -> UResult<()> {
+    let newline_char = opts.line_ending.into();
+    let mut buf_in = BufReader::new(reader);
+    let out_delim = opts.out_delimiter.unwrap_or(b"\t");
+
+    let result = buf_in.for_byte_record(newline_char, |line| {
+        // `pos` is the byte offset of the next character; `char_no` is the count
+        // of characters already consumed, so the next one is 1-based `char_no + 1`.
+        let mut pos = 0;
+        let mut char_no = 0;
+        let mut print_delim = false;
+        for &Range { low, high } in ranges {
+            // Skip forward to the first character of this range.
+            while char_no + 1 < low && pos < line.len() {
+                pos += mb_char_len(&line[pos..]).clamp(1, line.len() - pos);
+                char_no += 1;
+            }
+            // `low` lies past the end of the line: no later range can match either.
+            if char_no + 1 < low {
+                break;
+            }
+            let start = pos;
+            // Consume the characters selected by this range (clamped to line end).
+            while char_no < high && pos < line.len() {
+                pos += mb_char_len(&line[pos..]).clamp(1, line.len() - pos);
+                char_no += 1;
+            }
+            if pos == start {
+                break;
+            }
+            if print_delim {
+                out.write_all(out_delim)?;
+            } else if opts.out_delimiter.is_some() {
+                print_delim = true;
+            }
+            out.write_all(&line[start..pos])?;
+        }
+        out.write_all(&[newline_char])?;
+        Ok(true)
+    });
+
+    if let Err(e) = result {
+        return Err(USimpleError::new(1, e.to_string()));
+    }
+
+    Ok(())
+}
+
+/// Byte mode with `-n`: ranges index bytes, but a multi-byte character is
+/// emitted in full when (and only when) the range includes its last byte.
+fn cut_bytes_no_split<R: Read, W: Write>(
+    reader: R,
+    out: &mut W,
+    ranges: &[Range],
+    opts: &Options,
+) -> UResult<()> {
+    let newline_char = opts.line_ending.into();
+    let mut buf_in = BufReader::new(reader);
+    let out_delim = opts.out_delimiter.unwrap_or(b"\t");
+    let mut spans: Vec<(usize, usize)> = Vec::new();
+
+    let result = buf_in.for_byte_record(newline_char, |line| {
+        char_spans_into(line, &mut spans);
+        let mut print_delim = false;
+        for &Range { low, high } in ranges {
+            if low > line.len() {
+                break;
+            }
+            let high = high.min(line.len());
+            // A character's last byte is at 1-based position `end` (exclusive 0-based end).
+            // Emit the output delimiter lazily, only once this range has actually
+            // selected a character, so a range that matches nothing adds no delimiter.
+            let mut range_emitted = false;
+            for &(start, end) in &spans {
+                if end >= low && end <= high {
+                    if !range_emitted {
+                        if print_delim {
+                            out.write_all(out_delim)?;
+                        } else if opts.out_delimiter.is_some() {
+                            print_delim = true;
+                        }
+                        range_emitted = true;
+                    }
+                    out.write_all(&line[start..end])?;
+                }
+            }
+        }
+        out.write_all(&[newline_char])?;
+        Ok(true)
+    });
+
+    if let Err(e) = result {
+        return Err(USimpleError::new(1, e.to_string()));
+    }
+
+    Ok(())
+}
+
 /// Output delimiter is explicitly specified
 fn cut_fields_explicit_out_delim<R: Read, W: Write, M: Matcher>(
     reader: R,
@@ -458,8 +583,10 @@ where
             }
 
             show_if_err!(match mode {
+                Mode::Bytes(ranges, opts) if opts.byte_no_split =>
+                    cut_bytes_no_split(stdin(), &mut out, ranges, opts),
                 Mode::Bytes(ranges, opts) => cut_bytes(stdin(), &mut out, ranges, opts),
-                Mode::Characters(ranges, opts) => cut_bytes(stdin(), &mut out, ranges, opts),
+                Mode::Characters(ranges, opts) => cut_characters(stdin(), &mut out, ranges, opts),
                 Mode::Fields(ranges, opts) => cut_fields(stdin(), &mut out, ranges, opts),
             });
 
@@ -482,8 +609,12 @@ where
                     .map_err_context(|| filename.maybe_quote().to_string())
                     .and_then(|file| {
                         match &mode {
-                            Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => {
-                                cut_bytes(file, &mut out, ranges, opts)
+                            Mode::Bytes(ranges, opts) if opts.byte_no_split => {
+                                cut_bytes_no_split(file, &mut out, ranges, opts)
+                            }
+                            Mode::Bytes(ranges, opts) => cut_bytes(file, &mut out, ranges, opts),
+                            Mode::Characters(ranges, opts) => {
+                                cut_characters(file, &mut out, ranges, opts)
                             }
                             Mode::Fields(ranges, opts) => cut_fields(file, &mut out, ranges, opts),
                         }
@@ -514,12 +645,16 @@ fn get_delimiters(matches: &ArgMatches) -> UResult<(Delimiter<'_>, Option<&[u8]>
             if os_string.is_empty() {
                 Delimiter::Slice(b"\0")
             } else {
-                // For delimiter `-d` option value - allow both UTF-8 (possibly multi-byte) characters
-                // and Non UTF-8 (and not ASCII) single byte "characters", like `b"\xAD"` to align with GNU behavior
+                // For delimiter `-d` option value - allow a single character: a UTF-8
+                // character in a UTF-8 locale, or a single (possibly multi-byte)
+                // character of the current locale's encoding, e.g. a 2-byte GB18030
+                // character or any single byte like `b"\xAD"`, to align with GNU.
                 let bytes = os_str_as_bytes(os_string)?;
-                if os_string.to_str().is_some_and(|s| s.chars().count() > 1)
-                    || os_string.to_str().is_none() && bytes.len() > 1
-                {
+                let is_single_char = match os_string.to_str() {
+                    Some(s) => s.chars().count() == 1,
+                    None => mb_char_len(bytes) == bytes.len(),
+                };
+                if !is_single_char {
                     return Err(USimpleError::new(
                         1,
                         translate!("cut-error-delimiter-must-be-single-character"),
@@ -583,6 +718,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
 
     let (delimiter, out_delimiter) = get_delimiters(&matches)?;
     let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED));
+    // `-n`: only meaningful with `-b`; keeps multi-byte characters intact.
+    let byte_no_split = matches.get_flag(options::NOTHING);
 
     // Only one, and only one of cutting mode arguments, i.e. `-b`, `-c`, `-f`,
     // is expected. The number of those arguments is used for parsing a cutting
@@ -610,6 +747,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
                         out_delimiter,
                         line_ending,
                         field_opts: None,
+                        byte_no_split,
                     },
                 )
             })
@@ -623,6 +761,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
                         out_delimiter,
                         line_ending,
                         field_opts: None,
+                        byte_no_split,
                     },
                 )
             })
@@ -639,6 +778,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
                             delimiter,
                             only_delimited,
                         }),
+                        byte_no_split,
                     },
                 )
             })
@@ -776,7 +916,7 @@ pub fn uu_app() -> Command {
         .arg(
             Arg::new(options::NOTHING)
                 .short('n')
-                .help("(ignored)")
+                .help(translate!("cut-help-no-split-multibyte"))
                 .action(ArgAction::SetTrue),
         )
 }

src/uucore/src/lib/features/i18n/charmap.rs

@@ -10,6 +10,8 @@
 use std::sync::OnceLock;
 
 enum MbEncoding {
+    /// C/POSIX: no multibyte characters, every byte is one unit.
+    SingleByte,
     Utf8,
     Gb18030,
     EucJp,
@@ -34,8 +36,12 @@ fn get_encoding() -> &'static MbEncoding {
             .iter()
             .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty()));
         let s = match val.as_deref() {
-            Some(s) if s != "C" && s != "POSIX" => s,
-            _ => return MbEncoding::Utf8,
+            // Explicit C/POSIX locale: single-byte, no multibyte decoding.
+            Some("C" | "POSIX") => return MbEncoding::SingleByte,
+            Some(s) => s,
+            // No locale set: keep UTF-8 as the default so behavior is
+            // deterministic regardless of the environment.
+            None => return MbEncoding::Utf8,
         };
         if let Some(enc) = s.split('.').nth(1) {
             let enc = enc.split('@').next().unwrap_or(enc);
@@ -52,13 +58,25 @@ fn get_encoding() -> &'static MbEncoding {
 }
 
 /// Byte length of the first character in `bytes` under the current locale encoding.
+///
+/// The ASCII fast path is kept small and `#[inline]` so callers looping over a
+/// buffer (e.g. `cut -c`) pay only a byte comparison per ASCII byte, even in
+/// build profiles without cross-crate LTO. The multi-byte dispatch is outlined.
+#[inline]
 pub fn mb_char_len(bytes: &[u8]) -> usize {
     debug_assert!(!bytes.is_empty());
     let b0 = bytes[0];
     if b0 <= 0x7F {
         return 1;
     }
+    mb_char_len_multibyte(bytes, b0)
+}
+
+/// Cold slow path of [`mb_char_len`] for bytes that start a multi-byte sequence.
+#[cold]
+fn mb_char_len_multibyte(bytes: &[u8], b0: u8) -> usize {
     match get_encoding() {
+        MbEncoding::SingleByte => 1,
         MbEncoding::Utf8 => utf8_len(bytes, b0),
         MbEncoding::Gb18030 => gb18030_len(bytes, b0),
         MbEncoding::EucJp => eucjp_len(bytes, b0),