From 2bf150520e6059f0173ae16442e171df291277e0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 20 Oct 2023 15:48:30 +0100 Subject: [PATCH 01/78] gh-111201: A new Python REPL --- Lib/_pyrepl/__init__.py | 19 + Lib/_pyrepl/__main__.py | 78 ++++ Lib/_pyrepl/_minimal_curses.py | 68 ++++ Lib/_pyrepl/commands.py | 444 +++++++++++++++++++++ Lib/_pyrepl/completing_reader.py | 284 ++++++++++++++ Lib/_pyrepl/console.py | 91 +++++ Lib/_pyrepl/curses.py | 36 ++ Lib/_pyrepl/fancy_termios.py | 74 ++++ Lib/_pyrepl/historical_reader.py | 344 +++++++++++++++++ Lib/_pyrepl/input.py | 102 +++++ Lib/_pyrepl/keymap.py | 215 +++++++++++ Lib/_pyrepl/reader.py | 643 +++++++++++++++++++++++++++++++ Lib/_pyrepl/readline.py | 489 +++++++++++++++++++++++ Lib/_pyrepl/simple_interact.py | 95 +++++ Lib/_pyrepl/trace.py | 17 + Lib/_pyrepl/unix_console.py | 606 +++++++++++++++++++++++++++++ Lib/_pyrepl/unix_eventqueue.py | 150 +++++++ Lib/code.py | 6 +- Modules/main.c | 20 +- Python/pythonrun.c | 2 - 20 files changed, 3774 insertions(+), 9 deletions(-) create mode 100644 Lib/_pyrepl/__init__.py create mode 100644 Lib/_pyrepl/__main__.py create mode 100644 Lib/_pyrepl/_minimal_curses.py create mode 100644 Lib/_pyrepl/commands.py create mode 100644 Lib/_pyrepl/completing_reader.py create mode 100644 Lib/_pyrepl/console.py create mode 100644 Lib/_pyrepl/curses.py create mode 100644 Lib/_pyrepl/fancy_termios.py create mode 100644 Lib/_pyrepl/historical_reader.py create mode 100644 Lib/_pyrepl/input.py create mode 100644 Lib/_pyrepl/keymap.py create mode 100644 Lib/_pyrepl/reader.py create mode 100644 Lib/_pyrepl/readline.py create mode 100644 Lib/_pyrepl/simple_interact.py create mode 100644 Lib/_pyrepl/trace.py create mode 100644 Lib/_pyrepl/unix_console.py create mode 100644 Lib/_pyrepl/unix_eventqueue.py diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 00000000000000..1693cbd0b98b74 --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 00000000000000..e81ee16f8c857a --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,78 @@ +import os +import sys + +irc_header = "And now for something completely different" + + +def interactive_console(mainmodule=None, quiet=False): + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + # + run_interactive = run_simple_interactive_console + try: + if not os.isatty(sys.stdin.fileno()): + # Bail out if stdin is not tty-like, as pyrepl wouldn't be happy + # For example, with: + # subprocess.Popen(['pypy', '-i'], stdin=subprocess.PIPE) + raise ImportError + from .simple_interact import check + + if not check(): + raise ImportError + from .simple_interact import run_multiline_interactive_console + + run_interactive = run_multiline_interactive_console + # except ImportError: + # pass + except SyntaxError: + print("Warning: 'import pyrepl' failed with SyntaxError") + run_interactive(mainmodule) + + +def run_simple_interactive_console(mainmodule): + import code + + if mainmodule is None: + import __main__ as mainmodule + console = code.InteractiveConsole(mainmodule.__dict__, filename="") + # some parts of code.py are copied here because it was impossible + # to start an interactive console without printing at least one line + # of banner. This was fixed in 3.4; but then from 3.6 it prints a + # line when exiting. This can be disabled too---by passing an argument + # that doesn't exist in <= 3.5. So, too much mess: just copy the code. + more = 0 + while 1: + try: + if more: + prompt = getattr(sys, "ps2", "... ") + else: + prompt = getattr(sys, "ps1", ">>> ") + try: + line = input(prompt) + except EOFError: + console.write("\n") + break + else: + more = console.push(line) + except KeyboardInterrupt: + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + more = 0 + + +# ____________________________________________________________ + +if __name__ == "__main__": # for testing + if os.getenv("PYTHONSTARTUP"): + exec( + compile( + open(os.getenv("PYTHONSTARTUP")).read(), + os.getenv("PYTHONSTARTUP"), + "exec", + ) + ) + interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000000..7cff7d7ca2ea51 --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib(): + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if ctypes.cast(result, ctypes.c_void_p).value == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 00000000000000..0f65bf5a620976 --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,444 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import os + +# Catgories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +class Command: + finish = 0 + kills_digit_arg = 1 + + def __init__(self, reader, event_name, event): + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self): + pass + + +class KillCommand(Command): + def kill_range(self, start, end): + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = 1 + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = 1 + pass + + +def is_kill(command): + return command and issubclass(command, KillCommand) + + +def is_yank(command): + return command and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = 0 + + def do(self): + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = 1 + + +class clear_screen(Command): + def do(self): + r = self.reader + r.console.clear() + r.dirty = 1 + + +class refresh(Command): + def do(self): + self.reader.dirty = 1 + + +class repaint(Command): + def do(self): + self.reader.dirty = 1 + self.reader.console.repaint_prep() + + +class kill_line(KillCommand): + def do(self): + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self): + r = self.reader + self.kill_range(r.bol(), r.pos) + + +# XXX unix_word_rubout and backward_kill_word should actually +# do different things... + + +class unix_word_rubout(KillCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self): + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self): + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = 1 + + +class interrupt(FinishCommand): + def do(self): + import signal + + self.reader.console.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class suspend(Command): + def do(self): + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + r.posxy = 0, 0 + r.dirty = 1 + r.console.screen = [] + + +class up(MotionCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + bol1 = r.bol() + if bol1 == 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + bol2 = r.bol(bol1 - 1) + line_pos = r.pos - bol1 + if line_pos > bol1 - bol2 - 1: + r.sticky_y = line_pos + r.pos = bol1 - 1 + else: + r.pos = bol2 + line_pos + + +class down(MotionCommand): + def do(self): + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + bol1 = r.bol() + eol1 = r.eol() + if eol1 == len(b): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + eol2 = r.eol(eol1 + 1) + if r.pos - bol1 > eol2 - eol1 - 1: + r.pos = eol2 + else: + r.pos = eol1 + (r.pos - bol1) + 1 + + +class left(MotionCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self): + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self): + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self): + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self): + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self): + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self): + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self): + r = self.reader + r.insert(self.event * r.get_arg()) + + +class insert_nl(EditCommand): + def do(self): + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self): + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = 1 + + +class backspace(EditCommand): + def do(self): + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = 1 + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self): + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = 1 + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self): + pass + + +class help(Command): + def do(self): + self.reader.msg = self.reader.help_text + self.reader.dirty = 1 + + +class invalid_key(Command): + def do(self): + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self): + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class qIHelp(Command): + def do(self): + from .reader import disp_str + + r = self.reader + pending = r.console.getpending().data + disp = disp_str((self.event + pending).encode())[0] + r.insert(disp * r.get_arg()) + r.pop_input_trans() + + +class QITrans: + def push(self, evt): + self.evt = evt + + def get(self): + return ("qIHelp", self.evt.raw) + + +class quoted_insert(Command): + kills_digit_arg = 0 + + def do(self): + # XXX in Python 3, processing insert/C-q/C-v keys crashes + # because of a mixture of str and bytes. Disable these keys. + pass + # self.reader.push_input_trans(QITrans()) diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 00000000000000..ad29c1c0fe9a18 --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,284 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import re +from . import commands, reader +from .reader import Reader + + +def prefix(wordlist, j=0): + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s): + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s): + return len(stripcolor(s)) + + +def left_align(s, maxlen): + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu(cons, wordlist, start, use_brackets, sort_in_column): + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amout + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self): + r = self.reader + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = 1 + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + if not r.cmpltn_menu_vis: + r.cmpltn_menu_vis = 1 + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = 1 + elif stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = 1 + else: + r.msg = "[ not unique ]" + r.dirty = 1 + + +class self_insert(commands.self_insert): + def do(self): + commands.self_insert.do(self) + r = self.reader + if r.cmpltn_menu_vis: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +class CompletingReader(Reader): + """Adds completion support + + Adds instance variables: + * cmpltn_menu, cmpltn_menu_vis, cmpltn_menu_end, cmpltn_choices: + * + """ + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + def collect_keymap(self): + return super(CompletingReader, self).collect_keymap() + ( + (r'\t', 'complete'),) + + def __init__(self, console): + super(CompletingReader, self).__init__(console) + self.cmpltn_menu = ["[ menu 1 ]", "[ menu 2 ]"] + self.cmpltn_menu_vis = 0 + self.cmpltn_menu_end = 0 + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def after_command(self, cmd): + super(CompletingReader, self).after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self): + screen = super(CompletingReader, self).calc_screen() + if self.cmpltn_menu_vis: + ly = self.lxy[1] + screen[ly:ly] = self.cmpltn_menu + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + self.cxy = self.cxy[0], self.cxy[1] + len(self.cmpltn_menu) + return screen + + def finish(self): + super(CompletingReader, self).finish() + self.cmpltn_reset() + + def cmpltn_reset(self): + self.cmpltn_menu = [] + self.cmpltn_menu_vis = 0 + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self): + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem): + return [] + + +def test(): + class TestReader(CompletingReader): + def get_completions(self, stem): + return [s for l in self.history + for s in l.split() + if s and s.startswith(stem)] + + reader = TestReader() + reader.ps1 = "c**> " + reader.ps2 = "c/*> " + reader.ps3 = "c|*> " + reader.ps4 = r"c\*> " + while reader.readline(): + pass + + +if __name__ == '__main__': + test() diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 00000000000000..386203edfb5d28 --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,91 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import dataclasses + +@dataclasses.dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +class Console: + """Attributes: + + screen, + height, + width, + """ + + def refresh(self, screen, xy): + pass + + def prepare(self): + pass + + def restore(self): + pass + + def move_cursor(self, x, y): + pass + + def set_cursor_vis(self, vis): + pass + + def getheightwidth(self): + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + pass + + def get_event(self, block=1): + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + pass + + def beep(self): + pass + + def clear(self): + """Wipe the screen""" + pass + + def finish(self): + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + pass + + def flushoutput(self): + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + pass + + def forgetinput(self): + """Forget all pending, but not yet processed input.""" + pass + + def getpending(self): + """Return the characters that have been typed but not yet + processed.""" + pass + + def wait(self): + """Wait for an event.""" + pass diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 00000000000000..aedf46222d27d0 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,36 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# If we are running on top of pypy, we import only _minimal_curses. +# Don't try to fall back to _curses, because that's going to use cffi +# and fall again more loudly. + +try: + import _curses +except ImportError: + try: + import curses as _curses + except ImportError: + import _curses + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 00000000000000..5b85cb0f52521f --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,74 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 00000000000000..65a98af3743800 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,344 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from . import commands, input +from .reader import Reader as R + +isearch_keymap = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self): + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self): + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class restore_history(commands.Command): + def do(self): + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = 1 + + +class first_history(commands.Command): + def do(self): + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self): + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self): + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self): + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = 1 + + +class forward_history_isearch(commands.Command): + def do(self): + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = 1 + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self): + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = 1 + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self): + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = 1 + + +class isearch_add_character(commands.Command): + def do(self): + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = 1 + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self): + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = 1 + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self): + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self): + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self): + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = 1 + + +class HistoricalReader(R): + """Adds history support (with incremental history searching) to the + Reader class. + + Adds the following instance variables: + * history: + a list of strings + * historyi: + * transient_history: + * next_history: + * isearch_direction, isearch_term, isearch_start: + * yank_arg_i, yank_arg_yanked: + used by the yank-arg command; not actually manipulated by any + HistoricalReader instance methods. + """ + + def collect_keymap(self): + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "last-history"), + (r"\", "first-history"), + ) + + def __init__(self, console): + super().__init__(console) + self.history = [] + self.historyi = 0 + self.transient_history = {} + self.next_history = None + self.isearch_direction = ISEARCH_DIRECTION_NONE + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def select_item(self, i): + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i] + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = 1 + + def get_item(self, i): + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + def prepare(self): + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno, cursor_on_line): + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def isearch_next(self): + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self): + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True + + +def test(): + from .unix_console import UnixConsole + + reader = HistoricalReader(UnixConsole()) + reader.ps1 = "h**> " + reader.ps2 = "h/*> " + reader.ps3 = "h|*> " + reader.ps4 = r"h\*> " + while reader.readline(): + pass + + +if __name__ == "__main__": + test() diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 00000000000000..974df83e1a4283 --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,102 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. +import unicodedata +from collections import deque + + +class InputTranslator: + def push(self, evt): + pass + + def get(self): + pass + + def empty(self): + pass + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=0, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self): + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 00000000000000..31a02642ce8ceb --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,215 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +functions for parsing keyspecs + +Support for turning keyspecs into appropriate sequences. + +pyrepl uses it's own bastardized keyspec format, which is meant to be +a strict superset of readline's \"KEYSEQ\" format (which is to say +that if you can come up with a spec readline accepts that this +doesn't, you've found a bug and should tell me about it). + +Note that this is the `\\C-o' style of readline keyspec, not the +`Control-o' sort. + +A keyspec is a string representing a sequence of keypresses that can +be bound to a command. + +All characters other than the backslash represent themselves. In the +traditional manner, a backslash introduces a escape sequence. + +The extension to readline is that the sequence \\ denotes the +sequence of charaters produced by hitting KEY. + +Examples: + +`a' - what you get when you hit the `a' key +`\\EOA' - Escape - O - A (up, on my terminal) +`\\' - the up arrow key +`\\' - ditto (keynames are case insensitive) +`\\C-o', `\\c-o' - control-o +`\\M-.' - meta-period +`\\E.' - ditto (that's how meta works for pyrepl) +`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. Can you think of any more? +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def _parse_key1(key, s): + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) > 1: + raise KeySpecError("\\C- must be followed by a character") + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + if meta: + ret = ["\033", ret] + else: + ret = [ret] + return ret, s + + +def parse_keys(key): + s = 0 + r = [] + while s < len(key): + k, s = _parse_key1(key, s) + r.extend(k) + return r + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 00000000000000..277287dca79059 --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,643 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import re +import unicodedata + +from . import commands, input + +_r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]") + + +def _make_unctrl_map(): + uc_map = {} + for i in range(256): + c = chr(i) + if unicodedata.category(c)[0] != "C": + uc_map[i] = c + for i in range(32): + uc_map[i] = "^" + chr(ord("A") + i - 1) + uc_map[ord(b"\t")] = " " # display TABs as 4 characters + uc_map[ord(b"\177")] = "^?" + for i in range(256): + if i not in uc_map: + uc_map[i] = "\\%03o" % i + return uc_map + + +def _my_unctrl(c, u=_make_unctrl_map()): + # takes an integer, returns a unicode + if c in u: + return u[c] + else: + if unicodedata.category(c).startswith("C"): + return r"\u%04x" % ord(c) + else: + return c + + +if "a"[0] == b"a": + # When running tests with python2, bytes characters are bytes. + def _my_unctrl(c, uc=_my_unctrl): + return uc(ord(c)) + + +def disp_str(buffer, join="".join, uc=_my_unctrl): + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed represenation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + the list always contains 0s or 1s at present; it could conceivably + go higher as and when unicode support happens.""" + # disp_str proved to be a bottleneck for large inputs, + # so it needs to be rewritten in C; it's not required though. + s = [uc(x) for x in buffer] + b = [] # XXX: bytearray + for x in s: + b.append(1) + b.extend([0] * (len(x) - 1)) + return join(s), b + + +del _my_unctrl + +del _make_unctrl_map + +# syntax classes: + +[SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL] = range(3) + + +def make_default_syntax_table(): + # XXX perhaps should use some unicodedata here? + st = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +default_keymap = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-q", "quoted-insert"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-v", "quoted-insert"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + (r"\C-z", "suspend"), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + # (r'\M-\n', 'insert-nl'), + ("\\\\", "self-insert"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\", "right"), + (r"\", "quoted-insert"), + (r"\", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into `buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. I'd like to + get rid of it, because its contents are obtuse (to put it + mildly) but I haven't worked out if that is possible yet. + * cxy, lxy: + the position of the insertion point in screen ... XXX + * syntax_table: + Dictionary mapping characters to `syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + msg_at_bottom = True + + def __init__(self, console): + self.buffer = [] + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.pos = 0 + self.ps1 = "->> " + self.ps2 = "/>> " + self.ps3 = "|.. " + self.ps4 = r"\__ " + self.kill_ring = [] + self.arg = None + self.finished = 0 + self.console = console + self.commands = {} + self.msg = "" + for v in vars(commands).values(): + if ( + isinstance(v, type) + and issubclass(v, commands.Command) + and v.__name__[0].islower() + ): + self.commands[v.__name__] = v + self.commands[v.__name__.replace("_", "-")] = v + self.syntax_table = make_default_syntax_table() + self.input_trans_stack = [] + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def collect_keymap(self): + return default_keymap + + def calc_screen(self): + """The purpose of this method is to translate changes in + self.buffer into changes in self.screen. Currently it rips + everything down and starts from scratch, which whilst not + especially efficient is certainly simple(r). + """ + lines = self.get_unicode().split("\n") + screen = [] + screeninfo = [] + w = self.console.width - 1 + p = self.pos + for ln, line in zip(range(len(lines)), lines): + ll = len(line) + if 0 <= p <= ll: + if self.msg and not self.msg_at_bottom: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + self.lxy = p, ln + prompt = self.get_prompt(ln, ll >= p >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + screen.append(pre_prompt) + screeninfo.append((0, [])) + p -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (len(l) + lp) // w + if wrapcount == 0: + screen.append(prompt + l) + screeninfo.append((lp, l2 + [1])) + else: + screen.append(prompt + l[: w - lp] + "\\") + screeninfo.append((lp, l2[: w - lp])) + for i in range(-lp + w, -lp + wrapcount * w, w): + screen.append(l[i : i + w] + "\\") + screeninfo.append((0, l2[i : i + w])) + screen.append(l[wrapcount * w - lp :]) + screeninfo.append((0, l2[wrapcount * w - lp :] + [1])) + self.screeninfo = screeninfo + self.cxy = self.pos2xy(self.pos) + if self.msg and self.msg_at_bottom: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + return screen + + def process_prompt(self, prompt): + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + out_prompt = "" + l = len(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(len, _r_csi_seq.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(len, _r_csi_seq.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p=None): + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p=None): + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p=None): + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + # XXX there are problems here. + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p=None): + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def get_arg(self, default=1): + """Return any prefix argument that the user has supplied, + returning `default' if there is None. `default' defaults + (groan) to 1.""" + if self.arg is None: + return default + else: + return self.arg + + def get_prompt(self, lineno, cursor_on_line): + """Return what should be in the left-hand margin for line + `lineno'.""" + if self.arg is not None and cursor_on_line: + return "(arg: %s) " % self.arg + if "\n" in self.buffer: + if lineno == 0: + res = self.ps2 + elif lineno == self.buffer.count("\n"): + res = self.ps4 + else: + res = self.ps3 + else: + res = self.ps1 + # Lazily call str() on self.psN, and cache the results using as key + # the object on which str() was called. This ensures that even if the + # same object is used e.g. for ps1 and ps2, str() is called only once. + if res not in self._pscache: + self._pscache[res] = str(res) + return self._pscache[res] + + def push_input_trans(self, itrans): + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self): + self.input_trans = self.input_trans_stack.pop() + + def pos2xy(self, pos): + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + y = 0 + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer): + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + len(l2) - 1, y + else: + for p, l2 in self.screeninfo: + l = l2.count(1) + if l > pos: + break + else: + pos -= l + y += 1 + c = 0 + i = 0 + while c < pos: + c += l2[i] + i += 1 + while l2[i] == 0: + i += 1 + return p + i, y + + def insert(self, text): + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = 1 + + def update_cursor(self): + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy(self.pos) + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd): + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", 1): + if self.arg is not None: + self.dirty = 1 + self.arg = None + + def prepare(self): + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.screeninfo = [] + self.finished = 0 + del self.buffer[:] + self.pos = 0 + self.dirty = 1 + self.last_command = None + self._pscache = {} + except: + self.restore() + raise + + def last_command_is(self, klass): + if not self.last_command: + return 0 + return issubclass(klass, self.last_command) + + def restore(self): + """Clean up after a run.""" + self.console.restore() + + def finish(self): + """Called when a command signals that we're finished.""" + pass + + def error(self, msg="none"): + self.msg = "! " + msg + " " + self.dirty = 1 + self.console.beep() + + def update_screen(self): + if self.dirty: + self.refresh() + + def refresh(self): + """Recalculate and refresh the screen.""" + # this call sets up self.cxy, so call it first. + screen = self.calc_screen() + self.console.refresh(screen, self.cxy) + self.dirty = 0 # forgot this for a while (blush) + + def do_cmd(self, cmd): + # print cmd + if isinstance(cmd[0], str): + cmd = self.commands.get(cmd[0], commands.invalid_command)(self, *cmd) + elif isinstance(cmd[0], type): + cmd = cmd[0](self, *cmd) + else: + return # nothing to do + + cmd.do() + + self.after_command(cmd) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = cmd.__class__ + + self.finished = cmd.finish + if self.finished: + self.console.finish() + self.finish() + + def handle1(self, block=1): + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return None if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = 1 + + while 1: + event = self.console.get_event(block) + if not event: # can only happen if we're not blocking + return None + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = event.evt, event.data + + if cmd is None: + if block: + continue + else: + return None + + self.do_cmd(cmd) + return 1 + + def push_char(self, char): + self.console.push_char(char) + self.handle1(0) + + def readline(self, returns_unicode=False, startup_hook=None): + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + if returns_unicode: + return self.get_unicode() + return self.get_buffer() + finally: + self.restore() + + def bind(self, spec, command): + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_buffer(self, encoding=None): + if encoding is None: + encoding = self.console.encoding + return self.get_unicode().encode(encoding) + + def get_unicode(self): + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) + + +def test(): + from .unix_console import UnixConsole + + reader = Reader(UnixConsole()) + reader.ps1 = "**> " + reader.ps2 = "/*> " + reader.ps3 = "|*> " + reader.ps4 = r"\*> " + while reader.readline(): + pass + + +if __name__ == "__main__": + test() diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 00000000000000..868730eab9f9e8 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,489 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +import os +import readline +import sys + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .unix_console import UnixConsole, _error + +ENCODING = sys.getfilesystemencoding() or "latin1" # XXX review + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + "read_init_file", + "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +class ReadlineConfig: + readline_completer = readline.get_completer() + completer_delims = dict.fromkeys(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + def error(self, msg="none"): + pass # don't show error messages by default + + def get_stem(self): + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem): + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength): + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + # --- simplified support for reading multiline Python statements --- + + # This duplicates small parts of pyrepl.python_reader. I'm not + # reusing the PythonicReader class directly for two reasons. One is + # to try to keep as close as possible to CPython's prompt. The + # other is that it is the readline module that we are ultimately + # implementing here, and I don't want the built-in raw_input() to + # start trying to read multiline inputs just because what the user + # typed look like valid but incomplete Python code. So we get the + # multiline feature only when using the multiline_input() function + # directly (see _pypy_interact.py). + + more_lines = None + + def collect_keymap(self): + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def __init__(self, console): + super().__init__(console) + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def after_command(self, cmd): + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history): + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer, pos): + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer, pos): + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +class maybe_accept(commands.Command): + def do(self): + r = self.reader + r.dirty = 1 # this is needed to hide the completion menu, if visible + # + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + # + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + else: + self.finish = 1 + + +class backspace_dedent(commands.Command): + def do(self): + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = 1 + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +class _ReadlineWrapper: + reader = None + saved_history_length = -1 + startup_hook = None + config = ReadlineConfig() + + def __init__(self, f_in=None, f_out=None): + self.f_in = f_in if f_in is not None else os.dup(0) + self.f_out = f_out if f_out is not None else os.dup(1) + + def get_reader(self): + if self.reader is None: + console = UnixConsole(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console) + self.reader.config = self.config + return self.reader + + def raw_input(self, prompt=""): + try: + reader = self.get_reader() + except _error: + return _old_raw_input(prompt) + reader.ps1 = prompt + return reader.readline(returns_unicode=True, startup_hook=self.startup_hook) + + def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = reader.ps2 = ps1 + reader.ps3 = reader.ps4 = ps2 + return reader.readline(returns_unicode=returns_unicode) + finally: + reader.more_lines = saved + + def parse_and_bind(self, string): + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function=None): + self.config.readline_completer = function + + def get_completer(self): + return self.config.readline_completer + + def set_completer_delims(self, string): + self.config.completer_delims = dict.fromkeys(string) + + def get_completer_delims(self): + chars = list(self.config.completer_delims.keys()) + chars.sort() + return "".join(chars) + + def _histline(self, line): + line = line.rstrip("\n") + if isinstance(line, str): + return line # on py3k + return str(line, "utf-8", "replace") + + def get_history_length(self): + return self.saved_history_length + + def set_history_length(self, length): + self.saved_history_length = length + + def get_current_history_length(self): + return len(self.get_reader().history) + + def read_history_file(self, filename="~/.history"): + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + f = open(os.path.expanduser(filename), encoding="utf-8", errors="replace") + buffer = [] + for line in f: + if line.endswith("\r\n"): + buffer.append(line) + else: + line = self._histline(line) + if buffer: + line = "".join(buffer).replace("\r", "") + line + del buffer[:] + if line: + history.append(line) + f.close() + + def write_history_file(self, filename="~/.history"): + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + with open(os.path.expanduser(filename), "a", encoding="utf-8") as f: + for entry in history: + # if we are on py3k, we don't need to encode strings before + # writing it to a file + if isinstance(entry, str) and sys.version_info < (3,): + entry = entry.encode("utf-8") + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self): + del self.get_reader().history[:] + + def get_history_item(self, index): + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # blame readline.c for not raising + + def remove_history_item(self, index): + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # blame readline.c for raising ValueError + + def replace_history_item(self, index, line): + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # blame readline.c for raising ValueError + + def add_history(self, line): + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function=None): + self.startup_hook = function + + def get_line_buffer(self): + return self.get_reader().get_buffer() + + def _get_idxs(self): + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if str(buf[i]) in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self): + return self._get_idxs()[0] + + def get_endidx(self): + return self._get_idxs()[1] + + def insert_text(self, text): + return self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name, _ret): + def stub(*args, **kwds): + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.func_name = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup(): + global _old_raw_input + if _old_raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # this is not really what readline.c does. Better than nothing I guess + import builtins + + _old_raw_input = builtins.input + builtins.input = _wrapper.raw_input + + +_old_raw_input = None +_setup() diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 00000000000000..303f0c5575d99e --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,95 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +import linecache +import sys + +from .readline import _error, _get_reader, multiline_input + + +def check(): # returns False if there is a problem initializing the state + try: + _get_reader() + except _error: + return False + return True + + +def _strip_final_indent(text): + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +def run_multiline_interactive_console(mainmodule=None, future_flags=0): + import code + + import __main__ + + mainmodule = mainmodule or __main__ + console = code.InteractiveConsole(mainmodule.__dict__, filename="") + if future_flags: + console.compile.compiler.flags |= future_flags + + input_n = 0 + + def more_lines(unicodetext): + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2, returns_unicode=True) + except EOFError: + break + input_name = f"" + linecache._register_code(input_name, statement, "") + more = console.push(_strip_final_indent(statement), filename=input_name) + input_n += 1 + assert not more + except KeyboardInterrupt: + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 00000000000000..f141660220bf00 --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,17 @@ +import os + +trace_filename = os.environ.get("PYREPL_TRACE") + +if trace_filename is not None: + trace_file = open(trace_filename, "a") +else: + trace_file = None + + +def trace(line, *k, **kw): + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 00000000000000..2dd3eb7ef94635 --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,606 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import errno +import os +import re +import select +import signal +import struct +import sys +import termios +import time +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +# there are arguments for changing this to "refresh" +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + + +def _my_getstr(cap, optional=0): + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + "terminal doesn't have the required '%s' capability" % cap + ) + return r + + +# at this point, can we say: AAAAAAAAAAAAAAAAAAAAAARGH! +def maybe_add_baudrate(dict, rate): + name = "B%d" % rate + if hasattr(termios, name): + dict[getattr(termios, name)] = rate + + +ratedict = {} +for r in [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +]: + maybe_add_baudrate(ratedict, r) + +del r, maybe_add_baudrate + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll = select.poll +except AttributeError: + # this is exactly the minumum necessary to support what we + # do with poll objects + class poll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r, w, e = select.select([self.fd], [], []) + return r + + +POLLIN = getattr(select, "POLLIN", None) + + +class UnixConsole(Console): + def __init__(self, f_in=0, f_out=1, term=None, encoding=None): + if encoding is None: + encoding = sys.getdefaultencoding() + + self.encoding = encoding + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + self.pollob = poll() + self.pollob.register(self.input_fd, POLLIN) + curses.setupterm(term, self.output_fd) + self.term = term + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=1) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=1) + self._cub = _my_getstr("cub", optional=1) + self._cub1 = _my_getstr("cub1", 1) + self._cud = _my_getstr("cud", 1) + self._cud1 = _my_getstr("cud1", 1) + self._cuf = _my_getstr("cuf", 1) + self._cuf1 = _my_getstr("cuf1", 1) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", 1) + self._cuu1 = _my_getstr("cuu1", 1) + self._dch1 = _my_getstr("dch1", 1) + self._dch = _my_getstr("dch", 1) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", 1) + self._ich = _my_getstr("ich", 1) + self._ich1 = _my_getstr("ich1", 1) + self._ind = _my_getstr("ind", 1) + self._pad = _my_getstr("pad", 1) + self._ri = _my_getstr("ri", 1) + self._rmkx = _my_getstr("rmkx", 1) + self._smkx = _my_getstr("smkx", 1) + + ## work out how we're going to sling the cursor around + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + + def change_encoding(self, encoding): + self.encoding = encoding + + def refresh(self, screen, c_xy): + # this function is still too long (over 90 lines) + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.__posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.__posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.__posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + self.flushoutput() + + def __write_changed_line(self, y, oldline, newline, px): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + x = 0 + minlen = min(len(oldline), len(newline)) + # + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while x < minlen and oldline[x] == newline[x] and newline[x] != "\x1b": + x += 1 + if oldline[x:] == newline[x + 1 :] and self.ich1: + if ( + y == self.__posxy[1] + and x > self.__posxy[0] + and oldline[px:x] == newline[px + 1 : x + 1] + ): + x = px + self.__move(x, y) + self.__write_code(self.ich1) + self.__write(newline[x]) + self.__posxy = x + 1, y + elif x < minlen and oldline[x + 1 :] == newline[x + 1 :]: + self.__move(x, y) + self.__write(newline[x]) + self.__posxy = x + 1, y + elif ( + self.dch1 + and self.ich1 + and len(newline) == self.width + and x < len(newline) - 2 + and newline[x + 1 : -1] == oldline[x:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.__posxy = self.width - 2, y + self.__write_code(self.dch1) + self.__move(x, y) + self.__write_code(self.ich1) + self.__write(newline[x]) + self.__posxy = x + 1, y + else: + self.__hide_cursor() + self.__move(x, y) + if len(oldline) > len(newline): + self.__write_code(self._el) + self.__write(newline[x:]) + self.__posxy = len(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x): + if x != self.__posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def move_cursor(self, x, y): + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.__posxy = x, y + self.flushoutput() + + def prepare(self): + # per-readline preparations: + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.lflag &= ~( + termios.ICANON | termios.ECHO | termios.IEXTEN | (termios.ISIG * 1) + ) + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + def restore(self): + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def push_char(self, char): + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block=1): + while self.event_queue.empty(): + while 1: # All hail Unix! + try: + self.push_char(os.read(self.input_fd, 1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + if not block: + break + return self.event_queue.get() + + def wait(self): + self.pollob.poll() + + def set_cursor_vis(self, vis): + if vis: + self.__show_cursor() + else: + self.__hide_cursor() + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint_prep(self): + if not self.__gone_tall: + self.__posxy = 0, self.__posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.__posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + if TIOCGWINSZ: + + def getheightwidth(self): + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + height, width = struct.unpack( + "hhhh", ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + )[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + return 25, 80 + + def forgetinput(self): + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) + + def finish(self): + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = os.read(self.input_fd, amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = os.read(self.input_fd, amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.__posxy = 0, 0 + self.screen = [] diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 00000000000000..8c086e87c49076 --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,150 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# Bah, this would be easier to test if curses/terminfo didn't have so +# much non-introspectable global state. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +_keynames = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +#function keys x in 1-20 -> fX: kfX +_keynames.update(('f%d' % i, 'kf%d' % i) for i in range(1, 21)) + +# this is a bit of a hack: CTRL-left and CTRL-right are not standardized +# termios sequences: each terminal emulator implements its own slightly +# different incarnation, and as far as I know, there is no way to know +# programmatically which sequences correspond to CTRL-left and +# CTRL-right. In bash, these keys usually work because there are bindings +# in ~/.inputrc, but pyrepl does not support it. The workaround is to +# hard-code here a bunch of known sequences, which will be seen as "ctrl +# left" and "ctrl right" keys, which can be finally be mapped to commands +# by the reader's keymaps. +# +CTRL_ARROW_KEYCODE = { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def general_keycodes(): + keycodes = {} + for key, tiname in _keynames.items(): + keycode = curses.tigetstr(tiname) + trace('key {key} tiname {tiname} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODE) + return keycodes + + +def EventQueue(fd, encoding): + keycodes = general_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + keycodes[backspace] = 'backspace' + k = keymap.compile_keymap(keycodes) + trace('keymap {k!r}', k=k) + return EncodedQueue(k, encoding) + + +class EncodedQueue(object): + def __init__(self, keymap, encoding): + self.k = self.ck = keymap + self.events = deque() + self.buf = bytearray() + self.encoding = encoding + + def get(self): + if self.events: + return self.events.popleft() + else: + return None + + def empty(self): + return not self.events + + def flush_buf(self): + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event): + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char): + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.k: + if self.k is self.ck: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.k[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.k = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.k = self.ck + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py, in particular the line \\E. + trace('unrecognized escape sequence, propagating...') + self.k = self.ck + self.insert(Event('key', '\033', bytearray(b'\033'))) + for c in self.flush_buf()[1:]: + self.push(chr(c)) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.k = self.ck diff --git a/Lib/code.py b/Lib/code.py index f4aecddeca7813..a8f3bd12d77d4c 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -280,7 +280,7 @@ def interact(self, banner=None, exitmsg=None): elif exitmsg != '': self.write('%s\n' % exitmsg) - def push(self, line): + def push(self, line, filename=None): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -296,7 +296,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename) if not more: self.resetbuffer() return more diff --git a/Modules/main.c b/Modules/main.c index df2ce550245088..1eecf479a552d4 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -513,8 +513,12 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + if (!isatty(fileno(stdin))) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + return (run != 0); + } + int run = pymain_run_module(L"_pyrepl", 0); return (run != 0); } @@ -537,9 +541,15 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int res = PyRun_AnyFileFlags(stdin, "", &cf); - *exitcode = (res != 0); + if (!isatty(fileno(stdin))) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + *exitcode = (run != 0); + return; + } + int run = pymain_run_module(L"_pyrepl", 0); + *exitcode = (run != 0); + return; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2970248da13705..b1a4f264633977 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -83,8 +83,6 @@ _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, return res; } - -/* Parse input from a file and execute it */ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) From 35d322eeb969d4a10738efde63b8f678762ca0d6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 17:37:20 +0100 Subject: [PATCH 02/78] Clean some files --- Lib/_pyrepl/__main__.py | 17 +++++------ Lib/_pyrepl/simple_interact.py | 1 - Lib/_pyrepl/unix_console.py | 55 ++++++++++++++-------------------- 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index e81ee16f8c857a..3bb3a780c0bd51 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,7 +1,6 @@ import os import sys -irc_header = "And now for something completely different" def interactive_console(mainmodule=None, quiet=False): @@ -26,8 +25,6 @@ def interactive_console(mainmodule=None, quiet=False): from .simple_interact import run_multiline_interactive_console run_interactive = run_multiline_interactive_console - # except ImportError: - # pass except SyntaxError: print("Warning: 'import pyrepl' failed with SyntaxError") run_interactive(mainmodule) @@ -68,11 +65,13 @@ def run_simple_interactive_console(mainmodule): if __name__ == "__main__": # for testing if os.getenv("PYTHONSTARTUP"): - exec( - compile( - open(os.getenv("PYTHONSTARTUP")).read(), - os.getenv("PYTHONSTARTUP"), - "exec", + import tokenize + with tokenize.open(os.getenv("PYTHONSTARTUP")) as f: + exec( + compile( + f.read(), + os.getenv("PYTHONSTARTUP"), + "exec", + ) ) - ) interactive_console() diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 303f0c5575d99e..4c44461498d467 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -50,7 +50,6 @@ def _strip_final_indent(text): def run_multiline_interactive_console(mainmodule=None, future_flags=0): import code - import __main__ mainmodule = mainmodule or __main__ diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 2dd3eb7ef94635..b1289632c88587 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -43,7 +43,6 @@ class InvalidTerminal(RuntimeError): _error = (termios.error, curses.error, InvalidTerminal) -# there are arguments for changing this to "refresh" SIGWINCH_EVENT = "repaint" FIONREAD = getattr(termios, "FIONREAD", None) @@ -59,39 +58,30 @@ def _my_getstr(cap, optional=0): return r -# at this point, can we say: AAAAAAAAAAAAAAAAAAAAAARGH! -def maybe_add_baudrate(dict, rate): - name = "B%d" % rate - if hasattr(termios, name): - dict[getattr(termios, name)] = rate +# ------------ start of baudrate definitions ------------ +# Add (possibly) missing baudrates (check termios man page) to termios + +def add_supported_baudrates(dictionary, rate): + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + +# Check the termios man page (Line speed) to know where these +# values come from. +supported_baudrates = [ + 0, 110, 115200, 1200, 134, 150, 1800, 19200, 200, 230400, + 2400, 300, 38400, 460800, 4800, 50, 57600, 600, 75, 9600 +] ratedict = {} -for r in [ - 0, - 110, - 115200, - 1200, - 134, - 150, - 1800, - 19200, - 200, - 230400, - 2400, - 300, - 38400, - 460800, - 4800, - 50, - 57600, - 600, - 75, - 9600, -]: - maybe_add_baudrate(ratedict, r) - -del r, maybe_add_baudrate +for rate in supported_baudrates: + add_supported_baudrates(ratedict, rate) + +# ------------ end of baudrate definitions ------------ + +# Clean up variables to avoid unintended usage +del rate, add_supported_baudrates delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") @@ -203,7 +193,6 @@ def change_encoding(self, encoding): self.encoding = encoding def refresh(self, screen, c_xy): - # this function is still too long (over 90 lines) cx, cy = c_xy if not self.__gone_tall: while len(self.screen) < min(len(screen), self.height): @@ -445,7 +434,7 @@ def push_char(self, char): def get_event(self, block=1): while self.event_queue.empty(): - while 1: # All hail Unix! + while 1: try: self.push_char(os.read(self.input_fd, 1)) except OSError as err: From 4d2303bbfeb5b2768d90ca632be9977b7f2cd709 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 17:44:17 +0100 Subject: [PATCH 03/78] Fix history across sessions --- Lib/_pyrepl/readline.py | 30 +++++++++++++++++------------- Lib/site.py | 9 +++++++++ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 868730eab9f9e8..65a17b301e6e50 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -325,19 +325,23 @@ def read_history_file(self, filename="~/.history"): # history item: we use \r\n instead of just \n. If the history # file is passed to GNU readline, the extra \r are just ignored. history = self.get_reader().history - f = open(os.path.expanduser(filename), encoding="utf-8", errors="replace") - buffer = [] - for line in f: - if line.endswith("\r\n"): - buffer.append(line) - else: - line = self._histline(line) - if buffer: - line = "".join(buffer).replace("\r", "") + line - del buffer[:] - if line: - history.append(line) - f.close() + + with open(os.path.expanduser(filename), 'rb') as f: + lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + # Ignore readline history file header + if line.startswith("_HiStOrY_V2_"): + continue + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = "".join(buffer).replace("\r", "") + line + del buffer[:] + if line: + history.append(line) def write_history_file(self, filename="~/.history"): maxlength = self.saved_history_length diff --git a/Lib/site.py b/Lib/site.py index 93af9c453ac7bb..f3f2a95487ce82 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -485,6 +485,7 @@ def register_readline(): try: import readline import rlcompleter + import _pyrepl.readline except ImportError: return @@ -516,6 +517,10 @@ def register_readline(): readline.read_history_file(history) except OSError: pass + try: + _pyrepl.readline.read_history_file(history) + except OSError: + pass def write_history(): try: @@ -524,6 +529,10 @@ def write_history(): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass + try: + _pyrepl.readline.write_history_file(history) + except (FileNotFoundError, PermissionError): + pass atexit.register(write_history) From 7763c0513aaf94b6c17720df392fac340eb6b545 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 17:53:36 +0100 Subject: [PATCH 04/78] Fore fixes to history --- Lib/_pyrepl/readline.py | 4 ---- Lib/site.py | 18 ++++++++---------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 65a17b301e6e50..2ba326ffeb7f0a 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -348,10 +348,6 @@ def write_history_file(self, filename="~/.history"): history = self.get_reader().get_trimmed_history(maxlength) with open(os.path.expanduser(filename), "a", encoding="utf-8") as f: for entry in history: - # if we are on py3k, we don't need to encode strings before - # writing it to a file - if isinstance(entry, str) and sys.version_info < (3,): - entry = entry.encode("utf-8") entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") diff --git a/Lib/site.py b/Lib/site.py index f3f2a95487ce82..e33a961977372d 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -514,25 +514,23 @@ def register_readline(): # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() try: - readline.read_history_file(history) - except OSError: - pass - try: - _pyrepl.readline.read_history_file(history) + if os.getenv("PYTHON_OLD_REPL"): + readline.read_history_file(history) + else: + _pyrepl.readline.read_history_file(history) except OSError: pass def write_history(): try: - readline.write_history_file(history) + if os.getenv("PYTHON_OLD_REPL"): + readline.write_history_file(history) + else: + _pyrepl.readline.write_history_file(history) except (FileNotFoundError, PermissionError): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass - try: - _pyrepl.readline.write_history_file(history) - except (FileNotFoundError, PermissionError): - pass atexit.register(write_history) From e85d873a5015d4110cef1ad2e3fc6ac992060d42 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 18:05:23 +0100 Subject: [PATCH 05/78] Lukasz was wrong all along --- Lib/_pyrepl/readline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 2ba326ffeb7f0a..5a90a0ce2df382 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -346,7 +346,7 @@ def read_history_file(self, filename="~/.history"): def write_history_file(self, filename="~/.history"): maxlength = self.saved_history_length history = self.get_reader().get_trimmed_history(maxlength) - with open(os.path.expanduser(filename), "a", encoding="utf-8") as f: + with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: for entry in history: entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") From 842496ef702eca9671486135f609f221b39ad764 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 18:18:31 +0100 Subject: [PATCH 06/78] Implement REPL commands and remove F1 for help (curses shows that scrambled) --- Lib/_pyrepl/commands.py | 6 ------ Lib/_pyrepl/reader.py | 1 - Lib/_pyrepl/simple_interact.py | 19 +++++++++++++++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 0f65bf5a620976..6b2e1a47c13f68 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -396,12 +396,6 @@ def do(self): pass -class help(Command): - def do(self): - self.reader.msg = self.reader.help_text - self.reader.dirty = 1 - - class invalid_key(Command): def do(self): pending = self.reader.console.getpending() diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 277287dca79059..74f19fed7f23da 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -160,7 +160,6 @@ def make_default_syntax_table(): (r"\M-\", "backward-kill-word"), (r"\", "end-of-line"), # was 'end' (r"\", "beginning-of-line"), # was 'home' - (r"\", "help"), (r"\EOF", "end"), # the entries in the terminfo database for xterms (r"\EOH", "home"), # seem to be wrong. this is a less than ideal # workaround diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 4c44461498d467..71f7c2c9e1dfaa 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -23,6 +23,7 @@ allowing multiline input and multiline history entries. """ +import _sitebuiltins import linecache import sys @@ -48,6 +49,14 @@ def _strip_final_indent(text): return text +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": _sitebuiltins._Helper(), +} + + def run_multiline_interactive_console(mainmodule=None, future_flags=0): import code import __main__ @@ -81,11 +90,17 @@ def more_lines(unicodetext): statement = multiline_input(more_lines, ps1, ps2, returns_unicode=True) except EOFError: break + input_name = f"" linecache._register_code(input_name, statement, "") - more = console.push(_strip_final_indent(statement), filename=input_name) + maybe_repl_command = REPL_COMMANDS.get(statement.strip()) + if maybe_repl_command is not None: + maybe_repl_command() + continue + else: + more = console.push(_strip_final_indent(statement), filename=input_name) + assert not more input_n += 1 - assert not more except KeyboardInterrupt: console.write("\nKeyboardInterrupt\n") console.resetbuffer() From 1417b9b9eb75fc802b7679ca1b2f2d3eb879ed37 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 18:22:23 +0100 Subject: [PATCH 07/78] Restore F1 for help --- Lib/_pyrepl/commands.py | 8 ++++++++ Lib/_pyrepl/reader.py | 1 + 2 files changed, 9 insertions(+) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 6b2e1a47c13f68..9cdb700a24c6d7 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -396,6 +396,14 @@ def do(self): pass +class help(Command): + def do(self): + import _sitebuiltins + self.reader.console.restore() + self.reader.msg = _sitebuiltins._Helper()() + self.reader.dirty = 1 + + class invalid_key(Command): def do(self): pending = self.reader.console.getpending() diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 74f19fed7f23da..277287dca79059 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -160,6 +160,7 @@ def make_default_syntax_table(): (r"\M-\", "backward-kill-word"), (r"\", "end-of-line"), # was 'end' (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), (r"\EOF", "end"), # the entries in the terminfo database for xterms (r"\EOH", "home"), # seem to be wrong. this is a less than ideal # workaround From 4c698538945a8b70657885c53ec8fea50a4db332 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 18:26:59 +0100 Subject: [PATCH 08/78] Implement colored prompt --- Lib/_pyrepl/reader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 277287dca79059..b09d7782c0bdf9 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -21,6 +21,7 @@ import re import unicodedata +import traceback from . import commands, input @@ -420,6 +421,9 @@ def get_prompt(self, lineno, cursor_on_line): res = self.ps3 else: res = self.ps1 + + if traceback._can_colorize(): + res = traceback._ANSIColors.BOLD_MAGENTA + res + traceback._ANSIColors.RESET # Lazily call str() on self.psN, and cache the results using as key # the object on which str() was called. This ensures that even if the # same object is used e.g. for ps1 and ps2, str() is called only once. From 359407b81556e17b5805794dffb05476ac6e36bd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 18:53:12 +0100 Subject: [PATCH 09/78] WIP for paste mode --- Lib/_pyrepl/commands.py | 6 ++++++ Lib/_pyrepl/reader.py | 6 ++++++ Lib/_pyrepl/readline.py | 3 ++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 9cdb700a24c6d7..f10e726c49bb3c 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -444,3 +444,9 @@ def do(self): # because of a mixture of str and bytes. Disable these keys. pass # self.reader.push_input_trans(QITrans()) + +class paste_mode(Command): + + def do(self): + self.reader.paste_mode = True + self.reader.dirty = 1 \ No newline at end of file diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index b09d7782c0bdf9..c9974af1642b1a 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -126,6 +126,7 @@ def make_default_syntax_table(): (r"\C-x\C-u", "upcase-region"), (r"\C-y", "yank"), (r"\C-z", "suspend"), + (r"\C-t", "paste-mode"), (r"\M-b", "backward-word"), (r"\M-c", "capitalize-word"), (r"\M-d", "kill-word"), @@ -256,6 +257,8 @@ def __init__(self, console): self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) + self.paste_mode = False + def collect_keymap(self): return default_keymap @@ -421,6 +424,9 @@ def get_prompt(self, lineno, cursor_on_line): res = self.ps3 else: res = self.ps1 + + if self.paste_mode: + res= '(paste mode)' if traceback._can_colorize(): res = traceback._ANSIColors.BOLD_MAGENTA + res + traceback._ANSIColors.RESET diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 5a90a0ce2df382..5f30dacd4cba76 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -215,7 +215,7 @@ def do(self): # auto-indent the next line like the previous line prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) r.insert("\n") - if indent: + if not self.reader.paste_mode and indent: for i in range(prevlinestart, prevlinestart + indent): r.insert(r.buffer[i]) else: @@ -286,6 +286,7 @@ def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): return reader.readline(returns_unicode=returns_unicode) finally: reader.more_lines = saved + reader.paste_mode = False def parse_and_bind(self, string): pass # XXX we don't support parsing GNU-readline-style init files From 85a2b35c90e37eaf677d8fca5c18b89271d03b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 27 Apr 2024 21:51:23 +0200 Subject: [PATCH 10/78] Only execute PYTHONSTARTUP when asked (we do it in main.c anyway) --- Lib/_pyrepl/__main__.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 3bb3a780c0bd51..d08f2d2d514fe0 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -3,7 +3,14 @@ -def interactive_console(mainmodule=None, quiet=False): +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code) + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This # mimics what CPython does in pythonrun.c if not hasattr(sys, "ps1"): @@ -61,17 +68,5 @@ def run_simple_interactive_console(mainmodule): more = 0 -# ____________________________________________________________ - -if __name__ == "__main__": # for testing - if os.getenv("PYTHONSTARTUP"): - import tokenize - with tokenize.open(os.getenv("PYTHONSTARTUP")) as f: - exec( - compile( - f.read(), - os.getenv("PYTHONSTARTUP"), - "exec", - ) - ) +if __name__ == "__main__": interactive_console() From 05a88316a000024e97c0d7637141994c4cc0d064 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 22:13:19 +0100 Subject: [PATCH 11/78] Fixed coloring --- Lib/_pyrepl/simple_interact.py | 11 ++++++++++- Lib/code.py | 5 +++-- Lib/traceback.py | 5 +++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 71f7c2c9e1dfaa..31f92222c6f6e4 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -26,6 +26,8 @@ import _sitebuiltins import linecache import sys +import code +import traceback from .readline import _error, _get_reader, multiline_input @@ -56,13 +58,20 @@ def _strip_final_indent(text): "help": _sitebuiltins._Helper(), } +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.can_colorize = traceback._can_colorize() + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + def run_multiline_interactive_console(mainmodule=None, future_flags=0): import code import __main__ mainmodule = mainmodule or __main__ - console = code.InteractiveConsole(mainmodule.__dict__, filename="") + console = InteractiveColoredConsole(mainmodule.__dict__, filename="") if future_flags: console.compile.compiler.flags |= future_flags diff --git a/Lib/code.py b/Lib/code.py index a8f3bd12d77d4c..7a5ee6002395ef 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -130,7 +130,7 @@ def showsyntaxerror(self, filename=None): # over self.write sys.excepthook(type, value, tb) - def showtraceback(self): + def showtraceback(self, **kwargs): """Display the exception that just occurred. We remove the first stack item because it is our own code. @@ -138,11 +138,12 @@ def showtraceback(self): The output is written by self.write(), below. """ + colorize = kwargs.pop('colorize', False) sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() sys.last_traceback = last_tb sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) + lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next, colorize=colorize) if sys.excepthook is sys.__excepthook__: self.write(''.join(lines)) else: diff --git a/Lib/traceback.py b/Lib/traceback.py index fccec0c71c3695..771755e1a2ae63 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -172,7 +172,7 @@ def _print_exception_bltin(exc, /): def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -181,9 +181,10 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) def format_exception_only(exc, /, value=_sentinel, *, show_group=False): From c074ca6661f12b5a8b2e0cca59e364e45d295d15 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 23:09:54 +0100 Subject: [PATCH 12/78] Do not run commands if they are shadowed --- Lib/_pyrepl/simple_interact.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 31f92222c6f6e4..496120832cd090 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -102,8 +102,9 @@ def more_lines(unicodetext): input_name = f"" linecache._register_code(input_name, statement, "") - maybe_repl_command = REPL_COMMANDS.get(statement.strip()) - if maybe_repl_command is not None: + stripped_statement = statement.strip() + maybe_repl_command = REPL_COMMANDS.get(stripped_statement) + if maybe_repl_command is not None and stripped_statement not in console.locals: maybe_repl_command() continue else: From 9584b5be9245020bd5d0aec13c8b4197b9b324ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 00:29:36 +0200 Subject: [PATCH 13/78] Move pager routines to _pyrepl, so we can use it in the REPL without importing too much --- Lib/_pyrepl/pager.py | 157 +++++++++++++++++++++++++++++++++++++++++++ Lib/pydoc.py | 147 +--------------------------------------- 2 files changed, 159 insertions(+), 145 deletions(-) create mode 100644 Lib/_pyrepl/pager.py diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 00000000000000..5e7a9e2829e706 --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,157 @@ +import io +import os +import re +import sys + + +def get_pager(): + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plainpager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfilepager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text): + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s): + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text): + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text, title=''): + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + try: + import tty + fd = sys.stdin.fileno() + old = tty.tcgetattr(fd) + tty.setcbreak(fd) + getchar = lambda: sys.stdin.read(1) + except (ImportError, AttributeError, io.UnsupportedOperation): + tty = None + getchar = lambda: sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if tty: + tty.tcsetattr(fd, tty.TCSAFLUSH, old) + + +def plain_pager(text, title=''): + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text, cmd, title=''): + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text, cmd, title=''): + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 02af672a628f5a..6e1a52ad4aff4b 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -76,6 +76,7 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import get_pager # --------------------------------------------------------- common routines @@ -1640,153 +1641,9 @@ def bold(self, text): def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() + pager = get_pager() pager(text, title) -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - if sys.platform == "emscripten": - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text, title='': tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text, title='': pipepager(plain(text), use_pager, title) - else: - return lambda text, title='': pipepager(text, use_pager, title) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text, title='': tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text, title='': pipepager(text, 'less', title) - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text, title='': pipepager(text, 'more', title) - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def escape_less(s): - return re.sub(r'([?:.%\\])', r'\\\1', s) - -def pipepager(text, cmd, title=''): - """Page through text by feeding it to another program.""" - import subprocess - env = os.environ.copy() - if title: - title += ' ' - esc_title = escape_less(title) - prompt_string = ( - f' {esc_title}' + - '?ltline %lt?L/%L.' - ':byte %bB?s/%s.' - '.' - '?e (END):?pB %pB\\%..' - ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace', env=env) - try: - with proc.stdin as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd, title=''): - """Page through text by invoking a program on a temporary file.""" - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - filename = os.path.join(tempdir, 'pydoc.out') - with open(filename, 'w', errors='backslashreplace', - encoding=os.device_encoding(0) if - sys.platform == 'win32' else None - ) as file: - file.write(text) - os.system(cmd + ' "' + filename + '"') - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text, title=''): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text, title=''): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) - def describe(thing): """Produce a short description of the given thing.""" if inspect.ismodule(thing): From d7ddebd83d801809ecf5baa850d852f53e0f9c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 00:32:05 +0200 Subject: [PATCH 14/78] Don't assume stupid paths or you will be tested against your stupid assumptions --- Lib/_pyrepl/readline.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 5f30dacd4cba76..dc89d7504c2ef5 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -28,6 +28,7 @@ import os import readline +from site import gethistoryfile import sys from . import commands, historical_reader @@ -320,7 +321,7 @@ def set_history_length(self, length): def get_current_history_length(self): return len(self.get_reader().history) - def read_history_file(self, filename="~/.history"): + def read_history_file(self, filename=gethistoryfile()): # multiline extension (really a hack) for the end of lines that # are actually continuations inside a single multiline_input() # history item: we use \r\n instead of just \n. If the history @@ -344,7 +345,7 @@ def read_history_file(self, filename="~/.history"): if line: history.append(line) - def write_history_file(self, filename="~/.history"): + def write_history_file(self, filename=gethistoryfile()): maxlength = self.saved_history_length history = self.get_reader().get_trimmed_history(maxlength) with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: @@ -352,6 +353,9 @@ def write_history_file(self, filename="~/.history"): entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") + def copy_history(self): + return self.get_reader().history[:] + def clear_history(self): del self.get_reader().history[:] @@ -422,6 +426,7 @@ def insert_text(self, text): read_history_file = _wrapper.read_history_file write_history_file = _wrapper.write_history_file clear_history = _wrapper.clear_history +copy_history = _wrapper.copy_history get_history_item = _wrapper.get_history_item remove_history_item = _wrapper.remove_history_item replace_history_item = _wrapper.replace_history_item From 2d24cd2dfa2c5dadb9c532adcb343aea760393c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 00:32:49 +0200 Subject: [PATCH 15/78] shows history so you can copy it, is paste mode --- Lib/_pyrepl/commands.py | 14 +++++++++++++- Lib/_pyrepl/reader.py | 7 ++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index f10e726c49bb3c..cad8785d6b6822 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -445,8 +445,20 @@ def do(self): pass # self.reader.push_input_trans(QITrans()) + +class show_history(Command): + def do(self): + # self.reader.console.restore() + from _pyrepl.pager import get_pager + from _pyrepl.readline import copy_history + from site import gethistoryfile + pager = get_pager() + pager(os.linesep.join(copy_history()), gethistoryfile()) + self.reader.dirty = 1 + + class paste_mode(Command): def do(self): self.reader.paste_mode = True - self.reader.dirty = 1 \ No newline at end of file + self.reader.dirty = 1 diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index c9974af1642b1a..81e3359626193d 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -126,7 +126,6 @@ def make_default_syntax_table(): (r"\C-x\C-u", "upcase-region"), (r"\C-y", "yank"), (r"\C-z", "suspend"), - (r"\C-t", "paste-mode"), (r"\M-b", "backward-word"), (r"\M-c", "capitalize-word"), (r"\M-d", "kill-word"), @@ -163,6 +162,8 @@ def make_default_syntax_table(): (r"\", "end-of-line"), # was 'end' (r"\", "beginning-of-line"), # was 'home' (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), (r"\EOF", "end"), # the entries in the terminfo database for xterms (r"\EOH", "home"), # seem to be wrong. this is a less than ideal # workaround @@ -426,8 +427,8 @@ def get_prompt(self, lineno, cursor_on_line): res = self.ps1 if self.paste_mode: - res= '(paste mode)' - + res= '(paste) ' + if traceback._can_colorize(): res = traceback._ANSIColors.BOLD_MAGENTA + res + traceback._ANSIColors.RESET # Lazily call str() on self.psN, and cache the results using as key From fe1b3ac49b5c67f09f88f30a116876e6b2499fdb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Apr 2024 23:34:41 +0100 Subject: [PATCH 16/78] Do not include REPL commands in history --- Lib/_pyrepl/simple_interact.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 496120832cd090..ab963760753db4 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -105,6 +105,7 @@ def more_lines(unicodetext): stripped_statement = statement.strip() maybe_repl_command = REPL_COMMANDS.get(stripped_statement) if maybe_repl_command is not None and stripped_statement not in console.locals: + _get_reader().history.pop() maybe_repl_command() continue else: From e1abd39ef5ac0d5663a53bfdc030673258419761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 00:42:31 +0200 Subject: [PATCH 17/78] Add Blurb --- .../2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst new file mode 100644 index 00000000000000..9176da155d48fb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst @@ -0,0 +1,4 @@ +The :term:`interactive` interpreter is now implemented in Python, which +allows for a number of new features like colors, multiline input, history +viewing, and paste mode. Contributed by Pablo Galindo and Łukasz Langa based +on code from the PyPy project. From afbeaab0de07dfeccfd472954627dae170185910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 00:59:02 +0200 Subject: [PATCH 18/78] Reintroduce old names to fix pydoc tests --- Lib/pydoc.py | 13 ++++++++++++- Python/stdlib_module_names.h | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 6e1a52ad4aff4b..eaaf8249b205bc 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -76,7 +76,18 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only -from _pyrepl.pager import get_pager +from _pyrepl.pager import (get_pager, plain, escape_less, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 08a66f447e2258..32c6ab9b97364d 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -63,6 +63,7 @@ static const char* _Py_stdlib_module_names[] = { "_pydecimal", "_pyio", "_pylong", +"_pyrepl", "_queue", "_random", "_scproxy", From aee9fcfc7fd52b7f03f48698353aafa0904c318c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 13:48:22 +0100 Subject: [PATCH 19/78] Add some tests --- Lib/test/test_pyrepl.py | 298 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 Lib/test/test_pyrepl.py diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py new file mode 100644 index 00000000000000..77cccf35757974 --- /dev/null +++ b/Lib/test/test_pyrepl.py @@ -0,0 +1,298 @@ +from unittest import TestCase +from unittest.mock import MagicMock +from unittest.mock import call +from code import InteractiveConsole +from contextlib import suppress + +from _pyrepl.readline import ReadlineAlikeReader +from _pyrepl.readline import ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent +from _pyrepl.console import Event +from _pyrepl.console import Console + + +def more_lines(unicodetext, namespace=None): + if namespace is None: + namespace = {} + src = _strip_final_indent(unicodetext) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def multiline_input(reader): + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline(returns_unicode=True) + finally: + reader.more_lines = saved + reader.paste_mode = False + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8"): + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block=True): + return next(self.events) + + +class TestPyReplDriver(TestCase): + def prepare_reader(self, events): + console = MagicMock() + console.get_event.side_effect = events + reader = ReadlineAlikeReader(console) + reader.config = ReadlineConfig() + return reader, console + + def test_up_arrow(self): + events = [ + Event(evt="key", data="d", raw=bytearray(b"d")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="f", raw=bytearray(b"f")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data="f", raw=bytearray(b"f")), + Event(evt="key", data="(", raw=bytearray(b"(")), + Event(evt="key", data=")", raw=bytearray(b")")), + Event(evt="key", data=":", raw=bytearray(b":")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + reader, console = self.prepare_reader(events) + + with suppress(StopIteration): + _ = multiline_input(reader) + + console.move_cursor.assert_called_with(1, 3) + + def test_down_arrow(self): + events = [ + Event(evt="key", data="d", raw=bytearray(b"d")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="f", raw=bytearray(b"f")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data="f", raw=bytearray(b"f")), + Event(evt="key", data="(", raw=bytearray(b"(")), + Event(evt="key", data=")", raw=bytearray(b")")), + Event(evt="key", data=":", raw=bytearray(b":")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + reader, console = self.prepare_reader(events) + + with suppress(StopIteration): + _ = multiline_input(reader) + + console.move_cursor.assert_called_with(1, 5) + + def test_left_arrow(self): + events = [ + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + reader, console = self.prepare_reader(events) + + _ = multiline_input(reader) + + console.move_cursor.assert_has_calls( + [ + call(3, 1), + ] + ) + + def test_right_arrow(self): + events = [ + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + reader, console = self.prepare_reader(events) + + _ = multiline_input(reader) + + console.move_cursor.assert_has_calls( + [ + call(4, 1), + ] + ) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + reader = ReadlineAlikeReader(console) + reader.config = ReadlineConfig() + return reader + + def test_basic(self): + events = [ + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_multiline_edit(self): + events = [ + Event(evt="key", data="d", raw=bytearray(b"d")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="f", raw=bytearray(b"f")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data="f", raw=bytearray(b"f")), + Event(evt="key", data="(", raw=bytearray(b"(")), + Event(evt="key", data=")", raw=bytearray(b")")), + Event(evt="key", data=":", raw=bytearray(b":")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data=" ", raw=bytearray(b" ")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n ...\n ") + + def test_history_navigation_with_up_arrow(self): + events = [ + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="2", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="2", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_navigation_with_down_arrow(self): + events = [ + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="2", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="2", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ] + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_search(self): + events = [ + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="2", raw=bytearray(b"2")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="2", raw=bytearray(b"2")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="3", raw=bytearray(b"3")), + Event(evt="key", data="+", raw=bytearray(b"+")), + Event(evt="key", data="3", raw=bytearray(b"3")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + +if __name__ == "__main__": + unittest.main() From 77db9609d7538e8f155619442e2302ab8f1431a3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 14:21:25 +0100 Subject: [PATCH 20/78] Add moar tests --- Lib/test/test_pyrepl.py | 95 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 77cccf35757974..82eabd42eaef16 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -3,6 +3,9 @@ from unittest.mock import call from code import InteractiveConsole from contextlib import suppress +from functools import partial +import rlcompleter +import os from _pyrepl.readline import ReadlineAlikeReader from _pyrepl.readline import ReadlineConfig @@ -24,10 +27,10 @@ def more_lines(unicodetext, namespace=None): return code is None -def multiline_input(reader): +def multiline_input(reader, namespace=None): saved = reader.more_lines try: - reader.more_lines = more_lines + reader.more_lines = partial(more_lines, namespace=namespace) reader.ps1 = reader.ps2 = ">>>" reader.ps3 = reader.ps4 = "..." return reader.readline(returns_unicode=True) @@ -160,6 +163,7 @@ def prepare_reader(self, events): console = FakeConsole(events) reader = ReadlineAlikeReader(console) reader.config = ReadlineConfig() + reader.config.readline_completer = None return reader def test_basic(self): @@ -294,5 +298,92 @@ def test_history_search(self): self.assertEqual(output, "1+1") +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + reader = ReadlineAlikeReader(console) + reader.config = ReadlineConfig() + reader.config.readline_completer = rlcompleter.Completer(namespace).complete + return reader + + def test_simple_completion(self): + events = [ + Event(evt="key", data="o", raw=bytearray(b"o")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="t", raw=bytearray(b"t")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="n", raw=bytearray(b"n")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getenv") + + def test_completion_with_many_options(self): + events = [ + Event(evt="key", data="o", raw=bytearray(b"o")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="O", raw=bytearray(b"O")), + Event(evt="key", data="_", raw=bytearray(b"_")), + Event(evt="key", data="A", raw=bytearray(b"A")), + Event(evt="key", data="S", raw=bytearray(b"S")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_ASYNC") + + def test_empty_namespace_completion(self): + events = [ + Event(evt="key", data="o", raw=bytearray(b"o")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data=".", raw=bytearray(b".")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="t", raw=bytearray(b"t")), + Event(evt="key", data="e", raw=bytearray(b"e")), + Event(evt="key", data="n", raw=bytearray(b"n")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = [ + Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + +if __name__ == "__main__": + unittest.main() + + if __name__ == "__main__": unittest.main() From 2a64680c5ebeb2770052376e9fb69071a9be0ef3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 14:27:10 +0100 Subject: [PATCH 21/78] Add moar tests --- Lib/test/test_pyrepl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 82eabd42eaef16..d38ad5da708db0 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -370,6 +370,7 @@ def test_empty_namespace_completion(self): def test_global_namespace_completion(self): events = [ Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="y", raw=bytearray(b"y")), Event(evt="key", data="\t", raw=bytearray(b"\t")), Event(evt="key", data="\n", raw=bytearray(b"\n")), ] From 5bced59539ce9c8a607d438944f973a7d21c08e9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 15:46:46 +0100 Subject: [PATCH 22/78] Cleanup and refactor --- Lib/_pyrepl/unix_eventqueue.py | 90 +++++++++++++++++----------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 8c086e87c49076..3e4fa1b46178e7 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,9 +18,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# Bah, this would be easier to test if curses/terminfo didn't have so -# much non-introspectable global state. - from collections import deque from . import keymap @@ -31,7 +28,8 @@ import os -_keynames = { +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { "delete": "kdch1", "down": "kcud1", "end": "kend", @@ -46,20 +44,11 @@ } -#function keys x in 1-20 -> fX: kfX -_keynames.update(('f%d' % i, 'kf%d' % i) for i in range(1, 21)) +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) -# this is a bit of a hack: CTRL-left and CTRL-right are not standardized -# termios sequences: each terminal emulator implements its own slightly -# different incarnation, and as far as I know, there is no way to know -# programmatically which sequences correspond to CTRL-left and -# CTRL-right. In bash, these keys usually work because there are bindings -# in ~/.inputrc, but pyrepl does not support it. The workaround is to -# hard-code here a bunch of known sequences, which will be seen as "ctrl -# left" and "ctrl right" keys, which can be finally be mapped to commands -# by the reader's keymaps. -# -CTRL_ARROW_KEYCODE = { +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { # for xterm, gnome-terminal, xfce terminal, etc. b'\033[1;5D': 'ctrl left', b'\033[1;5C': 'ctrl right', @@ -68,74 +57,87 @@ b'\033Oc': 'ctrl right', } -def general_keycodes(): +def get_terminal_keycodes(): + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ keycodes = {} - for key, tiname in _keynames.items(): - keycode = curses.tigetstr(tiname) + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) trace('key {key} tiname {tiname} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key - keycodes.update(CTRL_ARROW_KEYCODE) + keycodes.update(CTRL_ARROW_KEYCODES) return keycodes - -def EventQueue(fd, encoding): - keycodes = general_keycodes() - if os.isatty(fd): - backspace = tcgetattr(fd)[6][VERASE] - keycodes[backspace] = 'backspace' - k = keymap.compile_keymap(keycodes) - trace('keymap {k!r}', k=k) - return EncodedQueue(k, encoding) - - -class EncodedQueue(object): - def __init__(self, keymap, encoding): - self.k = self.ck = keymap +class EventQueue(object): + def __init__(self, fd, encoding): + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding self.events = deque() self.buf = bytearray() - self.encoding = encoding def get(self): + """ + Retrieves the next event from the queue. + """ if self.events: return self.events.popleft() else: return None def empty(self): + """ + Checks if the queue is empty. + """ return not self.events def flush_buf(self): + """ + Flushes the buffer and returns its contents. + """ old = self.buf self.buf = bytearray() return old def insert(self, event): + """ + Inserts an event into the queue. + """ trace('added event {event}', event=event) self.events.append(event) def push(self, char): + """ + Processes a character by updating the buffer and handling special key mappings. + """ ord_char = char if isinstance(char, int) else ord(char) char = bytes(bytearray((ord_char,))) self.buf.append(ord_char) - if char in self.k: - if self.k is self.ck: + if char in self.keymap: + if self.keymap is self.compiled_keymap: #sanity check, buffer is empty when a special key comes assert len(self.buf) == 1 - k = self.k[char] + k = self.keymap[char] trace('found map {k!r}', k=k) if isinstance(k, dict): - self.k = k + self.keymap = k else: self.insert(Event('key', k, self.flush_buf())) - self.k = self.ck + self.keymap = self.compiled_keymap elif self.buf and self.buf[0] == 27: # escape # escape sequence not recognized by our keymap: propagate it # outside so that i can be recognized as an M-... key (see also - # the docstring in keymap.py, in particular the line \\E. + # the docstring in keymap.py trace('unrecognized escape sequence, propagating...') - self.k = self.ck + self.keymap = self.compiled_keymap self.insert(Event('key', '\033', bytearray(b'\033'))) for c in self.flush_buf()[1:]: self.push(chr(c)) @@ -147,4 +149,4 @@ def push(self, char): return else: self.insert(Event('key', decoded, self.flush_buf())) - self.k = self.ck + self.keymap = self.compiled_keymap From 20533d3b53d46842f019881d2589b9fda2621c4a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 16:04:00 +0100 Subject: [PATCH 23/78] Refactor unix_console.py --- Lib/_pyrepl/unix_console.py | 529 ++++++++++++++++++++++-------------- 1 file changed, 326 insertions(+), 203 deletions(-) diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index b1289632c88587..cad4bde22d4541 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -62,27 +62,47 @@ def _my_getstr(cap, optional=0): # Add (possibly) missing baudrates (check termios man page) to termios + def add_supported_baudrates(dictionary, rate): baudrate_name = "B%d" % rate if hasattr(termios, baudrate_name): dictionary[getattr(termios, baudrate_name)] = rate + # Check the termios man page (Line speed) to know where these # values come from. supported_baudrates = [ - 0, 110, 115200, 1200, 134, 150, 1800, 19200, 200, 230400, - 2400, 300, 38400, 460800, 4800, 50, 57600, 600, 75, 9600 + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, ] ratedict = {} for rate in supported_baudrates: add_supported_baudrates(ratedict, rate) -# ------------ end of baudrate definitions ------------ - # Clean up variables to avoid unintended usage del rate, add_supported_baudrates +# ------------ end of baudrate definitions ------------ + delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") try: @@ -90,7 +110,7 @@ def add_supported_baudrates(dictionary, rate): except AttributeError: # this is exactly the minumum necessary to support what we # do with poll objects - class poll: + class MinimalPoll: def __init__(self): pass @@ -101,12 +121,23 @@ def poll(self): # note: a 'timeout' argument would be *milliseconds* r, w, e = select.select([self.fd], [], []) return r + poll = MinimalPoll + POLLIN = getattr(select, "POLLIN", None) class UnixConsole(Console): def __init__(self, f_in=0, f_out=1, term=None, encoding=None): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ if encoding is None: encoding = sys.getdefaultencoding() @@ -152,47 +183,28 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): self._rmkx = _my_getstr("rmkx", 1) self._smkx = _my_getstr("smkx", 1) - ## work out how we're going to sling the cursor around - if 0 and self._hpa: # hpa don't work in windows telnet :-( - self.__move_x = self.__move_x_hpa - elif self._cub and self._cuf: - self.__move_x = self.__move_x_cub_cuf - elif self._cub1 and self._cuf1: - self.__move_x = self.__move_x_cub1_cuf1 - else: - raise RuntimeError("insufficient terminal (horizontal)") - - if self._cuu and self._cud: - self.__move_y = self.__move_y_cuu_cud - elif self._cuu1 and self._cud1: - self.__move_y = self.__move_y_cuu1_cud1 - else: - raise RuntimeError("insufficient terminal (vertical)") - - if self._dch1: - self.dch1 = self._dch1 - elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) - else: - self.dch1 = None - - if self._ich1: - self.ich1 = self._ich1 - elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) - else: - self.ich1 = None - - self.__move = self.__move_short + self.__setup_movement() self.event_queue = EventQueue(self.input_fd, self.encoding) self.cursor_visible = 1 - def change_encoding(self, encoding): + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ self.encoding = encoding def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ cx, cy = c_xy if not self.__gone_tall: while len(self.screen) < min(len(screen), self.height): @@ -268,114 +280,14 @@ def refresh(self, screen, c_xy): self.move_cursor(cx, cy) self.flushoutput() - def __write_changed_line(self, y, oldline, newline, px): - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) - x = 0 - minlen = min(len(oldline), len(newline)) - # - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequene - while x < minlen and oldline[x] == newline[x] and newline[x] != "\x1b": - x += 1 - if oldline[x:] == newline[x + 1 :] and self.ich1: - if ( - y == self.__posxy[1] - and x > self.__posxy[0] - and oldline[px:x] == newline[px + 1 : x + 1] - ): - x = px - self.__move(x, y) - self.__write_code(self.ich1) - self.__write(newline[x]) - self.__posxy = x + 1, y - elif x < minlen and oldline[x + 1 :] == newline[x + 1 :]: - self.__move(x, y) - self.__write(newline[x]) - self.__posxy = x + 1, y - elif ( - self.dch1 - and self.ich1 - and len(newline) == self.width - and x < len(newline) - 2 - and newline[x + 1 : -1] == oldline[x:-2] - ): - self.__hide_cursor() - self.__move(self.width - 2, y) - self.__posxy = self.width - 2, y - self.__write_code(self.dch1) - self.__move(x, y) - self.__write_code(self.ich1) - self.__write(newline[x]) - self.__posxy = x + 1, y - else: - self.__hide_cursor() - self.__move(x, y) - if len(oldline) > len(newline): - self.__write_code(self._el) - self.__write(newline[x:]) - self.__posxy = len(newline), y - - if "\x1b" in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) - - def __write(self, text): - self.__buffer.append((text, 0)) - - def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) - - def __maybe_write_code(self, fmt, *args): - if fmt: - self.__write_code(fmt, *args) - - def __move_y_cuu1_cud1(self, y): - dy = y - self.__posxy[1] - if dy > 0: - self.__write_code(dy * self._cud1) - elif dy < 0: - self.__write_code((-dy) * self._cuu1) - - def __move_y_cuu_cud(self, y): - dy = y - self.__posxy[1] - if dy > 0: - self.__write_code(self._cud, dy) - elif dy < 0: - self.__write_code(self._cuu, -dy) - - def __move_x_hpa(self, x): - if x != self.__posxy[0]: - self.__write_code(self._hpa, x) - - def __move_x_cub1_cuf1(self, x): - dx = x - self.__posxy[0] - if dx > 0: - self.__write_code(self._cuf1 * dx) - elif dx < 0: - self.__write_code(self._cub1 * (-dx)) - - def __move_x_cub_cuf(self, x): - dx = x - self.__posxy[0] - if dx > 0: - self.__write_code(self._cuf, dx) - elif dx < 0: - self.__write_code(self._cub, -dx) - - def __move_short(self, x, y): - self.__move_x(x) - self.__move_y(y) - - def __move_tall(self, x, y): - assert 0 <= y - self.__offset < self.height, y - self.__offset - self.__write_code(self._cup, y - self.__offset, x) - def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ if y < self.__offset or y >= self.__offset + self.height: self.event_queue.insert(Event("scroll", None)) else: @@ -384,7 +296,9 @@ def move_cursor(self, x, y): self.flushoutput() def prepare(self): - # per-readline preparations: + """ + Prepare the console for input/output operations. + """ self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) @@ -416,6 +330,9 @@ def prepare(self): pass def restore(self): + """ + Restore the console to the default state + """ self.__maybe_write_code(self._rmkx) self.flushoutput() tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) @@ -424,17 +341,28 @@ def restore(self): signal.signal(signal.SIGWINCH, self.old_sigwinch) del self.old_sigwinch - def __sigwinch(self, signum, frame): - self.height, self.width = self.getheightwidth() - self.event_queue.insert(Event("resize", None)) - def push_char(self, char): + """ + Push a character to the console event queue. + + Parameters: + - char (str): Character to push. + """ trace("push char {char!r}", char=char) self.event_queue.push(char) - def get_event(self, block=1): + def get_event(self, block=True): + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ while self.event_queue.empty(): - while 1: + while True: try: self.push_char(os.read(self.input_fd, 1)) except OSError as err: @@ -452,39 +380,32 @@ def get_event(self, block=1): return self.event_queue.get() def wait(self): + """ + Wait for events on the console. + """ self.pollob.poll() - def set_cursor_vis(self, vis): - if vis: + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: self.__show_cursor() else: self.__hide_cursor() - def __hide_cursor(self): - if self.cursor_visible: - self.__maybe_write_code(self._civis) - self.cursor_visible = 0 - - def __show_cursor(self): - if not self.cursor_visible: - self.__maybe_write_code(self._cnorm) - self.cursor_visible = 1 - - def repaint_prep(self): - if not self.__gone_tall: - self.__posxy = 0, self.__posxy[1] - self.__write("\r") - ns = len(self.screen) * ["\000" * self.width] - self.screen = ns - else: - self.__posxy = 0, self.__offset - self.__move(0, self.__offset) - ns = self.height * ["\000" * self.width] - self.screen = ns - if TIOCGWINSZ: def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ try: return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) except KeyError: @@ -498,15 +419,27 @@ def getheightwidth(self): else: def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ try: return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) except KeyError: return 25, 80 def forgetinput(self): + """ + Discard any pending input on the console. + """ termios.tcflush(self.input_fd, termios.TCIFLUSH) def flushoutput(self): + """ + Flush the output buffer. + """ for text, iscode in self.__buffer: if iscode: self.__tputs(text) @@ -514,34 +447,10 @@ def flushoutput(self): os.write(self.output_fd, text.encode(self.encoding, "replace")) del self.__buffer[:] - def __tputs(self, fmt, prog=delayprog): - """A Python implementation of the curses tputs function; the - curses one can't really be wrapped in a sane manner. - - I have the strong suspicion that this is complexity that - will never do anyone any good.""" - # using .get() means that things will blow up - # only if the bps is actually needed (which I'm - # betting is pretty unlkely) - bps = ratedict.get(self.__svtermstate.ospeed) - while 1: - m = prog.search(fmt) - if not m: - os.write(self.output_fd, fmt) - break - x, y = m.span() - os.write(self.output_fd, fmt[:x]) - fmt = fmt[y:] - delay = int(m.group(1)) - if b"*" in m.group(2): - delay *= self.height - if self._pad: - nchars = (bps * delay) / 1000 - os.write(self.output_fd, self._pad * nchars) - else: - time.sleep(float(delay) / 1000.0) - def finish(self): + """ + Finish console operations and flush the output buffer. + """ y = len(self.screen) - 1 while y >= 0 and not self.screen[y]: y -= 1 @@ -550,12 +459,21 @@ def finish(self): self.flushoutput() def beep(self): + """ + Emit a beep sound. + """ self.__maybe_write_code(self._bel) self.flushoutput() if FIONREAD: def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ e = Event("key", "", b"") while not self.event_queue.empty(): @@ -573,6 +491,12 @@ def getpending(self): else: def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ e = Event("key", "", b"") while not self.event_queue.empty(): @@ -588,8 +512,207 @@ def getpending(self): return e def clear(self): + """ + Clear the console screen. + """ self.__write_code(self._clear) self.__gone_tall = 1 self.__move = self.__move_tall self.__posxy = 0, 0 self.screen = [] + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + x = 0 + minlen = min(len(oldline), len(newline)) + # + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while x < minlen and oldline[x] == newline[x] and newline[x] != "\x1b": + x += 1 + if oldline[x:] == newline[x + 1 :] and self.ich1: + if ( + y == self.__posxy[1] + and x > self.__posxy[0] + and oldline[px:x] == newline[px + 1 : x + 1] + ): + x = px + self.__move(x, y) + self.__write_code(self.ich1) + self.__write(newline[x]) + self.__posxy = x + 1, y + elif x < minlen and oldline[x + 1 :] == newline[x + 1 :]: + self.__move(x, y) + self.__write(newline[x]) + self.__posxy = x + 1, y + elif ( + self.dch1 + and self.ich1 + and len(newline) == self.width + and x < len(newline) - 2 + and newline[x + 1 : -1] == oldline[x:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.__posxy = self.width - 2, y + self.__write_code(self.dch1) + self.__move(x, y) + self.__write_code(self.ich1) + self.__write(newline[x]) + self.__posxy = x + 1, y + else: + self.__hide_cursor() + self.__move(x, y) + if len(oldline) > len(newline): + self.__write_code(self._el) + self.__write(newline[x:]) + self.__posxy = len(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x): + if x != self.__posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint_prep(self): + if not self.__gone_tall: + self.__posxy = 0, self.__posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.__posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) From a07a0ce85a1fa7e0b8f1e6b7d4d77686bd166126 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 16:06:26 +0100 Subject: [PATCH 24/78] Refactor unix_console.py more --- Lib/_pyrepl/unix_console.py | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index cad4bde22d4541..72dfd378e5a1fb 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -159,29 +159,29 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): self.term = term self._bel = _my_getstr("bel") - self._civis = _my_getstr("civis", optional=1) + self._civis = _my_getstr("civis", optional=True) self._clear = _my_getstr("clear") - self._cnorm = _my_getstr("cnorm", optional=1) - self._cub = _my_getstr("cub", optional=1) - self._cub1 = _my_getstr("cub1", 1) - self._cud = _my_getstr("cud", 1) - self._cud1 = _my_getstr("cud1", 1) - self._cuf = _my_getstr("cuf", 1) - self._cuf1 = _my_getstr("cuf1", 1) + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) self._cup = _my_getstr("cup") - self._cuu = _my_getstr("cuu", 1) - self._cuu1 = _my_getstr("cuu1", 1) - self._dch1 = _my_getstr("dch1", 1) - self._dch = _my_getstr("dch", 1) + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) self._el = _my_getstr("el") - self._hpa = _my_getstr("hpa", 1) - self._ich = _my_getstr("ich", 1) - self._ich1 = _my_getstr("ich1", 1) - self._ind = _my_getstr("ind", 1) - self._pad = _my_getstr("pad", 1) - self._ri = _my_getstr("ri", 1) - self._rmkx = _my_getstr("rmkx", 1) - self._smkx = _my_getstr("smkx", 1) + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) self.__setup_movement() From b8b0e768eb5f5f6529439466fa8400da1d5a3307 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 16:07:28 +0100 Subject: [PATCH 25/78] Refactor unix_console.py more --- Lib/_pyrepl/unix_console.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 72dfd378e5a1fb..d4523c5cefa579 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -48,16 +48,6 @@ class InvalidTerminal(RuntimeError): FIONREAD = getattr(termios, "FIONREAD", None) TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) - -def _my_getstr(cap, optional=0): - r = curses.tigetstr(cap) - if not optional and r is None: - raise InvalidTerminal( - "terminal doesn't have the required '%s' capability" % cap - ) - return r - - # ------------ start of baudrate definitions ------------ # Add (possibly) missing baudrates (check termios man page) to termios @@ -158,6 +148,14 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): curses.setupterm(term, self.output_fd) self.term = term + def _my_getstr(cap, optional=0): + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + self._bel = _my_getstr("bel") self._civis = _my_getstr("civis", optional=True) self._clear = _my_getstr("clear") From c57017116b0ac3af2bdb9b85a0af00402aaf7e22 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 17:05:41 +0100 Subject: [PATCH 26/78] Moar tests --- Lib/_pyrepl/unix_eventqueue.py | 2 +- Lib/test/test_pyrepl.py | 124 ++++++++++++++++++++++++++++++--- 2 files changed, 114 insertions(+), 12 deletions(-) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 3e4fa1b46178e7..73e133de2dcab1 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -70,7 +70,7 @@ def get_terminal_keycodes(): keycodes.update(CTRL_ARROW_KEYCODES) return keycodes -class EventQueue(object): +class EventQueue: def __init__(self, fd, encoding): self.keycodes = get_terminal_keycodes() if os.isatty(fd): diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index d38ad5da708db0..fb25bebc49c014 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -1,17 +1,19 @@ -from unittest import TestCase -from unittest.mock import MagicMock -from unittest.mock import call +import curses +import os +import rlcompleter +import sys from code import InteractiveConsole from contextlib import suppress from functools import partial -import rlcompleter -import os +from io import BytesIO +from unittest import TestCase +from unittest.mock import MagicMock, call, patch -from _pyrepl.readline import ReadlineAlikeReader -from _pyrepl.readline import ReadlineConfig +import _pyrepl.unix_eventqueue as unix_eventqueue +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.simple_interact import _strip_final_indent -from _pyrepl.console import Event -from _pyrepl.console import Console +from _pyrepl.unix_eventqueue import EventQueue def more_lines(unicodetext, namespace=None): @@ -382,8 +384,108 @@ def test_global_namespace_completion(self): self.assertEqual(output, "python") -if __name__ == "__main__": - unittest.main() +class TestUnivEventQueue(TestCase): + def setUp(self) -> None: + curses.setupterm() + return super().setUp() + + def test_get(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") if __name__ == "__main__": From 169043f551c7d7ac5bb84fc70c3600ff4d263145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 18:08:28 +0200 Subject: [PATCH 27/78] Fix test___all__ --- Lib/_pyrepl/reader.py | 6 ------ Lib/_pyrepl/readline.py | 7 ++++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 81e3359626193d..1fa3ecf5edaafc 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -55,12 +55,6 @@ def _my_unctrl(c, u=_make_unctrl_map()): return c -if "a"[0] == b"a": - # When running tests with python2, bytes characters are bytes. - def _my_unctrl(c, uc=_my_unctrl): - return uc(ord(c)) - - def disp_str(buffer, join="".join, uc=_my_unctrl): """disp_str(buffer:string) -> (string, [int]) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index dc89d7504c2ef5..693a1cb1a23394 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -40,6 +40,7 @@ __all__ = [ "add_history", "clear_history", + "copy_history", "get_begidx", "get_completer", "get_completer_delims", @@ -51,15 +52,15 @@ "insert_text", "parse_and_bind", "read_history_file", - "read_init_file", - "redisplay", + # "read_init_file", + # "redisplay", "remove_history_item", "replace_history_item", "set_auto_history", "set_completer", "set_completer_delims", "set_history_length", - "set_pre_input_hook", + # "set_pre_input_hook", "set_startup_hook", "write_history_file", # ---- multiline extensions ---- From c613ae32d8a370e5a03e65c311d70f8c5c021152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 28 Apr 2024 18:19:10 +0200 Subject: [PATCH 28/78] Fix test_traceback --- Lib/test/test_traceback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 8927fccc289320..c8427b1fd56753 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -507,7 +507,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -546,7 +546,7 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=, tb=, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), From 98fbee29117a4a4185b52490e8204e7ef7ac0d42 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 17:22:24 +0100 Subject: [PATCH 29/78] Fix weird r-search scrambled text due to color codes --- Lib/_pyrepl/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 1fa3ecf5edaafc..c30a941e91b063 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -335,6 +335,8 @@ def process_prompt(self, prompt): keep = prompt[pos:] l -= sum(map(len, _r_csi_seq.findall(keep))) out_prompt += keep + if traceback._can_colorize(): + out_prompt = traceback._ANSIColors.BOLD_MAGENTA + out_prompt + traceback._ANSIColors.RESET return out_prompt, l def bow(self, p=None): @@ -423,8 +425,6 @@ def get_prompt(self, lineno, cursor_on_line): if self.paste_mode: res= '(paste) ' - if traceback._can_colorize(): - res = traceback._ANSIColors.BOLD_MAGENTA + res + traceback._ANSIColors.RESET # Lazily call str() on self.psN, and cache the results using as key # the object on which str() was called. This ensures that even if the # same object is used e.g. for ps1 and ps2, str() is called only once. From 254aaf25001defba6c2271a9bcdaa20256ff8505 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 17:24:28 +0100 Subject: [PATCH 30/78] Fix weird r-search scrambled text due to color codes better --- Lib/_pyrepl/reader.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index c30a941e91b063..44d9a580ba344d 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -55,6 +55,12 @@ def _my_unctrl(c, u=_make_unctrl_map()): return c +if "a"[0] == b"a": + # When running tests with python2, bytes characters are bytes. + def _my_unctrl(c, uc=_my_unctrl): + return uc(ord(c)) + + def disp_str(buffer, join="".join, uc=_my_unctrl): """disp_str(buffer:string) -> (string, [int]) @@ -283,6 +289,8 @@ def calc_screen(self): screeninfo.append((0, [])) p -= ll + 1 prompt, lp = self.process_prompt(prompt) + if traceback._can_colorize(): + prompt = traceback._ANSIColors.BOLD_MAGENTA + prompt + traceback._ANSIColors.RESET l, l2 = disp_str(line) wrapcount = (len(l) + lp) // w if wrapcount == 0: @@ -335,8 +343,6 @@ def process_prompt(self, prompt): keep = prompt[pos:] l -= sum(map(len, _r_csi_seq.findall(keep))) out_prompt += keep - if traceback._can_colorize(): - out_prompt = traceback._ANSIColors.BOLD_MAGENTA + out_prompt + traceback._ANSIColors.RESET return out_prompt, l def bow(self, p=None): From cf9bd265b8a020c4300fcce715f37937eddbe2f3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 28 Apr 2024 17:37:46 +0100 Subject: [PATCH 31/78] Disgusting fix for help state restoration --- Lib/_pyrepl/commands.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index cad8785d6b6822..d53d0e3d9640f0 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -400,7 +400,13 @@ class help(Command): def do(self): import _sitebuiltins self.reader.console.restore() + backup_ps = {} + for ps in ("ps1", "ps2", "ps3", "ps4"): + backup_ps[ps] = getattr(self.reader, ps) self.reader.msg = _sitebuiltins._Helper()() + for ps in ("ps1", "ps2", "ps3", "ps4"): + setattr(self.reader, ps, backup_ps[ps]) + self.reader.prepare() self.reader.dirty = 1 From b064e1f73d65586ec6d3cb1916f0755e91ef8e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Sun, 28 Apr 2024 23:47:39 +0200 Subject: [PATCH 32/78] Test pasting with/without paste mode --- Lib/test/test_pyrepl.py | 59 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index fb25bebc49c014..e29dc0bf90c9b6 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -1,4 +1,5 @@ import curses +import itertools import os import rlcompleter import sys @@ -379,7 +380,6 @@ def test_global_namespace_completion(self): namespace = {"python": None} reader = self.prepare_reader(events, namespace) - output = multiline_input(reader, namespace) self.assertEqual(output, "python") @@ -488,5 +488,62 @@ def test_push_unrecognized_escape_sequence(self): self.assertEqual(eq.events[2].data, "Z") +class TestPasteEvent(TestCase): + def setUp(self) -> None: + curses.setupterm() + return super().setUp() + + def prepare_reader(self, events): + console = FakeConsole(events) + reader = ReadlineAlikeReader(console) + reader.config = ReadlineConfig() + reader.config.readline_completer = None + return reader + + def code_to_events(self, code): + for c in code: + yield Event(evt='key', data=c, raw=bytearray(c.encode('utf-8'))) + + def test_paste(self): + code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n\n' + ) + + events = itertools.chain([ + Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), + ], self.code_to_events(code)) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code[:-1]) + + def test_paste_not_in_paste_mode(self): + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n\n' + ) + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:' + ) + + events = self.code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + if __name__ == "__main__": unittest.main() From 0fa01e0c0cb93155f460abc7056ec16b47e2a88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Mon, 29 Apr 2024 00:16:38 +0200 Subject: [PATCH 33/78] refactor events --- Lib/test/test_pyrepl.py | 194 ++++++++-------------------------------- 1 file changed, 36 insertions(+), 158 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index e29dc0bf90c9b6..4eac770fa5be2b 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -42,6 +42,11 @@ def multiline_input(reader, namespace=None): reader.paste_mode = False +def code_to_events(code): + for c in code: + yield Event(evt='key', data=c, raw=bytearray(c.encode('utf-8'))) + + class FakeConsole(Console): def __init__(self, events, encoding="utf-8"): self.events = iter(events) @@ -63,25 +68,14 @@ def prepare_reader(self, events): return reader, console def test_up_arrow(self): - events = [ - Event(evt="key", data="d", raw=bytearray(b"d")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="f", raw=bytearray(b"f")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data="f", raw=bytearray(b"f")), - Event(evt="key", data="(", raw=bytearray(b"(")), - Event(evt="key", data=")", raw=bytearray(b")")), - Event(evt="key", data=":", raw=bytearray(b":")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), + code = ( + 'def f():\n' + ' ...\n' + ) + events = itertools.chain(code_to_events(code), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader, console = self.prepare_reader(events) @@ -91,25 +85,14 @@ def test_up_arrow(self): console.move_cursor.assert_called_with(1, 3) def test_down_arrow(self): - events = [ - Event(evt="key", data="d", raw=bytearray(b"d")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="f", raw=bytearray(b"f")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data="f", raw=bytearray(b"f")), - Event(evt="key", data="(", raw=bytearray(b"(")), - Event(evt="key", data=")", raw=bytearray(b")")), - Event(evt="key", data=":", raw=bytearray(b":")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), + code = ( + 'def f():\n' + ' ...\n' + ) + events = itertools.chain(code_to_events(code), [ Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader, console = self.prepare_reader(events) @@ -119,15 +102,10 @@ def test_down_arrow(self): console.move_cursor.assert_called_with(1, 5) def test_left_arrow(self): - events = [ - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="1", raw=bytearray(b"1")), + events = itertools.chain(code_to_events('11+11'), [ Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader, console = self.prepare_reader(events) @@ -140,15 +118,10 @@ def test_left_arrow(self): ) def test_right_arrow(self): - events = [ - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="1", raw=bytearray(b"1")), + events = itertools.chain(code_to_events('11+11'), [ Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader, console = self.prepare_reader(events) @@ -170,35 +143,13 @@ def prepare_reader(self, events): return reader def test_basic(self): - events = [ - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] - reader = self.prepare_reader(events) + reader = self.prepare_reader(code_to_events('1+1\n')) output = multiline_input(reader) self.assertEqual(output, "1+1") def test_multiline_edit(self): - events = [ - Event(evt="key", data="d", raw=bytearray(b"d")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="f", raw=bytearray(b"f")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data="f", raw=bytearray(b"f")), - Event(evt="key", data="(", raw=bytearray(b"(")), - Event(evt="key", data=")", raw=bytearray(b")")), - Event(evt="key", data=":", raw=bytearray(b":")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data=" ", raw=bytearray(b" ")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), + events = itertools.chain(code_to_events('def f():\n ...\n\n'), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), @@ -210,7 +161,7 @@ def test_multiline_edit(self): Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader = self.prepare_reader(events) output = multiline_input(reader) @@ -219,22 +170,14 @@ def test_multiline_edit(self): self.assertEqual(output, "def g():\n ...\n ") def test_history_navigation_with_up_arrow(self): - events = [ - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="2", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="2", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), + events = itertools.chain(code_to_events('1+1\n2+2\n'), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="\n", raw=bytearray(b"\n")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader = self.prepare_reader(events) @@ -248,21 +191,13 @@ def test_history_navigation_with_up_arrow(self): self.assertEqual(output, "1+1") def test_history_navigation_with_down_arrow(self): - events = [ - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="2", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="2", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), + events = itertools.chain(code_to_events('1+1\n2+2\n'), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="\n", raw=bytearray(b"\n")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ] + ]) reader = self.prepare_reader(events) @@ -270,24 +205,12 @@ def test_history_navigation_with_down_arrow(self): self.assertEqual(output, "1+1") def test_history_search(self): - events = [ - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="2", raw=bytearray(b"2")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="2", raw=bytearray(b"2")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="3", raw=bytearray(b"3")), - Event(evt="key", data="+", raw=bytearray(b"+")), - Event(evt="key", data="3", raw=bytearray(b"3")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), + events = itertools.chain(code_to_events('1+1\n2+2\n3+3\n'), [ Event(evt="key", data="\x12", raw=bytearray(b"\x12")), Event(evt="key", data="1", raw=bytearray(b"1")), Event(evt="key", data="\n", raw=bytearray(b"\n")), Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + ]) reader = self.prepare_reader(events) @@ -310,18 +233,7 @@ def prepare_reader(self, events, namespace): return reader def test_simple_completion(self): - events = [ - Event(evt="key", data="o", raw=bytearray(b"o")), - Event(evt="key", data="s", raw=bytearray(b"s")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data="g", raw=bytearray(b"g")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="t", raw=bytearray(b"t")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="n", raw=bytearray(b"n")), - Event(evt="key", data="\t", raw=bytearray(b"\t")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + events = code_to_events('os.geten\t\n') namespace = {"os": os} reader = self.prepare_reader(events, namespace) @@ -330,19 +242,7 @@ def test_simple_completion(self): self.assertEqual(output, "os.getenv") def test_completion_with_many_options(self): - events = [ - Event(evt="key", data="o", raw=bytearray(b"o")), - Event(evt="key", data="s", raw=bytearray(b"s")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data="\t", raw=bytearray(b"\t")), - Event(evt="key", data="\t", raw=bytearray(b"\t")), - Event(evt="key", data="O", raw=bytearray(b"O")), - Event(evt="key", data="_", raw=bytearray(b"_")), - Event(evt="key", data="A", raw=bytearray(b"A")), - Event(evt="key", data="S", raw=bytearray(b"S")), - Event(evt="key", data="\t", raw=bytearray(b"\t")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] + events = code_to_events('os.\t\tO_AS\t\n') namespace = {"os": os} reader = self.prepare_reader(events, namespace) @@ -351,19 +251,7 @@ def test_completion_with_many_options(self): self.assertEqual(output, "os.O_ASYNC") def test_empty_namespace_completion(self): - events = [ - Event(evt="key", data="o", raw=bytearray(b"o")), - Event(evt="key", data="s", raw=bytearray(b"s")), - Event(evt="key", data=".", raw=bytearray(b".")), - Event(evt="key", data="g", raw=bytearray(b"g")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="t", raw=bytearray(b"t")), - Event(evt="key", data="e", raw=bytearray(b"e")), - Event(evt="key", data="n", raw=bytearray(b"n")), - Event(evt="key", data="\t", raw=bytearray(b"\t")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] - + events = code_to_events('os.geten\t\n') namespace = {} reader = self.prepare_reader(events, namespace) @@ -371,13 +259,7 @@ def test_empty_namespace_completion(self): self.assertEqual(output, "os.geten") def test_global_namespace_completion(self): - events = [ - Event(evt="key", data="p", raw=bytearray(b"p")), - Event(evt="key", data="y", raw=bytearray(b"y")), - Event(evt="key", data="\t", raw=bytearray(b"\t")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ] - + events = code_to_events('py\t\n') namespace = {"python": None} reader = self.prepare_reader(events, namespace) output = multiline_input(reader, namespace) @@ -500,10 +382,6 @@ def prepare_reader(self, events): reader.config.readline_completer = None return reader - def code_to_events(self, code): - for c in code: - yield Event(evt='key', data=c, raw=bytearray(c.encode('utf-8'))) - def test_paste(self): code = ( 'def a():\n' @@ -516,7 +394,7 @@ def test_paste(self): events = itertools.chain([ Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), - ], self.code_to_events(code)) + ], code_to_events(code)) reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, code[:-1]) @@ -539,7 +417,7 @@ def test_paste_not_in_paste_mode(self): ' else:' ) - events = self.code_to_events(input_code) + events = code_to_events(input_code) reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, output_code) From a6d54e65c8a030ff01ad5f43fb2cbff5656625ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 29 Apr 2024 00:25:43 +0200 Subject: [PATCH 34/78] help() uses its own history, doesn't pollute main history --- Lib/_pyrepl/commands.py | 20 +++++---------- Lib/_pyrepl/console.py | 1 + Lib/_pyrepl/historical_reader.py | 12 +++++++++ Lib/_pyrepl/reader.py | 18 ++++++++++++-- Lib/_pyrepl/readline.py | 5 ---- Lib/_pyrepl/simple_interact.py | 42 ++++++++++++++++++++++++-------- 6 files changed, 67 insertions(+), 31 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index d53d0e3d9640f0..2e22c62d7a9ebf 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -399,15 +399,8 @@ def do(self): class help(Command): def do(self): import _sitebuiltins - self.reader.console.restore() - backup_ps = {} - for ps in ("ps1", "ps2", "ps3", "ps4"): - backup_ps[ps] = getattr(self.reader, ps) - self.reader.msg = _sitebuiltins._Helper()() - for ps in ("ps1", "ps2", "ps3", "ps4"): - setattr(self.reader, ps, backup_ps[ps]) - self.reader.prepare() - self.reader.dirty = 1 + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() class invalid_key(Command): @@ -454,13 +447,12 @@ def do(self): class show_history(Command): def do(self): - # self.reader.console.restore() from _pyrepl.pager import get_pager - from _pyrepl.readline import copy_history from site import gethistoryfile - pager = get_pager() - pager(os.linesep.join(copy_history()), gethistoryfile()) - self.reader.dirty = 1 + history = os.linesep.join(self.reader.history[:]) + with self.reader.suspend(): + pager = get_pager() + pager(history, gethistoryfile()) class paste_mode(Command): diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 386203edfb5d28..17e66bbd3ba77f 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,6 +19,7 @@ import dataclasses + @dataclasses.dataclass class Event: evt: str diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index 65a98af3743800..ddfac903782047 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -17,6 +17,8 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from contextlib import contextmanager + from . import commands, input from .reader import Reader as R @@ -264,6 +266,16 @@ def get_item(self, i): else: return self.transient_history.get(i, self.get_unicode()) + @contextmanager + def suspend(self): + with super().suspend(): + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + def prepare(self): super().prepare() try: diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 44d9a580ba344d..d7d1ce9b4cff14 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -19,6 +19,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from contextlib import contextmanager import re import unicodedata import traceback @@ -502,8 +503,9 @@ def prepare(self): self.pos = 0 self.dirty = 1 self.last_command = None + # XXX should kill ring be here? self._pscache = {} - except: + except BaseException: self.restore() raise @@ -516,6 +518,19 @@ def restore(self): """Clean up after a run.""" self.console.restore() + @contextmanager + def suspend(self): + """A context manager to delegate to another reader.""" + prev_state = dict(self.__dict__) + try: + self.restore() + yield + finally: + for arg in ('msg', 'ps1', 'ps2', 'ps3', 'ps4', 'paste_mode'): + setattr(self, arg, prev_state[arg]) + self.prepare() + pass + def finish(self): """Called when a command signals that we're finished.""" pass @@ -537,7 +552,6 @@ def refresh(self): self.dirty = 0 # forgot this for a while (blush) def do_cmd(self, cmd): - # print cmd if isinstance(cmd[0], str): cmd = self.commands.get(cmd[0], commands.invalid_command)(self, *cmd) elif isinstance(cmd[0], type): diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 693a1cb1a23394..e7a2e738686b85 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -40,7 +40,6 @@ __all__ = [ "add_history", "clear_history", - "copy_history", "get_begidx", "get_completer", "get_completer_delims", @@ -354,9 +353,6 @@ def write_history_file(self, filename=gethistoryfile()): entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") - def copy_history(self): - return self.get_reader().history[:] - def clear_history(self): del self.get_reader().history[:] @@ -427,7 +423,6 @@ def insert_text(self, text): read_history_file = _wrapper.read_history_file write_history_file = _wrapper.write_history_file clear_history = _wrapper.clear_history -copy_history = _wrapper.copy_history get_history_item = _wrapper.get_history_item remove_history_item = _wrapper.remove_history_item replace_history_item = _wrapper.replace_history_item diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index ab963760753db4..b79b71b926b1c3 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -29,6 +29,7 @@ import code import traceback +from .console import Event from .readline import _error, _get_reader, multiline_input @@ -55,13 +56,14 @@ def _strip_final_indent(text): "exit": _sitebuiltins.Quitter('exit', ''), "quit": _sitebuiltins.Quitter('quit' ,''), "copyright": _sitebuiltins._Printer('copyright', sys.copyright), - "help": _sitebuiltins._Helper(), + "help": "help", } class InteractiveColoredConsole(code.InteractiveConsole): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.can_colorize = traceback._can_colorize() + def showtraceback(self): super().showtraceback(colorize=self.can_colorize) @@ -77,6 +79,29 @@ def run_multiline_interactive_console(mainmodule=None, future_flags=0): input_n = 0 + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + command() + return True + + if isinstance(command, str): + # Internal readline commands require a prepared reader like + # inside multiline_input. + reader.prepare() + reader.refresh() + reader.do_cmd([command, Event(evt=command, data=command)]) + reader.restore() + return True + + return False + def more_lines(unicodetext): # ooh, look at the hack: src = _strip_final_indent(unicodetext) @@ -93,6 +118,7 @@ def more_lines(unicodetext): sys.stdout.flush() except Exception: pass + ps1 = getattr(sys, "ps1", ">>> ") ps2 = getattr(sys, "ps2", "... ") try: @@ -100,17 +126,13 @@ def more_lines(unicodetext): except EOFError: break + if maybe_run_command(statement): + continue + input_name = f"" linecache._register_code(input_name, statement, "") - stripped_statement = statement.strip() - maybe_repl_command = REPL_COMMANDS.get(stripped_statement) - if maybe_repl_command is not None and stripped_statement not in console.locals: - _get_reader().history.pop() - maybe_repl_command() - continue - else: - more = console.push(_strip_final_indent(statement), filename=input_name) - assert not more + more = console.push(_strip_final_indent(statement), filename=input_name) + assert not more input_n += 1 except KeyboardInterrupt: console.write("\nKeyboardInterrupt\n") From ffbf24b8ac0f836b1a11f05514c4fbdd578087dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 29 Apr 2024 17:58:14 +0200 Subject: [PATCH 35/78] Add typing to reader.py --- Lib/_pyrepl/historical_reader.py | 18 +- Lib/_pyrepl/mypy.ini | 29 +++ Lib/_pyrepl/reader.py | 305 ++++++++++++++----------------- Lib/_pyrepl/readline.py | 9 +- Lib/_pyrepl/simple_interact.py | 5 +- Lib/_pyrepl/types.py | 6 + Lib/test/test_pyrepl.py | 2 +- 7 files changed, 178 insertions(+), 196 deletions(-) create mode 100644 Lib/_pyrepl/mypy.ini create mode 100644 Lib/_pyrepl/types.py diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index ddfac903782047..380fc63ce1a435 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -258,7 +258,7 @@ def select_item(self, i): self.buffer = list(buf) self.historyi = i self.pos = len(self.buffer) - self.dirty = 1 + self.dirty = True def get_item(self, i): if i != len(self.history): @@ -338,19 +338,3 @@ def finish(self): should_auto_add_history = True - - -def test(): - from .unix_console import UnixConsole - - reader = HistoricalReader(UnixConsole()) - reader.ps1 = "h**> " - reader.ps2 = "h/*> " - reader.ps3 = "h|*> " - reader.ps4 = r"h\*> " - while reader.readline(): - pass - - -if __name__ == "__main__": - test() diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 00000000000000..92617569e5e22a --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,29 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_any_generics = False +disallow_incomplete_defs = False +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False +warn_return_any = False + +disable_error_code = return + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index d7d1ce9b4cff14..85b7a2e0e9cef1 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -19,50 +19,26 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import annotations + from contextlib import contextmanager +from dataclasses import dataclass, field, fields import re import unicodedata -import traceback +from traceback import _can_colorize, _ANSIColors # type: ignore[attr-defined] -from . import commands, input +from . import commands, console, input _r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]") -def _make_unctrl_map(): - uc_map = {} - for i in range(256): - c = chr(i) - if unicodedata.category(c)[0] != "C": - uc_map[i] = c - for i in range(32): - uc_map[i] = "^" + chr(ord("A") + i - 1) - uc_map[ord(b"\t")] = " " # display TABs as 4 characters - uc_map[ord(b"\177")] = "^?" - for i in range(256): - if i not in uc_map: - uc_map[i] = "\\%03o" % i - return uc_map - - -def _my_unctrl(c, u=_make_unctrl_map()): - # takes an integer, returns a unicode - if c in u: - return u[c] - else: - if unicodedata.category(c).startswith("C"): - return r"\u%04x" % ord(c) - else: - return c - - -if "a"[0] == b"a": - # When running tests with python2, bytes characters are bytes. - def _my_unctrl(c, uc=_my_unctrl): - return uc(ord(c)) +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName -def disp_str(buffer, join="".join, uc=_my_unctrl): +def disp_str(buffer: str) -> tuple[str, list[int]]: """disp_str(buffer:string) -> (string, [int]) Return the string that should be the printed represenation of @@ -74,28 +50,25 @@ def disp_str(buffer, join="".join, uc=_my_unctrl): the list always contains 0s or 1s at present; it could conceivably go higher as and when unicode support happens.""" - # disp_str proved to be a bottleneck for large inputs, - # so it needs to be rewritten in C; it's not required though. - s = [uc(x) for x in buffer] - b = [] # XXX: bytearray - for x in s: + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) b.append(1) - b.extend([0] * (len(x) - 1)) - return join(s), b - - -del _my_unctrl + b.extend([0] * (len(c) - 1)) + return "".join(s), b -del _make_unctrl_map # syntax classes: -[SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL] = range(3) +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) -def make_default_syntax_table(): +def make_default_syntax_table() -> dict[str, int]: # XXX perhaps should use some unicodedata here? - st = {} + st: dict[str, int] = {} for c in map(chr, range(256)): st[c] = SYNTAX_SYMBOL for c in [a for a in map(chr, range(256)) if a.isalnum()]: @@ -104,7 +77,20 @@ def make_default_syntax_table(): return st -default_keymap = tuple( +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if ( + isinstance(v, type) + and issubclass(v, Command) + and v.__name__[0].islower() + ): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( [ (r"\C-a", "beginning-of-line"), (r"\C-b", "left"), @@ -171,6 +157,7 @@ def make_default_syntax_table(): ] ) +@dataclass(slots=True) class Reader: """The Reader class implements the bare bones of a command reader, handling such details as editing and cursor motion. What it does @@ -189,11 +176,9 @@ class Reader: is. * screeninfo: Ahem. This list contains some info needed to move the - insertion point around reasonably efficiently. I'd like to - get rid of it, because its contents are obtuse (to put it - mildly) but I haven't worked out if that is possible yet. + insertion point around reasonably efficiently. * cxy, lxy: - the position of the insertion point in screen ... XXX + the position of the insertion point in screen ... * syntax_table: Dictionary mapping characters to `syntax class'; read the emacs docs to see what this means :-) @@ -226,53 +211,56 @@ class Reader: that we're done. """ - msg_at_bottom = True - - def __init__(self, console): - self.buffer = [] + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + msg_at_bottom: bool = True + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + + def __post_init__(self) -> None: # Enable the use of `insert` without a `prepare` call - necessary to # facilitate the tab completion hack implemented for # . - self.pos = 0 - self.ps1 = "->> " - self.ps2 = "/>> " - self.ps3 = "|.. " - self.ps4 = r"\__ " - self.kill_ring = [] - self.arg = None - self.finished = 0 - self.console = console - self.commands = {} - self.msg = "" - for v in vars(commands).values(): - if ( - isinstance(v, type) - and issubclass(v, commands.Command) - and v.__name__[0].islower() - ): - self.commands[v.__name__] = v - self.commands[v.__name__.replace("_", "-")] = v - self.syntax_table = make_default_syntax_table() - self.input_trans_stack = [] self.keymap = self.collect_keymap() self.input_trans = input.KeymapTranslator( self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) - - self.paste_mode = False + self.screeninfo = [(0, [0])] + self.cxy = self.pos2xy(self.pos) + self.lxy = (self.pos, 0) def collect_keymap(self): return default_keymap - def calc_screen(self): + def calc_screen(self) -> list[str]: """The purpose of this method is to translate changes in self.buffer into changes in self.screen. Currently it rips everything down and starts from scratch, which whilst not especially efficient is certainly simple(r). """ lines = self.get_unicode().split("\n") - screen = [] - screeninfo = [] + screen: list[str] = [] + screeninfo: list[tuple[int, list[int]]] = [] w = self.console.width - 1 p = self.pos for ln, line in zip(range(len(lines)), lines): @@ -290,8 +278,8 @@ def calc_screen(self): screeninfo.append((0, [])) p -= ll + 1 prompt, lp = self.process_prompt(prompt) - if traceback._can_colorize(): - prompt = traceback._ANSIColors.BOLD_MAGENTA + prompt + traceback._ANSIColors.RESET + if _can_colorize(): + prompt = _ANSIColors.BOLD_MAGENTA + prompt + _ANSIColors.RESET l, l2 = disp_str(line) wrapcount = (len(l) + lp) // w if wrapcount == 0: @@ -313,7 +301,7 @@ def calc_screen(self): screeninfo.append((0, [])) return screen - def process_prompt(self, prompt): + def process_prompt(self, prompt: str) -> tuple[str, int]: """Process the prompt. This means calculate the length of the prompt. The character \x01 @@ -346,7 +334,7 @@ def process_prompt(self, prompt): out_prompt += keep return out_prompt, l - def bow(self, p=None): + def bow(self, p: int | None = None) -> int: """Return the 0-based index of the word break preceding p most immediately. @@ -363,7 +351,7 @@ def bow(self, p=None): p -= 1 return p + 1 - def eow(self, p=None): + def eow(self, p: int | None = None) -> int: """Return the 0-based index of the word break following p most immediately. @@ -379,12 +367,11 @@ def eow(self, p=None): p += 1 return p - def bol(self, p=None): + def bol(self, p: int | None = None) -> int: """Return the 0-based index of the line break preceding p most immediately. p defaults to self.pos.""" - # XXX there are problems here. if p is None: p = self.pos b = self.buffer @@ -393,7 +380,7 @@ def bol(self, p=None): p -= 1 return p + 1 - def eol(self, p=None): + def eol(self, p: int | None = None) -> int: """Return the 0-based index of the line break following p most immediately. @@ -405,48 +392,42 @@ def eol(self, p=None): p += 1 return p - def get_arg(self, default=1): + def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, - returning `default' if there is None. `default' defaults - (groan) to 1.""" + returning `default' if there is None. Defaults to 1. + """ if self.arg is None: return default else: return self.arg - def get_prompt(self, lineno, cursor_on_line): + def get_prompt(self, lineno, cursor_on_line) -> str: """Return what should be in the left-hand margin for line `lineno'.""" if self.arg is not None and cursor_on_line: return "(arg: %s) " % self.arg + + if self.paste_mode: + return '(paste) ' + if "\n" in self.buffer: if lineno == 0: - res = self.ps2 + return self.ps2 elif lineno == self.buffer.count("\n"): - res = self.ps4 + return self.ps4 else: - res = self.ps3 - else: - res = self.ps1 + return self.ps3 - if self.paste_mode: - res= '(paste) ' + return self.ps1 - # Lazily call str() on self.psN, and cache the results using as key - # the object on which str() was called. This ensures that even if the - # same object is used e.g. for ps1 and ps2, str() is called only once. - if res not in self._pscache: - self._pscache[res] = str(res) - return self._pscache[res] - - def push_input_trans(self, itrans): + def push_input_trans(self, itrans) -> None: self.input_trans_stack.append(self.input_trans) self.input_trans = itrans - def pop_input_trans(self): + def pop_input_trans(self) -> None: self.input_trans = self.input_trans_stack.pop() - def pos2xy(self, pos): + def pos2xy(self, pos: int) -> tuple[int, int]: """Return the x, y coordinates of position 'pos'.""" # this *is* incomprehensible, yes. y = 0 @@ -472,56 +453,58 @@ def pos2xy(self, pos): i += 1 return p + i, y - def insert(self, text): + def insert(self, text) -> None: """Insert 'text' at the insertion point.""" self.buffer[self.pos : self.pos] = list(text) self.pos += len(text) - self.dirty = 1 + self.dirty = True - def update_cursor(self): + def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" self.cxy = self.pos2xy(self.pos) self.console.move_cursor(*self.cxy) - def after_command(self, cmd): + def after_command(self, cmd) -> None: """This function is called to allow post command cleanup.""" if getattr(cmd, "kills_digit_arg", 1): if self.arg is not None: - self.dirty = 1 + self.dirty = True self.arg = None - def prepare(self): + def prepare(self) -> None: """Get ready to run. Call restore when finished. You must not write to the console in between the calls to prepare and restore.""" try: self.console.prepare() self.arg = None - self.screeninfo = [] - self.finished = 0 + self.finished = False del self.buffer[:] self.pos = 0 - self.dirty = 1 + self.dirty = True self.last_command = None + self.calc_screen() + # self.screeninfo = [(0, [0])] + # self.cxy = self.pos2xy(self.pos) + # self.lxy = (self.pos, 0) # XXX should kill ring be here? - self._pscache = {} except BaseException: self.restore() raise - def last_command_is(self, klass): + def last_command_is(self, cls: type) -> bool: if not self.last_command: - return 0 - return issubclass(klass, self.last_command) + return False + return issubclass(cls, self.last_command) - def restore(self): + def restore(self) -> None: """Clean up after a run.""" self.console.restore() @contextmanager - def suspend(self): + def suspend(self) -> SimpleContextManager: """A context manager to delegate to another reader.""" - prev_state = dict(self.__dict__) + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} try: self.restore() yield @@ -531,27 +514,27 @@ def suspend(self): self.prepare() pass - def finish(self): + def finish(self) -> None: """Called when a command signals that we're finished.""" pass - def error(self, msg="none"): + def error(self, msg: str = "none") -> None: self.msg = "! " + msg + " " - self.dirty = 1 + self.dirty = True self.console.beep() - def update_screen(self): + def update_screen(self) -> None: if self.dirty: self.refresh() - def refresh(self): + def refresh(self) -> None: """Recalculate and refresh the screen.""" # this call sets up self.cxy, so call it first. screen = self.calc_screen() self.console.refresh(screen, self.cxy) - self.dirty = 0 # forgot this for a while (blush) + self.dirty = False - def do_cmd(self, cmd): + def do_cmd(self, cmd) -> None: if isinstance(cmd[0], str): cmd = self.commands.get(cmd[0], commands.invalid_command)(self, *cmd) elif isinstance(cmd[0], type): @@ -571,24 +554,24 @@ def do_cmd(self, cmd): if not isinstance(cmd, commands.digit_arg): self.last_command = cmd.__class__ - self.finished = cmd.finish + self.finished = bool(cmd.finish) if self.finished: self.console.finish() self.finish() - def handle1(self, block=1): + def handle1(self, block: bool = True) -> bool: """Handle a single event. Wait as long as it takes if block - is true (the default), otherwise return None if no event is + is true (the default), otherwise return False if no event is pending.""" if self.msg: self.msg = "" - self.dirty = 1 + self.dirty = True - while 1: + while True: event = self.console.get_event(block) if not event: # can only happen if we're not blocking - return None + return False translate = True @@ -610,16 +593,16 @@ def handle1(self, block=1): if block: continue else: - return None + return False self.do_cmd(cmd) - return 1 + return True - def push_char(self, char): + def push_char(self, char) -> None: self.console.push_char(char) - self.handle1(0) + self.handle1(block=False) - def readline(self, returns_unicode=False, startup_hook=None): + def readline(self, startup_hook: Callback | None = None) -> str: """Read a line. The implementation of this method also shows how to drive Reader if you want more control over the event loop.""" @@ -630,39 +613,17 @@ def readline(self, returns_unicode=False, startup_hook=None): self.refresh() while not self.finished: self.handle1() - if returns_unicode: - return self.get_unicode() - return self.get_buffer() + return self.get_unicode() + finally: self.restore() - def bind(self, spec, command): + def bind(self, spec: KeySpec, command: CommandName) -> None: self.keymap = self.keymap + ((spec, command),) self.input_trans = input.KeymapTranslator( self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) - def get_buffer(self, encoding=None): - if encoding is None: - encoding = self.console.encoding - return self.get_unicode().encode(encoding) - - def get_unicode(self): + def get_unicode(self) -> str: """Return the current buffer as a unicode string.""" return "".join(self.buffer) - - -def test(): - from .unix_console import UnixConsole - - reader = Reader(UnixConsole()) - reader.ps1 = "**> " - reader.ps2 = "/*> " - reader.ps3 = "|*> " - reader.ps4 = r"\*> " - while reader.readline(): - pass - - -if __name__ == "__main__": - test() diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index e7a2e738686b85..28722c4c433dfc 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -271,9 +271,9 @@ def raw_input(self, prompt=""): except _error: return _old_raw_input(prompt) reader.ps1 = prompt - return reader.readline(returns_unicode=True, startup_hook=self.startup_hook) + return reader.readline(startup_hook=self.startup_hook) - def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): + def multiline_input(self, more_lines, ps1, ps2): """Read an input on possibly multiple lines, asking for more lines as long as 'more_lines(unicodetext)' returns an object whose boolean value is true. @@ -284,7 +284,7 @@ def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 - return reader.readline(returns_unicode=returns_unicode) + return reader.readline() finally: reader.more_lines = saved reader.paste_mode = False @@ -386,7 +386,8 @@ def set_startup_hook(self, function=None): self.startup_hook = function def get_line_buffer(self): - return self.get_reader().get_buffer() + buf_str = self.get_reader().get_unicode() + return buf_str.encode() def _get_idxs(self): start = cursor = self.get_reader().pos diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index b79b71b926b1c3..4b18b58397a403 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -30,7 +30,8 @@ import traceback from .console import Event -from .readline import _error, _get_reader, multiline_input +from .readline import _get_reader, multiline_input +from .unix_console import _error def check(): # returns False if there is a problem initializing the state @@ -122,7 +123,7 @@ def more_lines(unicodetext): ps1 = getattr(sys, "ps1", ">>> ") ps2 = getattr(sys, "ps2", "... ") try: - statement = multiline_input(more_lines, ps1, ps2, returns_unicode=True) + statement = multiline_input(more_lines, ps1, ps2) except EOFError: break diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 00000000000000..e31686bfbb85e6 --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,6 @@ +from typing import Any, Callable, Iterator + +Callback = Callable[[], Any] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 4eac770fa5be2b..5f6d60ac93c7e5 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -36,7 +36,7 @@ def multiline_input(reader, namespace=None): reader.more_lines = partial(more_lines, namespace=namespace) reader.ps1 = reader.ps2 = ">>>" reader.ps3 = reader.ps4 = "..." - return reader.readline(returns_unicode=True) + return reader.readline() finally: reader.more_lines = saved reader.paste_mode = False From 6c188fb57c735ffd7beda9858d3a871c3df1d144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 29 Apr 2024 19:29:45 +0200 Subject: [PATCH 36/78] Add types to pager.py --- Lib/_pyrepl/pager.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 5e7a9e2829e706..ecf5ddc79a5efa 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -1,10 +1,20 @@ +from __future__ import annotations + import io import os import re import sys -def get_pager(): +# types +if False: + from typing import Protocol, Any + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: """Decide what method to use for paging through text.""" if not hasattr(sys.stdin, "isatty"): return plain_pager @@ -41,32 +51,34 @@ def get_pager(): os.unlink(filename) -def escape_stdout(text): +def escape_stdout(text: str) -> str: # Escape non-encodable characters to avoid encoding errors later encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' return text.encode(encoding, 'backslashreplace').decode(encoding) -def escape_less(s): +def escape_less(s: str) -> str: return re.sub(r'([?:.%\\])', r'\\\1', s) -def plain(text): +def plain(text: str) -> str: """Remove boldface formatting from text.""" return re.sub('.\b', '', text) -def tty_pager(text, title=''): +def tty_pager(text: str, title: str = '') -> None: """Page through text on a text terminal.""" lines = plain(escape_stdout(text)).split('\n') + has_tty = False try: import tty + import termios fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) + old = termios.tcgetattr(fd) tty.setcbreak(fd) getchar = lambda: sys.stdin.read(1) + has_tty = True except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None getchar = lambda: sys.stdin.readline()[:-1][:1] try: @@ -97,16 +109,16 @@ def tty_pager(text, title=''): r = r + inc finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) -def plain_pager(text, title=''): +def plain_pager(text: str, title: str = '') -> None: """Simply print unformatted text. This is the ultimate fallback.""" sys.stdout.write(plain(escape_stdout(text))) -def pipe_pager(text, cmd, title=''): +def pipe_pager(text: str, cmd: str, title: str = '') -> None: """Page through text by feeding it to another program.""" import subprocess env = os.environ.copy() @@ -144,7 +156,7 @@ def pipe_pager(text, cmd, title=''): pass -def tempfile_pager(text, cmd, title=''): +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: """Page through text by invoking a program on a temporary file.""" import tempfile with tempfile.TemporaryDirectory() as tempdir: From 7919baeab61d2e485eadd37db24bd8a84369ddd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 29 Apr 2024 19:43:42 +0200 Subject: [PATCH 37/78] Add types to input.py, console.pu, and commands.py --- Lib/_pyrepl/commands.py | 164 +++++++++++++------------------ Lib/_pyrepl/console.py | 80 +++++++++------ Lib/_pyrepl/historical_reader.py | 15 ++- Lib/_pyrepl/input.py | 24 +++-- Lib/_pyrepl/reader.py | 9 +- Lib/_pyrepl/readline.py | 2 +- Lib/_pyrepl/types.py | 1 + Lib/_pyrepl/unix_console.py | 2 +- Lib/test/test_pyrepl.py | 44 ++++++++- 9 files changed, 198 insertions(+), 143 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 2e22c62d7a9ebf..fd25988b674aac 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -19,9 +19,10 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import annotations import os -# Catgories of actions: +# Categories of actions: # killing # yanking # motion @@ -31,21 +32,28 @@ # [completion] +# types +if False: + from .reader import Reader + from .historical_reader import HistoricalReader + from .console import Event + + class Command: - finish = 0 - kills_digit_arg = 1 + finish: bool = False + kills_digit_arg: bool = True - def __init__(self, reader, event_name, event): + def __init__(self, reader: HistoricalReader, event_name: str, event: str) -> None: self.reader = reader self.event = event self.event_name = event_name - def do(self): + def do(self) -> None: pass class KillCommand(Command): - def kill_range(self, start, end): + def kill_range(self, start: int, end: int) -> None: if start == end: return r = self.reader @@ -60,7 +68,7 @@ def kill_range(self, start, end): else: r.kill_ring.append(text) r.pos = start - r.dirty = 1 + r.dirty = True class YankCommand(Command): @@ -76,25 +84,25 @@ class EditCommand(Command): class FinishCommand(Command): - finish = 1 + finish = True pass -def is_kill(command): - return command and issubclass(command, KillCommand) +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) -def is_yank(command): - return command and issubclass(command, YankCommand) +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) # etc class digit_arg(Command): - kills_digit_arg = 0 + kills_digit_arg = False - def do(self): + def do(self) -> None: r = self.reader c = self.event[-1] if c == "-": @@ -111,29 +119,29 @@ def do(self): r.arg = 10 * r.arg - d else: r.arg = 10 * r.arg + d - r.dirty = 1 + r.dirty = True class clear_screen(Command): - def do(self): + def do(self) -> None: r = self.reader r.console.clear() - r.dirty = 1 + r.dirty = True class refresh(Command): - def do(self): - self.reader.dirty = 1 + def do(self) -> None: + self.reader.dirty = True class repaint(Command): - def do(self): - self.reader.dirty = 1 + def do(self) -> None: + self.reader.dirty = True self.reader.console.repaint_prep() class kill_line(KillCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer eol = r.eol() @@ -146,38 +154,34 @@ def do(self): class unix_line_discard(KillCommand): - def do(self): + def do(self) -> None: r = self.reader self.kill_range(r.bol(), r.pos) -# XXX unix_word_rubout and backward_kill_word should actually -# do different things... - - class unix_word_rubout(KillCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): self.kill_range(r.bow(), r.pos) class kill_word(KillCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): self.kill_range(r.pos, r.eow()) class backward_kill_word(KillCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): self.kill_range(r.bow(), r.pos) class yank(YankCommand): - def do(self): + def do(self) -> None: r = self.reader if not r.kill_ring: r.error("nothing to yank") @@ -186,7 +190,7 @@ def do(self): class yank_pop(YankCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer if not r.kill_ring: @@ -200,11 +204,11 @@ def do(self): t = r.kill_ring[-1] b[r.pos - repl : r.pos] = t r.pos = r.pos - repl + len(t) - r.dirty = 1 + r.dirty = True class interrupt(FinishCommand): - def do(self): + def do(self) -> None: import signal self.reader.console.finish() @@ -212,7 +216,7 @@ def do(self): class suspend(Command): - def do(self): + def do(self) -> None: import signal r = self.reader @@ -223,13 +227,13 @@ def do(self): ## in a handler for SIGCONT? r.console.prepare() r.pos = p - r.posxy = 0, 0 - r.dirty = 1 + r.posxy = 0, 0 # XXX this is invalid + r.dirty = True r.console.screen = [] class up(MotionCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): bol1 = r.bol() @@ -243,14 +247,13 @@ def do(self): bol2 = r.bol(bol1 - 1) line_pos = r.pos - bol1 if line_pos > bol1 - bol2 - 1: - r.sticky_y = line_pos r.pos = bol1 - 1 else: r.pos = bol2 + line_pos class down(MotionCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer for i in range(r.get_arg()): @@ -272,7 +275,7 @@ def do(self): class left(MotionCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): p = r.pos - 1 @@ -283,7 +286,7 @@ def do(self): class right(MotionCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer for i in range(r.get_arg()): @@ -295,53 +298,53 @@ def do(self): class beginning_of_line(MotionCommand): - def do(self): + def do(self) -> None: self.reader.pos = self.reader.bol() class end_of_line(MotionCommand): - def do(self): + def do(self) -> None: self.reader.pos = self.reader.eol() class home(MotionCommand): - def do(self): + def do(self) -> None: self.reader.pos = 0 class end(MotionCommand): - def do(self): + def do(self) -> None: self.reader.pos = len(self.reader.buffer) class forward_word(MotionCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): r.pos = r.eow() class backward_word(MotionCommand): - def do(self): + def do(self) -> None: r = self.reader for i in range(r.get_arg()): r.pos = r.bow() class self_insert(EditCommand): - def do(self): + def do(self) -> None: r = self.reader r.insert(self.event * r.get_arg()) class insert_nl(EditCommand): - def do(self): + def do(self) -> None: r = self.reader r.insert("\n" * r.get_arg()) class transpose_characters(EditCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer s = r.pos - 1 @@ -355,24 +358,24 @@ def do(self): del b[s] b.insert(t, c) r.pos = t - r.dirty = 1 + r.dirty = True class backspace(EditCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer for i in range(r.get_arg()): if r.pos > 0: r.pos -= 1 del b[r.pos] - r.dirty = 1 + r.dirty = True else: self.reader.error("can't backspace at start") class delete(EditCommand): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer if ( @@ -386,69 +389,40 @@ def do(self): for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] - r.dirty = 1 + r.dirty = True else: self.reader.error("end of buffer") class accept(FinishCommand): - def do(self): + def do(self) -> None: pass class help(Command): - def do(self): + def do(self) -> None: import _sitebuiltins with self.reader.suspend(): - self.reader.msg = _sitebuiltins._Helper()() + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] class invalid_key(Command): - def do(self): + def do(self) -> None: pending = self.reader.console.getpending() s = "".join(self.event) + pending.data self.reader.error("`%r' not bound" % s) class invalid_command(Command): - def do(self): + def do(self) -> None: s = self.event_name self.reader.error("command `%s' not known" % s) -class qIHelp(Command): - def do(self): - from .reader import disp_str - - r = self.reader - pending = r.console.getpending().data - disp = disp_str((self.event + pending).encode())[0] - r.insert(disp * r.get_arg()) - r.pop_input_trans() - - -class QITrans: - def push(self, evt): - self.evt = evt - - def get(self): - return ("qIHelp", self.evt.raw) - - -class quoted_insert(Command): - kills_digit_arg = 0 - - def do(self): - # XXX in Python 3, processing insert/C-q/C-v keys crashes - # because of a mixture of str and bytes. Disable these keys. - pass - # self.reader.push_input_trans(QITrans()) - - class show_history(Command): - def do(self): - from _pyrepl.pager import get_pager - from site import gethistoryfile + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] history = os.linesep.join(self.reader.history[:]) with self.reader.suspend(): pager = get_pager() @@ -457,6 +431,6 @@ def do(self): class paste_mode(Command): - def do(self): + def do(self) -> None: self.reader.paste_mode = True - self.reader.dirty = 1 + self.reader.dirty = True diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 17e66bbd3ba77f..7f12f20e13e827 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -17,6 +17,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from abc import ABC, abstractmethod import dataclasses @@ -27,7 +28,7 @@ class Event: raw: bytes = b"" -class Console: +class Console(ABC): """Attributes: screen, @@ -35,58 +36,79 @@ class Console: width, """ - def refresh(self, screen, xy): - pass + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + ... - def prepare(self): - pass + @abstractmethod + def prepare(self) -> None: + ... - def restore(self): - pass + @abstractmethod + def restore(self) -> None: + ... - def move_cursor(self, x, y): - pass + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: + ... - def set_cursor_vis(self, vis): - pass + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: + ... - def getheightwidth(self): + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: """Return (height, width) where height and width are the height and width of the terminal window in characters.""" - pass + ... - def get_event(self, block=1): + @abstractmethod + def get_event(self, block: bool = True) -> Event: """Return an Event instance. Returns None if |block| is false and there is no event pending, otherwise waits for the completion of an event.""" - pass + ... - def beep(self): - pass + @abstractmethod + def push_char(self, char: str) -> None: + """ + Push a character to the console event queue. + """ + ... - def clear(self): + @abstractmethod + def beep(self) -> None: + ... + + @abstractmethod + def clear(self) -> None: """Wipe the screen""" - pass + ... - def finish(self): + @abstractmethod + def finish(self) -> None: """Move the cursor to the end of the display and otherwise get ready for end. XXX could be merged with restore? Hmm.""" - pass + ... - def flushoutput(self): + @abstractmethod + def flushoutput(self) -> None: """Flush all output to the screen (assuming there's some buffering going on somewhere).""" - pass + ... - def forgetinput(self): + @abstractmethod + def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" - pass + ... - def getpending(self): + @abstractmethod + def getpending(self) -> Event: """Return the characters that have been typed but not yet processed.""" - pass + ... - def wait(self): + @abstractmethod + def wait(self) -> None: """Wait for an event.""" - pass + ... diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index 380fc63ce1a435..05a86d92377a09 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -17,12 +17,19 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import annotations + from contextlib import contextmanager from . import commands, input -from .reader import Reader as R +from .reader import Reader + + +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName -isearch_keymap = tuple( + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + [ @@ -46,7 +53,7 @@ class next_history(commands.Command): - def do(self): + def do(self) -> None: r = self.reader if r.historyi == len(r.history): r.error("end of history list") @@ -190,7 +197,7 @@ def do(self): r.dirty = 1 -class HistoricalReader(R): +class HistoricalReader(Reader): """Adds history support (with incremental history searching) to the Reader class. diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py index 974df83e1a4283..300e16d1d25441 100644 --- a/Lib/_pyrepl/input.py +++ b/Lib/_pyrepl/input.py @@ -32,20 +32,32 @@ # executive, temporary decision: [tab] and [C-i] are distinct, but # [meta-key] is identified with [esc key]. We demand that any console # class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod import unicodedata from collections import deque -class InputTranslator: - def push(self, evt): - pass +# types +if False: + from .types import EventTuple - def get(self): - pass - def empty(self): +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: pass + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + class KeymapTranslator(InputTranslator): def __init__(self, keymap, verbose=0, invalid_cls=None, character_cls=None): diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 85b7a2e0e9cef1..9c93e0830d4cac 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -105,10 +105,8 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\C-k", "kill-line"), (r"\C-l", "clear-screen"), (r"\C-m", "accept"), - (r"\C-q", "quoted-insert"), (r"\C-t", "transpose-characters"), (r"\C-u", "unix-line-discard"), - (r"\C-v", "quoted-insert"), (r"\C-w", "unix-word-rubout"), (r"\C-x\C-u", "upcase-region"), (r"\C-y", "yank"), @@ -142,7 +140,6 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\", "down"), (r"\", "left"), (r"\", "right"), - (r"\", "quoted-insert"), (r"\", "delete"), (r"\", "backspace"), (r"\M-\", "backward-kill-word"), @@ -464,9 +461,9 @@ def update_cursor(self) -> None: self.cxy = self.pos2xy(self.pos) self.console.move_cursor(*self.cxy) - def after_command(self, cmd) -> None: + def after_command(self, cmd: Command) -> None: """This function is called to allow post command cleanup.""" - if getattr(cmd, "kills_digit_arg", 1): + if getattr(cmd, "kills_digit_arg", True): if self.arg is not None: self.dirty = True self.arg = None @@ -537,7 +534,7 @@ def refresh(self) -> None: def do_cmd(self, cmd) -> None: if isinstance(cmd[0], str): cmd = self.commands.get(cmd[0], commands.invalid_command)(self, *cmd) - elif isinstance(cmd[0], type): + elif isinstance(cmd[0], type) and issubclass(cmd[0], commands.Command): cmd = cmd[0](self, *cmd) else: return # nothing to do diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 28722c4c433dfc..2f01b8c3030b42 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -28,7 +28,7 @@ import os import readline -from site import gethistoryfile +from site import gethistoryfile # type: ignore[attr-defined] import sys from . import commands, historical_reader diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index e31686bfbb85e6..21921627871064 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -4,3 +4,4 @@ SimpleContextManager = Iterator[None] KeySpec = str # like r"\C-c" CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index d4523c5cefa579..1130fd36e2c24f 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -349,7 +349,7 @@ def push_char(self, char): trace("push char {char!r}", char=char) self.event_queue.push(char) - def get_event(self, block=True): + def get_event(self, block: bool = True) -> Event: """ Get an event from the console event queue. diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 5f6d60ac93c7e5..0898335219e547 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -55,9 +55,51 @@ def __init__(self, events, encoding="utf-8"): self.height = 100 self.width = 80 - def get_event(self, block=True): + def get_event(self, block: bool = True) -> Event: return next(self.events) + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: str) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self) -> None: + pass + class TestPyReplDriver(TestCase): def prepare_reader(self, events): From 2a7e81d3d772a23fe9803526916a56eb4fe0788b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 29 Apr 2024 20:11:07 +0200 Subject: [PATCH 38/78] Refactor termios stuff in unix console --- Lib/_pyrepl/_minimal_curses.py | 68 -------- Lib/_pyrepl/fancy_termios.py | 223 ++++++++++++++++++++++-- Lib/_pyrepl/unix_console.py | 303 ++++++++++++++++++--------------- 3 files changed, 375 insertions(+), 219 deletions(-) delete mode 100644 Lib/_pyrepl/_minimal_curses.py diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index 7cff7d7ca2ea51..00000000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib(): - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if ctypes.cast(result, ctypes.c_void_p).value == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 5b85cb0f52521f..f8144387d45697 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -17,9 +17,14 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import curses import termios +class InvalidTerminal(RuntimeError): + pass + + class TermState: def __init__(self, tuples): ( @@ -55,20 +60,212 @@ def tcsetattr(fd, when, attrs): termios.tcsetattr(fd, when, attrs.as_list()) -class Term(TermState): - TS__init__ = TermState.__init__ +class TermCapability: + """Base class for all terminal capabilities we need""" + + def __init__(self, buffer): + self._buffer = buffer + self._args = [] + + self._bytes = curses.tigetstr(self.name) + if self._bytes is None and not self.optional: + raise InvalidTerminal( + f"terminal doesn't have the required {self.name!r} capability" + ) + + self.supported = self._bytes is not None + + def __call__(self, *args): + if self.supported: + self._args = args + self._buffer.push(self) + + def text(self): + return curses.tparm(self._bytes, *self._args) + + +class Bell(TermCapability): + """Audible signal (bell)""" + + name = "bel" + optional = False + + +class CursorInvisible(TermCapability): + """Make cursor invisible""" + + name = "civis" + optional = True + + +class ClearScreen(TermCapability): + """Clear screen and home cursor""" + + name = "clear" + optional = False + + +class CursorNormal(TermCapability): + """Make cursor appear normal (undo civis/cvvis)""" + + name = "cnorm" + optional = True + + +class ParmLeftCursor(TermCapability): + """Move #1 characters to the left""" + + name = "cub" + optional = True + + +class CursorLeft(TermCapability): + """Move left one space""" + + name = "cub1" + optional = True + + def text(self): + assert len(self._args) == 1 # cursor_down needs to have been called with dx + return curses.tparm(self._args[0] * self._bytes) + + +class ParmDownCursor(TermCapability): + """Down #1 lines""" + + name = "cud" + optional = True + + +class CursorDown(TermCapability): + """Down one line""" + + name = "cud1" + optional = True + + def text(self): + assert len(self._args) == 1 # cursor_down needs to have been called with dy + return curses.tparm(self._args[0] * self._bytes) + + +class ParmRightCursor(TermCapability): + """Move #1 characters to the right""" + + name = "cuf" + optional = True + + +class CursorRight(TermCapability): + """Non-destructive space (move right one space)""" + + name = "cuf1" + optional = True + + def text(self): + assert len(self._args) == 1 # cursor_down needs to have been called with dx + return curses.tparm(self._args[0] * self._bytes) + + +class CursorAddress(TermCapability): + """Move to row #1 columns #2""" + + name = "cup" + optional = False + + +class ParmUpCursor(TermCapability): + """Up #1 lines""" + + name = "cuu" + optional = True + + +class CursorUp(TermCapability): + """Up 1 line""" + + name = "cuu1" + optional = True + + def text(self): + assert len(self._args) == 1 # cursor_down needs to have been called with dy + return curses.tparm(self._args[0] * self._bytes) + + +class ParmDeleteCharacter(TermCapability): + """Delete #1 characters""" + + name = "dch" + optional = True + + +class DeleteCharacter(TermCapability): + """Delete character""" + + name = "dch1" + optional = True + + +class ClearEol(TermCapability): + """Clear to end of line""" + + name = "el" + optional = False + + +class ColumnAddress(TermCapability): + """Horizontal position #1, absolute""" + + name = "hpa" + optional = True + + +class ParmInsertCharacter(TermCapability): + """Insert #1 characters""" + + name = "ich" + optional = True + + +class InsertCharacter(TermCapability): + """Insert character""" + + name = "ich1" + optional = True + + +class ScrollForward(TermCapability): + """Scroll text up""" + + name = "ind" + optional = True + + +class PadChar(TermCapability): + """Padding char (instead of null)""" + + name = "pad" + optional = True + + def text(self, nchars): + return self._bytes * nchars + + +class ScrollReverse(TermCapability): + """Scroll text down""" + + name = "ri" + optional = True + + +class KeypadLocal(TermCapability): + """Leave 'keyboard_transmit' mode""" - def __init__(self, fd=0): - self.TS__init__(termios.tcgetattr(fd)) - self.fd = fd - self.stack = [] + name = "rmkx" + optional = True - def save(self): - self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): - termios.tcsetattr(self.fd, when, self.as_list()) +class KeypadXmit(TermCapability): + """Enter 'keyboard_transmit' mode""" - def restore(self): - self.TS__init__(self.stack.pop()) - self.set() + name = "smkx" + optional = True diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 1130fd36e2c24f..5eabbad1f5c3b4 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -20,6 +20,7 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import errno +import functools import os import re import select @@ -32,15 +33,40 @@ from . import curses from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr +from .fancy_termios import ( + Bell, + ClearEol, + ClearScreen, + ColumnAddress, + CursorAddress, + CursorDown, + CursorInvisible, + CursorLeft, + CursorNormal, + CursorRight, + CursorUp, + DeleteCharacter, + InsertCharacter, + InvalidTerminal, + KeypadLocal, + KeypadXmit, + PadChar, + ParmDeleteCharacter, + ParmDownCursor, + ParmInsertCharacter, + ParmLeftCursor, + ParmRightCursor, + ParmUpCursor, + ScrollForward, + ScrollReverse, + TermCapability, + tcgetattr, + tcsetattr, +) from .trace import trace from .unix_eventqueue import EventQueue -class InvalidTerminal(RuntimeError): - pass - - _error = (termios.error, curses.error, InvalidTerminal) SIGWINCH_EVENT = "repaint" @@ -117,6 +143,55 @@ def poll(self): # note: a 'timeout' argument would be *milliseconds* POLLIN = getattr(select, "POLLIN", None) +class Buffer: + def __init__(self, svtermstate, output_fd, encoding): + self.__svtermstate = svtermstate + self.__output_fd = output_fd + self.__pad_char = PadChar(self) + self.__encoding = encoding + self.__buf = [] + + def push(self, item): + self.__buf.append(item) + + def clear(self): + self.__buf.clear() + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.__output_fd, fmt) + break + x, y = m.span() + os.write(self.__output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self.__pad_char.supported and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.__output_fd, self.__pad_char.text(nchars)) + else: + time.sleep(float(delay) / 1000.0) + + def flush(self): + for item in self.__buf: + if isinstance(item, TermCapability): + self.__tputs(item.text()) + else: + os.write(self.__output_fd, item.encode(self.__encoding, "replace")) + self.clear() + + class UnixConsole(Console): def __init__(self, f_in=0, f_out=1, term=None, encoding=None): """ @@ -148,38 +223,32 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): curses.setupterm(term, self.output_fd) self.term = term - def _my_getstr(cap, optional=0): - r = curses.tigetstr(cap) - if not optional and r is None: - raise InvalidTerminal( - f"terminal doesn't have the required {cap} capability" - ) - return r - - self._bel = _my_getstr("bel") - self._civis = _my_getstr("civis", optional=True) - self._clear = _my_getstr("clear") - self._cnorm = _my_getstr("cnorm", optional=True) - self._cub = _my_getstr("cub", optional=True) - self._cub1 = _my_getstr("cub1", optional=True) - self._cud = _my_getstr("cud", optional=True) - self._cud1 = _my_getstr("cud1", optional=True) - self._cuf = _my_getstr("cuf", optional=True) - self._cuf1 = _my_getstr("cuf1", optional=True) - self._cup = _my_getstr("cup") - self._cuu = _my_getstr("cuu", optional=True) - self._cuu1 = _my_getstr("cuu1", optional=True) - self._dch1 = _my_getstr("dch1", optional=True) - self._dch = _my_getstr("dch", optional=True) - self._el = _my_getstr("el") - self._hpa = _my_getstr("hpa", optional=True) - self._ich = _my_getstr("ich", optional=True) - self._ich1 = _my_getstr("ich1", optional=True) - self._ind = _my_getstr("ind", optional=True) - self._pad = _my_getstr("pad", optional=True) - self._ri = _my_getstr("ri", optional=True) - self._rmkx = _my_getstr("rmkx", optional=True) - self._smkx = _my_getstr("smkx", optional=True) + self.__svtermstate = tcgetattr(self.input_fd) + self.__buffer = Buffer(self.__svtermstate, f_out, self.encoding) + + self._bell = Bell(self.__buffer) + self._cursor_invsible = CursorInvisible(self.__buffer) + self._clear_screen = ClearScreen(self.__buffer) + self._cursor_normal = CursorNormal(self.__buffer) + self._parm_left_cursor = ParmLeftCursor(self.__buffer) + self._cursor_left = CursorLeft(self.__buffer) + self._parm_down_cursor = ParmDownCursor(self.__buffer) + self._cursor_down = CursorDown(self.__buffer) + self._parm_right_cursor = ParmRightCursor(self.__buffer) + self._cursor_right = CursorRight(self.__buffer) + self._cursor_address = CursorAddress(self.__buffer) + self._parm_up_cursor = ParmUpCursor(self.__buffer) + self._cursor_up = CursorUp(self.__buffer) + self._parm_delete_character = ParmDeleteCharacter(self.__buffer) + self._delete_character = DeleteCharacter(self.__buffer) + self._clear_eol = ClearEol(self.__buffer) + self._column_address = ColumnAddress(self.__buffer) + self._parm_insert_character = ParmInsertCharacter(self.__buffer) + self._insert_character = InsertCharacter(self.__buffer) + self._scroll_forward = ScrollForward(self.__buffer) + self._scroll_reverse = ScrollReverse(self.__buffer) + self._keypad_local = KeypadLocal(self.__buffer) + self._keypad_xmit = KeypadXmit(self.__buffer) self.__setup_movement() @@ -237,20 +306,20 @@ def refresh(self, screen, c_xy): newscr = screen[offset : offset + height] # use hardware scrolling if we have it. - if old_offset > offset and self._ri: + if old_offset > offset and self._scroll_reverse.supported: self.__hide_cursor() - self.__write_code(self._cup, 0, 0) + self._cursor_address(0, 0) self.__posxy = 0, old_offset for i in range(old_offset - offset): - self.__write_code(self._ri) + self._scroll_reverse() oldscr.pop(-1) oldscr.insert(0, "") - elif old_offset < offset and self._ind: + elif old_offset < offset and self._scroll_forward.supported: self.__hide_cursor() - self.__write_code(self._cup, self.height - 1, 0) + self._cursor_address(self.height - 1, 0) self.__posxy = 0, old_offset + self.height - 1 for i in range(offset - old_offset): - self.__write_code(self._ind) + self._scroll_forward() oldscr.pop(0) oldscr.append("") @@ -269,7 +338,7 @@ def refresh(self, screen, c_xy): self.__hide_cursor() self.__move(0, y) self.__posxy = 0, y - self.__write_code(self._el) + self._clear_eol() y += 1 self.__show_cursor() @@ -297,7 +366,6 @@ def prepare(self): """ Prepare the console for input/output operations. """ - self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) raw.oflag &= ~(termios.OPOST) @@ -313,14 +381,14 @@ def prepare(self): self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] + self.__buffer.clear() self.__posxy = 0, 0 self.__gone_tall = 0 self.__move = self.__move_short self.__offset = 0 - self.__maybe_write_code(self._smkx) + self._keypad_xmit() try: self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) @@ -331,7 +399,7 @@ def restore(self): """ Restore the console to the default state """ - self.__maybe_write_code(self._rmkx) + self._keypad_local() self.flushoutput() tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) @@ -438,12 +506,7 @@ def flushoutput(self): """ Flush the output buffer. """ - for text, iscode in self.__buffer: - if iscode: - self.__tputs(text) - else: - os.write(self.output_fd, text.encode(self.encoding, "replace")) - del self.__buffer[:] + self.__buffer.flush() def finish(self): """ @@ -460,7 +523,7 @@ def beep(self): """ Emit a beep sound. """ - self.__maybe_write_code(self._bel) + self._bell() self.flushoutput() if FIONREAD: @@ -513,7 +576,7 @@ def clear(self): """ Clear the console screen. """ - self.__write_code(self._clear) + self._clear_screen() self.__gone_tall = 1 self.__move = self.__move_tall self.__posxy = 0, 0 @@ -523,41 +586,39 @@ def __setup_movement(self): """ Set up the movement functions based on the terminal capabilities. """ - if 0 and self._hpa: # hpa don't work in windows telnet :-( - self.__move_x = self.__move_x_hpa - elif self._cub and self._cuf: - self.__move_x = self.__move_x_cub_cuf - elif self._cub1 and self._cuf1: - self.__move_x = self.__move_x_cub1_cuf1 + if self._parm_left_cursor.supported and self._parm_right_cursor.supported: + self.__move_x = self.__move_x_parm_cursor + elif self._cursor_left.supported and self._cursor_right.supported: + self.__move_x = self.__move_x_cursor else: - raise RuntimeError("insufficient terminal (horizontal)") + raise InvalidTerminal("insufficient terminal (horizontal)") - if self._cuu and self._cud: - self.__move_y = self.__move_y_cuu_cud - elif self._cuu1 and self._cud1: - self.__move_y = self.__move_y_cuu1_cud1 + if self._parm_up_cursor.supported and self._parm_down_cursor.supported: + self.__move_y = self.__move_y_parm_cursor + elif self._cursor_up.supported and self._cursor_down.supported: + self.__move_y = self.__move_y_cursor else: - raise RuntimeError("insufficient terminal (vertical)") + raise InvalidTerminal("insufficient terminal (vertical)") - if self._dch1: - self.dch1 = self._dch1 - elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + if self._delete_character.supported: + self.delete_character = self._delete_character + elif self._parm_delete_character.supported: + self.delete_character = functools.partial(self._parm_delete_character, 1) else: - self.dch1 = None + self.delete_character = None - if self._ich1: - self.ich1 = self._ich1 - elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + if self._insert_character.supported: + self.insert_character = self._insert_character + elif self._parm_insert_character.supported: + self.insert_character = functools.partial(self._parm_insert_character, 1) else: - self.ich1 = None + self.insert_character = None self.__move = self.__move_short def __write_changed_line(self, y, oldline, newline, px): # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of + # self.delete_character inside the loop -- but alternative ways of # structuring this function are equally painful (I'm trying to # avoid writing code generators these days...) x = 0 @@ -568,7 +629,7 @@ def __write_changed_line(self, y, oldline, newline, px): # sequene while x < minlen and oldline[x] == newline[x] and newline[x] != "\x1b": x += 1 - if oldline[x:] == newline[x + 1 :] and self.ich1: + if oldline[x:] == newline[x + 1 :] and self.insert_character is not None: if ( y == self.__posxy[1] and x > self.__posxy[0] @@ -576,7 +637,7 @@ def __write_changed_line(self, y, oldline, newline, px): ): x = px self.__move(x, y) - self.__write_code(self.ich1) + self.insert_character() self.__write(newline[x]) self.__posxy = x + 1, y elif x < minlen and oldline[x + 1 :] == newline[x + 1 :]: @@ -584,8 +645,8 @@ def __write_changed_line(self, y, oldline, newline, px): self.__write(newline[x]) self.__posxy = x + 1, y elif ( - self.dch1 - and self.ich1 + self.delete_character is not None + and self.insert_character is not None and len(newline) == self.width and x < len(newline) - 2 and newline[x + 1 : -1] == oldline[x:-2] @@ -593,16 +654,16 @@ def __write_changed_line(self, y, oldline, newline, px): self.__hide_cursor() self.__move(self.width - 2, y) self.__posxy = self.width - 2, y - self.__write_code(self.dch1) + self.delete_character() self.__move(x, y) - self.__write_code(self.ich1) + self.insert_character() self.__write(newline[x]) self.__posxy = x + 1, y else: self.__hide_cursor() self.__move(x, y) if len(oldline) > len(newline): - self.__write_code(self._el) + self._clear_eol() self.__write(newline[x:]) self.__posxy = len(newline), y @@ -613,46 +674,39 @@ def __write_changed_line(self, y, oldline, newline, px): self.move_cursor(0, y) def __write(self, text): - self.__buffer.append((text, 0)) - - def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.push(text) - def __maybe_write_code(self, fmt, *args): - if fmt: - self.__write_code(fmt, *args) - - def __move_y_cuu1_cud1(self, y): + def __move_y_cursor(self, y): dy = y - self.__posxy[1] if dy > 0: - self.__write_code(dy * self._cud1) + self._cursor_down(dy) elif dy < 0: - self.__write_code((-dy) * self._cuu1) + self._cursor_up(-dy) - def __move_y_cuu_cud(self, y): + def __move_y_parm_cursor(self, y): dy = y - self.__posxy[1] if dy > 0: - self.__write_code(self._cud, dy) + self._parm_down_cursor(dy) elif dy < 0: - self.__write_code(self._cuu, -dy) + self._parm_up_cursor(-dy) - def __move_x_hpa(self, x): + def __move_x_column_address(self, x): if x != self.__posxy[0]: - self.__write_code(self._hpa, x) + self._column_address(x) - def __move_x_cub1_cuf1(self, x): + def __move_x_cursor(self, x): dx = x - self.__posxy[0] if dx > 0: - self.__write_code(self._cuf1 * dx) + self._cursor_right(dx) elif dx < 0: - self.__write_code(self._cub1 * (-dx)) + self._cursor_left(-dx) - def __move_x_cub_cuf(self, x): + def __move_x_parm_cursor(self, x): dx = x - self.__posxy[0] if dx > 0: - self.__write_code(self._cuf, dx) + self._parm_right_cursor(dx) elif dx < 0: - self.__write_code(self._cub, -dx) + self._parm_left_cursor(-dx) def __move_short(self, x, y): self.__move_x(x) @@ -660,7 +714,7 @@ def __move_short(self, x, y): def __move_tall(self, x, y): assert 0 <= y - self.__offset < self.height, y - self.__offset - self.__write_code(self._cup, y - self.__offset, x) + self._cursor_address(y - self.__offset, x) def __sigwinch(self, signum, frame): self.height, self.width = self.getheightwidth() @@ -668,12 +722,12 @@ def __sigwinch(self, signum, frame): def __hide_cursor(self): if self.cursor_visible: - self.__maybe_write_code(self._civis) + self._cursor_invsible() self.cursor_visible = 0 def __show_cursor(self): if not self.cursor_visible: - self.__maybe_write_code(self._cnorm) + self._cursor_normal() self.cursor_visible = 1 def repaint_prep(self): @@ -687,30 +741,3 @@ def repaint_prep(self): self.__move(0, self.__offset) ns = self.height * ["\000" * self.width] self.screen = ns - - def __tputs(self, fmt, prog=delayprog): - """A Python implementation of the curses tputs function; the - curses one can't really be wrapped in a sane manner. - - I have the strong suspicion that this is complexity that - will never do anyone any good.""" - # using .get() means that things will blow up - # only if the bps is actually needed (which I'm - # betting is pretty unlkely) - bps = ratedict.get(self.__svtermstate.ospeed) - while 1: - m = prog.search(fmt) - if not m: - os.write(self.output_fd, fmt) - break - x, y = m.span() - os.write(self.output_fd, fmt[:x]) - fmt = fmt[y:] - delay = int(m.group(1)) - if b"*" in m.group(2): - delay *= self.height - if self._pad and bps is not None: - nchars = (bps * delay) / 1000 - os.write(self.output_fd, self._pad * nchars) - else: - time.sleep(float(delay) / 1000.0) From 8c368d0650601d5ebcf6b8a9c83a5ffc1160102e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 29 Apr 2024 19:35:45 +0100 Subject: [PATCH 39/78] Revert "Refactor termios stuff in unix console" This reverts commit 2a7e81d3d772a23fe9803526916a56eb4fe0788b. --- Lib/_pyrepl/_minimal_curses.py | 68 ++++++++ Lib/_pyrepl/fancy_termios.py | 223 ++---------------------- Lib/_pyrepl/unix_console.py | 303 +++++++++++++++------------------ 3 files changed, 219 insertions(+), 375 deletions(-) create mode 100644 Lib/_pyrepl/_minimal_curses.py diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000000..7cff7d7ca2ea51 --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib(): + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if ctypes.cast(result, ctypes.c_void_p).value == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index f8144387d45697..5b85cb0f52521f 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -17,14 +17,9 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -import curses import termios -class InvalidTerminal(RuntimeError): - pass - - class TermState: def __init__(self, tuples): ( @@ -60,212 +55,20 @@ def tcsetattr(fd, when, attrs): termios.tcsetattr(fd, when, attrs.as_list()) -class TermCapability: - """Base class for all terminal capabilities we need""" - - def __init__(self, buffer): - self._buffer = buffer - self._args = [] - - self._bytes = curses.tigetstr(self.name) - if self._bytes is None and not self.optional: - raise InvalidTerminal( - f"terminal doesn't have the required {self.name!r} capability" - ) - - self.supported = self._bytes is not None - - def __call__(self, *args): - if self.supported: - self._args = args - self._buffer.push(self) - - def text(self): - return curses.tparm(self._bytes, *self._args) - - -class Bell(TermCapability): - """Audible signal (bell)""" - - name = "bel" - optional = False - - -class CursorInvisible(TermCapability): - """Make cursor invisible""" - - name = "civis" - optional = True - - -class ClearScreen(TermCapability): - """Clear screen and home cursor""" - - name = "clear" - optional = False - - -class CursorNormal(TermCapability): - """Make cursor appear normal (undo civis/cvvis)""" - - name = "cnorm" - optional = True - - -class ParmLeftCursor(TermCapability): - """Move #1 characters to the left""" - - name = "cub" - optional = True - - -class CursorLeft(TermCapability): - """Move left one space""" - - name = "cub1" - optional = True - - def text(self): - assert len(self._args) == 1 # cursor_down needs to have been called with dx - return curses.tparm(self._args[0] * self._bytes) - - -class ParmDownCursor(TermCapability): - """Down #1 lines""" - - name = "cud" - optional = True - - -class CursorDown(TermCapability): - """Down one line""" - - name = "cud1" - optional = True - - def text(self): - assert len(self._args) == 1 # cursor_down needs to have been called with dy - return curses.tparm(self._args[0] * self._bytes) - - -class ParmRightCursor(TermCapability): - """Move #1 characters to the right""" - - name = "cuf" - optional = True - - -class CursorRight(TermCapability): - """Non-destructive space (move right one space)""" - - name = "cuf1" - optional = True - - def text(self): - assert len(self._args) == 1 # cursor_down needs to have been called with dx - return curses.tparm(self._args[0] * self._bytes) - - -class CursorAddress(TermCapability): - """Move to row #1 columns #2""" - - name = "cup" - optional = False - - -class ParmUpCursor(TermCapability): - """Up #1 lines""" - - name = "cuu" - optional = True - - -class CursorUp(TermCapability): - """Up 1 line""" - - name = "cuu1" - optional = True - - def text(self): - assert len(self._args) == 1 # cursor_down needs to have been called with dy - return curses.tparm(self._args[0] * self._bytes) - - -class ParmDeleteCharacter(TermCapability): - """Delete #1 characters""" - - name = "dch" - optional = True - - -class DeleteCharacter(TermCapability): - """Delete character""" - - name = "dch1" - optional = True - - -class ClearEol(TermCapability): - """Clear to end of line""" - - name = "el" - optional = False - - -class ColumnAddress(TermCapability): - """Horizontal position #1, absolute""" - - name = "hpa" - optional = True - - -class ParmInsertCharacter(TermCapability): - """Insert #1 characters""" - - name = "ich" - optional = True - - -class InsertCharacter(TermCapability): - """Insert character""" - - name = "ich1" - optional = True - - -class ScrollForward(TermCapability): - """Scroll text up""" - - name = "ind" - optional = True - - -class PadChar(TermCapability): - """Padding char (instead of null)""" - - name = "pad" - optional = True - - def text(self, nchars): - return self._bytes * nchars - - -class ScrollReverse(TermCapability): - """Scroll text down""" - - name = "ri" - optional = True - - -class KeypadLocal(TermCapability): - """Leave 'keyboard_transmit' mode""" +class Term(TermState): + TS__init__ = TermState.__init__ - name = "rmkx" - optional = True + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + def save(self): + self.stack.append(self.as_list()) -class KeypadXmit(TermCapability): - """Enter 'keyboard_transmit' mode""" + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) - name = "smkx" - optional = True + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 5eabbad1f5c3b4..1130fd36e2c24f 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -20,7 +20,6 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import errno -import functools import os import re import select @@ -33,40 +32,15 @@ from . import curses from .console import Console, Event -from .fancy_termios import ( - Bell, - ClearEol, - ClearScreen, - ColumnAddress, - CursorAddress, - CursorDown, - CursorInvisible, - CursorLeft, - CursorNormal, - CursorRight, - CursorUp, - DeleteCharacter, - InsertCharacter, - InvalidTerminal, - KeypadLocal, - KeypadXmit, - PadChar, - ParmDeleteCharacter, - ParmDownCursor, - ParmInsertCharacter, - ParmLeftCursor, - ParmRightCursor, - ParmUpCursor, - ScrollForward, - ScrollReverse, - TermCapability, - tcgetattr, - tcsetattr, -) +from .fancy_termios import tcgetattr, tcsetattr from .trace import trace from .unix_eventqueue import EventQueue +class InvalidTerminal(RuntimeError): + pass + + _error = (termios.error, curses.error, InvalidTerminal) SIGWINCH_EVENT = "repaint" @@ -143,55 +117,6 @@ def poll(self): # note: a 'timeout' argument would be *milliseconds* POLLIN = getattr(select, "POLLIN", None) -class Buffer: - def __init__(self, svtermstate, output_fd, encoding): - self.__svtermstate = svtermstate - self.__output_fd = output_fd - self.__pad_char = PadChar(self) - self.__encoding = encoding - self.__buf = [] - - def push(self, item): - self.__buf.append(item) - - def clear(self): - self.__buf.clear() - - def __tputs(self, fmt, prog=delayprog): - """A Python implementation of the curses tputs function; the - curses one can't really be wrapped in a sane manner. - I have the strong suspicion that this is complexity that - will never do anyone any good.""" - # using .get() means that things will blow up - # only if the bps is actually needed (which I'm - # betting is pretty unlkely) - bps = ratedict.get(self.__svtermstate.ospeed) - while 1: - m = prog.search(fmt) - if not m: - os.write(self.__output_fd, fmt) - break - x, y = m.span() - os.write(self.__output_fd, fmt[:x]) - fmt = fmt[y:] - delay = int(m.group(1)) - if b"*" in m.group(2): - delay *= self.height - if self.__pad_char.supported and bps is not None: - nchars = (bps * delay) / 1000 - os.write(self.__output_fd, self.__pad_char.text(nchars)) - else: - time.sleep(float(delay) / 1000.0) - - def flush(self): - for item in self.__buf: - if isinstance(item, TermCapability): - self.__tputs(item.text()) - else: - os.write(self.__output_fd, item.encode(self.__encoding, "replace")) - self.clear() - - class UnixConsole(Console): def __init__(self, f_in=0, f_out=1, term=None, encoding=None): """ @@ -223,32 +148,38 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): curses.setupterm(term, self.output_fd) self.term = term - self.__svtermstate = tcgetattr(self.input_fd) - self.__buffer = Buffer(self.__svtermstate, f_out, self.encoding) - - self._bell = Bell(self.__buffer) - self._cursor_invsible = CursorInvisible(self.__buffer) - self._clear_screen = ClearScreen(self.__buffer) - self._cursor_normal = CursorNormal(self.__buffer) - self._parm_left_cursor = ParmLeftCursor(self.__buffer) - self._cursor_left = CursorLeft(self.__buffer) - self._parm_down_cursor = ParmDownCursor(self.__buffer) - self._cursor_down = CursorDown(self.__buffer) - self._parm_right_cursor = ParmRightCursor(self.__buffer) - self._cursor_right = CursorRight(self.__buffer) - self._cursor_address = CursorAddress(self.__buffer) - self._parm_up_cursor = ParmUpCursor(self.__buffer) - self._cursor_up = CursorUp(self.__buffer) - self._parm_delete_character = ParmDeleteCharacter(self.__buffer) - self._delete_character = DeleteCharacter(self.__buffer) - self._clear_eol = ClearEol(self.__buffer) - self._column_address = ColumnAddress(self.__buffer) - self._parm_insert_character = ParmInsertCharacter(self.__buffer) - self._insert_character = InsertCharacter(self.__buffer) - self._scroll_forward = ScrollForward(self.__buffer) - self._scroll_reverse = ScrollReverse(self.__buffer) - self._keypad_local = KeypadLocal(self.__buffer) - self._keypad_xmit = KeypadXmit(self.__buffer) + def _my_getstr(cap, optional=0): + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) self.__setup_movement() @@ -306,20 +237,20 @@ def refresh(self, screen, c_xy): newscr = screen[offset : offset + height] # use hardware scrolling if we have it. - if old_offset > offset and self._scroll_reverse.supported: + if old_offset > offset and self._ri: self.__hide_cursor() - self._cursor_address(0, 0) + self.__write_code(self._cup, 0, 0) self.__posxy = 0, old_offset for i in range(old_offset - offset): - self._scroll_reverse() + self.__write_code(self._ri) oldscr.pop(-1) oldscr.insert(0, "") - elif old_offset < offset and self._scroll_forward.supported: + elif old_offset < offset and self._ind: self.__hide_cursor() - self._cursor_address(self.height - 1, 0) + self.__write_code(self._cup, self.height - 1, 0) self.__posxy = 0, old_offset + self.height - 1 for i in range(offset - old_offset): - self._scroll_forward() + self.__write_code(self._ind) oldscr.pop(0) oldscr.append("") @@ -338,7 +269,7 @@ def refresh(self, screen, c_xy): self.__hide_cursor() self.__move(0, y) self.__posxy = 0, y - self._clear_eol() + self.__write_code(self._el) y += 1 self.__show_cursor() @@ -366,6 +297,7 @@ def prepare(self): """ Prepare the console for input/output operations. """ + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) raw.oflag &= ~(termios.OPOST) @@ -381,14 +313,14 @@ def prepare(self): self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer.clear() + self.__buffer = [] self.__posxy = 0, 0 self.__gone_tall = 0 self.__move = self.__move_short self.__offset = 0 - self._keypad_xmit() + self.__maybe_write_code(self._smkx) try: self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) @@ -399,7 +331,7 @@ def restore(self): """ Restore the console to the default state """ - self._keypad_local() + self.__maybe_write_code(self._rmkx) self.flushoutput() tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) @@ -506,7 +438,12 @@ def flushoutput(self): """ Flush the output buffer. """ - self.__buffer.flush() + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] def finish(self): """ @@ -523,7 +460,7 @@ def beep(self): """ Emit a beep sound. """ - self._bell() + self.__maybe_write_code(self._bel) self.flushoutput() if FIONREAD: @@ -576,7 +513,7 @@ def clear(self): """ Clear the console screen. """ - self._clear_screen() + self.__write_code(self._clear) self.__gone_tall = 1 self.__move = self.__move_tall self.__posxy = 0, 0 @@ -586,39 +523,41 @@ def __setup_movement(self): """ Set up the movement functions based on the terminal capabilities. """ - if self._parm_left_cursor.supported and self._parm_right_cursor.supported: - self.__move_x = self.__move_x_parm_cursor - elif self._cursor_left.supported and self._cursor_right.supported: - self.__move_x = self.__move_x_cursor + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 else: - raise InvalidTerminal("insufficient terminal (horizontal)") + raise RuntimeError("insufficient terminal (horizontal)") - if self._parm_up_cursor.supported and self._parm_down_cursor.supported: - self.__move_y = self.__move_y_parm_cursor - elif self._cursor_up.supported and self._cursor_down.supported: - self.__move_y = self.__move_y_cursor + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 else: - raise InvalidTerminal("insufficient terminal (vertical)") + raise RuntimeError("insufficient terminal (vertical)") - if self._delete_character.supported: - self.delete_character = self._delete_character - elif self._parm_delete_character.supported: - self.delete_character = functools.partial(self._parm_delete_character, 1) + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) else: - self.delete_character = None + self.dch1 = None - if self._insert_character.supported: - self.insert_character = self._insert_character - elif self._parm_insert_character.supported: - self.insert_character = functools.partial(self._parm_insert_character, 1) + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) else: - self.insert_character = None + self.ich1 = None self.__move = self.__move_short def __write_changed_line(self, y, oldline, newline, px): # this is frustrating; there's no reason to test (say) - # self.delete_character inside the loop -- but alternative ways of + # self.dch1 inside the loop -- but alternative ways of # structuring this function are equally painful (I'm trying to # avoid writing code generators these days...) x = 0 @@ -629,7 +568,7 @@ def __write_changed_line(self, y, oldline, newline, px): # sequene while x < minlen and oldline[x] == newline[x] and newline[x] != "\x1b": x += 1 - if oldline[x:] == newline[x + 1 :] and self.insert_character is not None: + if oldline[x:] == newline[x + 1 :] and self.ich1: if ( y == self.__posxy[1] and x > self.__posxy[0] @@ -637,7 +576,7 @@ def __write_changed_line(self, y, oldline, newline, px): ): x = px self.__move(x, y) - self.insert_character() + self.__write_code(self.ich1) self.__write(newline[x]) self.__posxy = x + 1, y elif x < minlen and oldline[x + 1 :] == newline[x + 1 :]: @@ -645,8 +584,8 @@ def __write_changed_line(self, y, oldline, newline, px): self.__write(newline[x]) self.__posxy = x + 1, y elif ( - self.delete_character is not None - and self.insert_character is not None + self.dch1 + and self.ich1 and len(newline) == self.width and x < len(newline) - 2 and newline[x + 1 : -1] == oldline[x:-2] @@ -654,16 +593,16 @@ def __write_changed_line(self, y, oldline, newline, px): self.__hide_cursor() self.__move(self.width - 2, y) self.__posxy = self.width - 2, y - self.delete_character() + self.__write_code(self.dch1) self.__move(x, y) - self.insert_character() + self.__write_code(self.ich1) self.__write(newline[x]) self.__posxy = x + 1, y else: self.__hide_cursor() self.__move(x, y) if len(oldline) > len(newline): - self._clear_eol() + self.__write_code(self._el) self.__write(newline[x:]) self.__posxy = len(newline), y @@ -674,39 +613,46 @@ def __write_changed_line(self, y, oldline, newline, px): self.move_cursor(0, y) def __write(self, text): - self.__buffer.push(text) + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) - def __move_y_cursor(self, y): + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): dy = y - self.__posxy[1] if dy > 0: - self._cursor_down(dy) + self.__write_code(dy * self._cud1) elif dy < 0: - self._cursor_up(-dy) + self.__write_code((-dy) * self._cuu1) - def __move_y_parm_cursor(self, y): + def __move_y_cuu_cud(self, y): dy = y - self.__posxy[1] if dy > 0: - self._parm_down_cursor(dy) + self.__write_code(self._cud, dy) elif dy < 0: - self._parm_up_cursor(-dy) + self.__write_code(self._cuu, -dy) - def __move_x_column_address(self, x): + def __move_x_hpa(self, x): if x != self.__posxy[0]: - self._column_address(x) + self.__write_code(self._hpa, x) - def __move_x_cursor(self, x): + def __move_x_cub1_cuf1(self, x): dx = x - self.__posxy[0] if dx > 0: - self._cursor_right(dx) + self.__write_code(self._cuf1 * dx) elif dx < 0: - self._cursor_left(-dx) + self.__write_code(self._cub1 * (-dx)) - def __move_x_parm_cursor(self, x): + def __move_x_cub_cuf(self, x): dx = x - self.__posxy[0] if dx > 0: - self._parm_right_cursor(dx) + self.__write_code(self._cuf, dx) elif dx < 0: - self._parm_left_cursor(-dx) + self.__write_code(self._cub, -dx) def __move_short(self, x, y): self.__move_x(x) @@ -714,7 +660,7 @@ def __move_short(self, x, y): def __move_tall(self, x, y): assert 0 <= y - self.__offset < self.height, y - self.__offset - self._cursor_address(y - self.__offset, x) + self.__write_code(self._cup, y - self.__offset, x) def __sigwinch(self, signum, frame): self.height, self.width = self.getheightwidth() @@ -722,12 +668,12 @@ def __sigwinch(self, signum, frame): def __hide_cursor(self): if self.cursor_visible: - self._cursor_invsible() + self.__maybe_write_code(self._civis) self.cursor_visible = 0 def __show_cursor(self): if not self.cursor_visible: - self._cursor_normal() + self.__maybe_write_code(self._cnorm) self.cursor_visible = 1 def repaint_prep(self): @@ -741,3 +687,30 @@ def repaint_prep(self): self.__move(0, self.__offset) ns = self.height * ["\000" * self.width] self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) From 977e79ececae977cc105acaa2f358a8d835bae5c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 10:40:26 +0200 Subject: [PATCH 40/78] Fix cursor position for double-width characters --- Lib/_pyrepl/reader.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 9c93e0830d4cac..d41fb797f54741 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -38,6 +38,13 @@ from .types import Callback, SimpleContextManager, KeySpec, CommandName +def str_width(c: str) -> int: + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + def disp_str(buffer: str) -> tuple[str, list[int]]: """disp_str(buffer:string) -> (string, [int]) @@ -57,7 +64,7 @@ def disp_str(buffer: str) -> tuple[str, list[int]]: c = r"\u%04x" % ord(c) s.append(c) b.append(1) - b.extend([0] * (len(c) - 1)) + b.extend([0] * (str_width(c) - 1)) return "".join(s), b @@ -260,7 +267,7 @@ def calc_screen(self) -> list[str]: screeninfo: list[tuple[int, list[int]]] = [] w = self.console.width - 1 p = self.pos - for ln, line in zip(range(len(lines)), lines): + for ln, line in enumerate(lines): ll = len(line) if 0 <= p <= ll: if self.msg and not self.msg_at_bottom: From 05b1142b0110e5a996718e8b910825cf13c1cb9e Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 10:50:00 +0200 Subject: [PATCH 41/78] Add _pyrepl to installed Lib subdirs --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 0e52e10602cf85..393c27f17a652d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2335,6 +2335,7 @@ LIBSUBDIRS= asyncio \ xmlrpc \ zipfile zipfile/_path \ zoneinfo \ + _pyrepl \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ From bc31d3adda3b8bee803453d442e0dec7dda9c905 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 12:45:37 +0200 Subject: [PATCH 42/78] Add tests --- Lib/test/test_pyrepl.py | 106 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 0898335219e547..807bee198f86fb 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -176,6 +176,112 @@ def test_right_arrow(self): ) +class TestCursorPosition(TestCase): + def prepare_reader(self, events): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + + reader = ReadlineAlikeReader(console) + reader.config = ReadlineConfig() + reader.more_lines = partial(more_lines, namespace=None) + return reader, console + + def handle_all_events(self, events): + reader, _ = self.prepare_reader(events) + try: + while True: + reader.handle1() + except StopIteration: + pass + return reader + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader = self.handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 3 for prompt, 1 for space, 1 for simple character + self.assertEqual(reader.pos2xy(reader.pos), (5, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader = self.handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 3 for prompt, 1 for space, 2 for wide character + self.assertEqual(reader.pos2xy(reader.pos), (6, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain(code_to_events("樂"), [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ]) + + reader = self.handle_all_events(events) + self.assertEqual(reader.pos, 0) + + # 3 for prompt, 1 for space + self.assertEqual(reader.pos2xy(reader.pos), (4, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain(code_to_events("樂"), [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ]) + + reader = self.handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 3 for prompt, 1 for space, 2 for wide character + self.assertEqual(reader.pos2xy(reader.pos), (6, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + events = itertools.chain(code_to_events(f"{for_loop}\n ' 可口可乐; 可口可樂'"), [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ]) + + reader = self.handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.pos2xy(reader.pos), (4 + len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + events = itertools.chain(code_to_events(f"{for_loop}\n ' 可口可乐; 可口可樂'"), [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ]) + + reader = self.handle_all_events(events) + + # cursor her (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + # TODO: Would we like the cursor to go back to end of line 2? + self.assertEqual(reader.pos, 23) + self.assertEqual(reader.pos2xy(reader.pos), (20, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain(code_to_events("' 可口可乐; 可口可樂'"), [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ]) + + reader = self.handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 3 for prompt, 1 for space, 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.pos2xy(reader.pos), (20, 0)) + + + class TestPyReplOutput(TestCase): def prepare_reader(self, events): console = FakeConsole(events) From 373a8a01b4d86192f306cba2d466b38b615a4058 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 30 Apr 2024 12:35:33 +0100 Subject: [PATCH 43/78] Update Lib/test/test_pyrepl.py --- Lib/test/test_pyrepl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 807bee198f86fb..aefc1c3a736190 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -259,7 +259,7 @@ def test_cursor_position_double_width_characters_move_up_down(self): reader = self.handle_all_events(events) - # cursor her (showing 2nd line only): + # cursor here (showing 2nd line only): # < ' 可口可乐; 可口可樂'> # ^ # TODO: Would we like the cursor to go back to end of line 2? From cba260fb60050c89e9ee4114911b0e19fafc6d0a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 30 Apr 2024 13:53:03 +0100 Subject: [PATCH 44/78] Implement better fallback with PYTHON_OLD_REPL --- Lib/_pyrepl/__main__.py | 49 +++++-------------------------------- Modules/main.c | 3 ++- Python/clinic/sysmodule.c.h | 20 ++++++++++++++- Python/sysmodule.c | 16 ++++++++++++ 4 files changed, 43 insertions(+), 45 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index d08f2d2d514fe0..72a3efbbc3218b 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,8 +1,6 @@ import os import sys - - def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): startup_path = os.getenv("PYTHONSTARTUP") if pythonstartup and startup_path: @@ -18,55 +16,20 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): if not hasattr(sys, "ps2"): sys.ps2 = "... " # - run_interactive = run_simple_interactive_console + run_interactive = None try: if not os.isatty(sys.stdin.fileno()): - # Bail out if stdin is not tty-like, as pyrepl wouldn't be happy - # For example, with: - # subprocess.Popen(['pypy', '-i'], stdin=subprocess.PIPE) raise ImportError from .simple_interact import check - if not check(): raise ImportError from .simple_interact import run_multiline_interactive_console - run_interactive = run_multiline_interactive_console - except SyntaxError: - print("Warning: 'import pyrepl' failed with SyntaxError") - run_interactive(mainmodule) - - -def run_simple_interactive_console(mainmodule): - import code - - if mainmodule is None: - import __main__ as mainmodule - console = code.InteractiveConsole(mainmodule.__dict__, filename="") - # some parts of code.py are copied here because it was impossible - # to start an interactive console without printing at least one line - # of banner. This was fixed in 3.4; but then from 3.6 it prints a - # line when exiting. This can be disabled too---by passing an argument - # that doesn't exist in <= 3.5. So, too much mess: just copy the code. - more = 0 - while 1: - try: - if more: - prompt = getattr(sys, "ps2", "... ") - else: - prompt = getattr(sys, "ps1", ">>> ") - try: - line = input(prompt) - except EOFError: - console.write("\n") - break - else: - more = console.push(line) - except KeyboardInterrupt: - console.write("\nKeyboardInterrupt\n") - console.resetbuffer() - more = 0 - + except Exception as e: + print(f"Warning: 'import _pyrepl' failed with '{e}'", sys.stderr) + if run_interactive is None: + return sys._baserepl() + return run_interactive(mainmodule) if __name__ == "__main__": interactive_console() diff --git a/Modules/main.c b/Modules/main.c index 1eecf479a552d4..e33d3c64a8e33a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -513,7 +513,8 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - if (!isatty(fileno(stdin))) { + if (!isatty(fileno(stdin)) || _Py_GetEnv(config->use_environment, "PYTHON_OLD_REPL")) + { PyCompilerFlags cf = _PyCompilerFlags_INIT; int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); return (run != 0); diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 31f66e807a8547..b61de7f5e90808 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1485,6 +1485,24 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(sys__baserepl__doc__, +"_baserepl($module, /)\n" +"--\n" +"\n" +"Private function for getting the base REPL"); + +#define SYS__BASEREPL_METHODDEF \ + {"_baserepl", (PyCFunction)sys__baserepl, METH_NOARGS, sys__baserepl__doc__}, + +static PyObject * +sys__baserepl_impl(PyObject *module); + +static PyObject * +sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__baserepl_impl(module); +} + #ifndef SYS_GETWINDOWSVERSION_METHODDEF #define SYS_GETWINDOWSVERSION_METHODDEF #endif /* !defined(SYS_GETWINDOWSVERSION_METHODDEF) */ @@ -1528,4 +1546,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=518424ee03e353b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=10609bc672b7f884 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7af363678e8e86..ab5719c4af826c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2391,6 +2391,21 @@ sys__get_cpu_count_config_impl(PyObject *module) return config->cpu_count; } +/*[clinic input] +sys._baserepl + +Private function for getting the base REPL +[clinic start generated code]*/ + +static PyObject * +sys__baserepl_impl(PyObject *module) +/*[clinic end generated code: output=f19a36375ebe0a45 input=7ca599425413e734]*/ +{ + PyCompilerFlags cf = _PyCompilerFlags_INIT; + PyRun_AnyFileExFlags(stdin, "", 0, &cf); + Py_RETURN_NONE; +} + static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2560,6 +2575,7 @@ static PyMethodDef sys_methods[] = { SYS_UNRAISABLEHOOK_METHODDEF SYS_GET_INT_MAX_STR_DIGITS_METHODDEF SYS_SET_INT_MAX_STR_DIGITS_METHODDEF + SYS__BASEREPL_METHODDEF #ifdef Py_STATS SYS__STATS_ON_METHODDEF SYS__STATS_OFF_METHODDEF From 233da022b0fc192e6e066e13d90a04f0483f8363 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 30 Apr 2024 14:06:16 +0100 Subject: [PATCH 45/78] Cache failures in pyrepl --- Lib/_pyrepl/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 72a3efbbc3218b..67d456330b35d8 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,7 +1,13 @@ import os import sys +CAN_USE_PYREPL = True + def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + global CAN_USE_PYREPL + if not CAN_USE_PYREPL: + return sys._baserepl() + startup_path = os.getenv("PYTHONSTARTUP") if pythonstartup and startup_path: import tokenize @@ -27,6 +33,7 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): run_interactive = run_multiline_interactive_console except Exception as e: print(f"Warning: 'import _pyrepl' failed with '{e}'", sys.stderr) + CAN_USE_PYREPL = False if run_interactive is None: return sys._baserepl() return run_interactive(mainmodule) From 07695f7a1984e8d48217d7e486917be8448726af Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 15:07:55 +0200 Subject: [PATCH 46/78] Stay at eol when moving up/down --- Lib/_pyrepl/commands.py | 21 ++++++++++-- Lib/test/test_pyrepl.py | 73 ++++++++++++++++++++++++++++++++--------- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index fd25988b674aac..0d5e240978f1b0 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -235,8 +235,9 @@ def do(self) -> None: class up(MotionCommand): def do(self) -> None: r = self.reader - for i in range(r.get_arg()): + for _ in range(r.get_arg()): bol1 = r.bol() + eol1 = r.eol() if bol1 == 0: if r.historyi > 0: r.select_item(r.historyi - 1) @@ -246,7 +247,14 @@ def do(self) -> None: return bol2 = r.bol(bol1 - 1) line_pos = r.pos - bol1 - if line_pos > bol1 - bol2 - 1: + + # Adjusting `pos` instead of (x, y) coordinates means that moving + # between lines that have mixed single-width and double-width + # characters does not align. Most editors do this, so leave it as-is. + if line_pos > bol1 - bol2 - 1 or ( + # If at end-of-non-empty-line, move to end-of-line + r.pos == eol1 and any(not i.isspace() for i in r.buffer[bol1:]) + ): r.pos = bol1 - 1 else: r.pos = bol2 + line_pos @@ -268,7 +276,14 @@ def do(self) -> None: r.error("end of buffer") return eol2 = r.eol(eol1 + 1) - if r.pos - bol1 > eol2 - eol1 - 1: + + # Adjusting `pos` instead of (x, y) coordinates means that moving + # between lines that have mixed single-width and double-width + # characters does not align. Most editors do this, so leave it as-is. + if r.pos - bol1 > eol2 - eol1 - 1 or ( + # If at end-of-non-empty-line, move to end-of-line + r.pos == eol1 and any(not i.isspace() for i in r.buffer[bol1:]) + ): r.pos = eol2 else: r.pos = eol1 + (r.pos - bol1) + 1 diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index aefc1c3a736190..ec1fb40789d24a 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -186,6 +186,12 @@ def prepare_reader(self, events): reader = ReadlineAlikeReader(console) reader.config = ReadlineConfig() reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) return reader, console def handle_all_events(self, events): @@ -203,8 +209,8 @@ def test_cursor_position_simple_character(self): reader = self.handle_all_events(events) self.assertEqual(reader.pos, 1) - # 3 for prompt, 1 for space, 1 for simple character - self.assertEqual(reader.pos2xy(reader.pos), (5, 0)) + # 1 for simple character + self.assertEqual(reader.pos2xy(reader.pos), (1, 0)) def test_cursor_position_double_width_character(self): events = itertools.chain(code_to_events("樂")) @@ -212,8 +218,8 @@ def test_cursor_position_double_width_character(self): reader = self.handle_all_events(events) self.assertEqual(reader.pos, 1) - # 3 for prompt, 1 for space, 2 for wide character - self.assertEqual(reader.pos2xy(reader.pos), (6, 0)) + # 2 for wide character + self.assertEqual(reader.pos2xy(reader.pos), (2, 0)) def test_cursor_position_double_width_character_move_left(self): events = itertools.chain(code_to_events("樂"), [ @@ -222,9 +228,7 @@ def test_cursor_position_double_width_character_move_left(self): reader = self.handle_all_events(events) self.assertEqual(reader.pos, 0) - - # 3 for prompt, 1 for space - self.assertEqual(reader.pos2xy(reader.pos), (4, 0)) + self.assertEqual(reader.pos2xy(reader.pos), (0, 0)) def test_cursor_position_double_width_character_move_left_right(self): events = itertools.chain(code_to_events("樂"), [ @@ -235,8 +239,8 @@ def test_cursor_position_double_width_character_move_left_right(self): reader = self.handle_all_events(events) self.assertEqual(reader.pos, 1) - # 3 for prompt, 1 for space, 2 for wide character - self.assertEqual(reader.pos2xy(reader.pos), (6, 0)) + # 2 for wide character + self.assertEqual(reader.pos2xy(reader.pos), (2, 0)) def test_cursor_position_double_width_characters_move_up(self): for_loop = "for _ in _:" @@ -248,12 +252,13 @@ def test_cursor_position_double_width_characters_move_up(self): # cursor at end of first line self.assertEqual(reader.pos, len(for_loop)) - self.assertEqual(reader.pos2xy(reader.pos), (4 + len(for_loop), 0)) + self.assertEqual(reader.pos2xy(reader.pos), (len(for_loop), 0)) def test_cursor_position_double_width_characters_move_up_down(self): for_loop = "for _ in _:" events = itertools.chain(code_to_events(f"{for_loop}\n ' 可口可乐; 可口可樂'"), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), ]) @@ -261,10 +266,9 @@ def test_cursor_position_double_width_characters_move_up_down(self): # cursor here (showing 2nd line only): # < ' 可口可乐; 可口可樂'> - # ^ - # TODO: Would we like the cursor to go back to end of line 2? - self.assertEqual(reader.pos, 23) - self.assertEqual(reader.pos2xy(reader.pos), (20, 1)) + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.pos2xy(reader.pos), (14, 1)) def test_cursor_position_multiple_double_width_characters_move_left(self): events = itertools.chain(code_to_events("' 可口可乐; 可口可樂'"), [ @@ -276,10 +280,47 @@ def test_cursor_position_multiple_double_width_characters_move_left(self): reader = self.handle_all_events(events) self.assertEqual(reader.pos, 10) - # 3 for prompt, 1 for space, 1 for quote, 1 for space, 2 per wide character, + # 1 for quote, 1 for space, 2 per wide character, # 1 for semicolon, 1 for space, 2 per wide character - self.assertEqual(reader.pos2xy(reader.pos), (20, 0)) + self.assertEqual(reader.pos2xy(reader.pos), (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + for_loop = "for _ in _" + code = f"{for_loop}:\n hello\n h\n hel" + events = itertools.chain(code_to_events(code), [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ]) + + reader = self.handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.pos2xy(reader.pos), (len(for_loop), 0)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + code = f"for _ in _:\n hello\n h\n{last_line}" + events = itertools.chain(code_to_events(code), [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ]) + + reader = self.handle_all_events(events) + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.pos2xy(reader.pos), (len(last_line), 3)) class TestPyReplOutput(TestCase): From 63dabfdcebd24a4f7917050b8f45a10546bed8c0 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 15:18:53 +0200 Subject: [PATCH 47/78] Fix linter --- Python/sysmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ab5719c4af826c..6b7c136aeb2db1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2392,7 +2392,7 @@ sys__get_cpu_count_config_impl(PyObject *module) } /*[clinic input] -sys._baserepl +sys._baserepl Private function for getting the base REPL [clinic start generated code]*/ From 162252a8c25d171ea26d751107bc72c14001a55c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 30 Apr 2024 14:20:58 +0100 Subject: [PATCH 48/78] fix early errors --- Lib/_pyrepl/__main__.py | 2 +- Lib/site.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 67d456330b35d8..fd7314544b4757 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -32,7 +32,7 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): from .simple_interact import run_multiline_interactive_console run_interactive = run_multiline_interactive_console except Exception as e: - print(f"Warning: 'import _pyrepl' failed with '{e}'", sys.stderr) + print(f"Warning: 'import _pyrepl' failed with '{e}'", file=sys.stderr) CAN_USE_PYREPL = False if run_interactive is None: return sys._baserepl() diff --git a/Lib/site.py b/Lib/site.py index e33a961977372d..3d6477e9f65ea8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -486,6 +486,7 @@ def register_readline(): import readline import rlcompleter import _pyrepl.readline + import _pyrepl.unix_console except ImportError: return @@ -518,7 +519,7 @@ def register_readline(): readline.read_history_file(history) else: _pyrepl.readline.read_history_file(history) - except OSError: + except (OSError,* _pyrepl.unix_console._error): pass def write_history(): From 970fd856c2f3fd3421608a27b708ba7bb9cbdb09 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 30 Apr 2024 14:21:20 +0100 Subject: [PATCH 49/78] Fix CI --- .github/workflows/reusable-ubuntu.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index e6fbaaf74c5a4b..1faa4ebe3f7ddd 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -17,6 +17,8 @@ jobs: FORCE_COLOR: 1 OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 + TERMINFO: /bin/bash + TERM: linux steps: - uses: actions/checkout@v4 - name: Register gcc problem matcher From 3dcf7046d9e9396618558edaa080f27f2c833b0a Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 15:44:04 +0200 Subject: [PATCH 50/78] Fix paste mode when there are empty line in the middle Paste mode needs to be exited with F3 after pasting now. Co-authored-by: Pablo Galindo --- Lib/_pyrepl/commands.py | 2 +- Lib/_pyrepl/readline.py | 4 +++- Lib/test/test_pyrepl.py | 44 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index fd25988b674aac..65f1b478423e9b 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -432,5 +432,5 @@ def do(self) -> None: class paste_mode(Command): def do(self) -> None: - self.reader.paste_mode = True + self.reader.paste_mode = not self.reader.paste_mode self.reader.dirty = True diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 2f01b8c3030b42..bbc0cf69ff4a6e 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -219,8 +219,10 @@ def do(self): if not self.reader.paste_mode and indent: for i in range(prevlinestart, prevlinestart + indent): r.insert(r.buffer[i]) - else: + elif not self.reader.paste_mode: self.finish = 1 + else: + r.insert("\n") class backspace_dedent(commands.Command): diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index aefc1c3a736190..df996416232932 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -537,15 +537,53 @@ def test_paste(self): ' if x%2:\n' ' print(x)\n' ' else:\n' - ' pass\n\n' + ' pass\n' + ) + + events = itertools.chain([ + Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), + ], code_to_events(code), [ + Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), + ], code_to_events("\n")) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n' ) events = itertools.chain([ Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), - ], code_to_events(code)) + ], code_to_events(code), [ + Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), + ], code_to_events("\n")) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n\n' + ) + + expected = ( + 'def f():\n' + ' x = y\n' + ' ' + ) + + events = code_to_events(code) reader = self.prepare_reader(events) output = multiline_input(reader) - self.assertEqual(output, code[:-1]) + self.assertEqual(output, expected) def test_paste_not_in_paste_mode(self): input_code = ( From 755728d30623ff502a1b8813ad76d2db2854548b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 30 Apr 2024 16:02:02 +0200 Subject: [PATCH 51/78] Fix disp_str for control characters --- Lib/_pyrepl/reader.py | 7 +++++-- Lib/test/test_pyrepl.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index d41fb797f54741..62d476d93654e1 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -62,9 +62,12 @@ def disp_str(buffer: str) -> tuple[str, list[int]]: for c in buffer: if unicodedata.category(c).startswith("C"): c = r"\u%04x" % ord(c) + b.append(1) + b.extend([0] * (len(c) - 1)) + else: + b.append(1) + b.extend([0] * (str_width(c) - 1)) s.append(c) - b.append(1) - b.extend([0] * (str_width(c) - 1)) return "".join(s), b diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index aefc1c3a736190..854768e2340f6d 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -371,6 +371,12 @@ def test_history_search(self): output = multiline_input(reader) self.assertEqual(output, "1+1") + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + class TestPyReplCompleter(TestCase): def prepare_reader(self, events, namespace): From 121ce2b3025af5dc7c2b00c7e517b752a13acb1c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 30 Apr 2024 15:02:52 +0100 Subject: [PATCH 52/78] Fix mac CI --- .github/workflows/reusable-macos.yml | 1 + .github/workflows/reusable-ubuntu.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index dabeca8c81ece1..e3319f141bd7f5 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -22,6 +22,7 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux strategy: fail-fast: false matrix: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 1faa4ebe3f7ddd..021859179af7cc 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -17,7 +17,6 @@ jobs: FORCE_COLOR: 1 OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 - TERMINFO: /bin/bash TERM: linux steps: - uses: actions/checkout@v4 From ba26254debf4b303fc9bf6ddce497b9d2f6e9cd3 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 1 May 2024 17:22:10 +0200 Subject: [PATCH 53/78] Various fixes to handle wide characters correctly Many of the fixes incorporated from https://github.com/pypy/pyrepl/pull/34/files --- Lib/_pyrepl/reader.py | 128 ++++++++++++++++++------------------ Lib/_pyrepl/unix_console.py | 75 +++++++++++++-------- Lib/test/test_pyrepl.py | 18 ++--- 3 files changed, 122 insertions(+), 99 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 62d476d93654e1..6af9c437ab20bf 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -29,7 +29,7 @@ from . import commands, console, input -_r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]") +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") # types @@ -45,6 +45,13 @@ def str_width(c: str) -> int: return 2 +def wlen(s: str) -> int: + length = sum(str_width(i) for i in s) + + # remove lengths of any escape sequences + return length - sum(len(i) for i in ANSI_ESCAPE_SEQUENCE.findall(s)) + + def disp_str(buffer: str) -> tuple[str, list[int]]: """disp_str(buffer:string) -> (string, [int]) @@ -55,19 +62,15 @@ def disp_str(buffer: str) -> tuple[str, list[int]]: >>> disp_str(chr(3)) ('^C', [1, 0]) - the list always contains 0s or 1s at present; it could conceivably - go higher as and when unicode support happens.""" + """ b: list[int] = [] s: list[str] = [] for c in buffer: if unicodedata.category(c).startswith("C"): c = r"\u%04x" % ord(c) - b.append(1) - b.extend([0] * (len(c) - 1)) - else: - b.append(1) - b.extend([0] * (str_width(c) - 1)) s.append(c) + b.append(wlen(c)) + b.extend([0] * (len(c) - 1)) return "".join(s), b @@ -253,7 +256,7 @@ def __post_init__(self) -> None: self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) self.screeninfo = [(0, [0])] - self.cxy = self.pos2xy(self.pos) + self.cxy = self.pos2xy() self.lxy = (self.pos, 0) def collect_keymap(self): @@ -268,40 +271,47 @@ def calc_screen(self) -> list[str]: lines = self.get_unicode().split("\n") screen: list[str] = [] screeninfo: list[tuple[int, list[int]]] = [] - w = self.console.width - 1 - p = self.pos + width = self.console.width - 1 + pos = self.pos for ln, line in enumerate(lines): ll = len(line) - if 0 <= p <= ll: + if 0 <= pos <= ll: if self.msg and not self.msg_at_bottom: for mline in self.msg.split("\n"): screen.append(mline) screeninfo.append((0, [])) - self.lxy = p, ln - prompt = self.get_prompt(ln, ll >= p >= 0) + self.lxy = pos, ln + prompt = self.get_prompt(ln, ll >= pos >= 0) while "\n" in prompt: pre_prompt, _, prompt = prompt.partition("\n") screen.append(pre_prompt) screeninfo.append((0, [])) - p -= ll + 1 + pos -= ll + 1 prompt, lp = self.process_prompt(prompt) - if _can_colorize(): - prompt = _ANSIColors.BOLD_MAGENTA + prompt + _ANSIColors.RESET l, l2 = disp_str(line) - wrapcount = (len(l) + lp) // w + wrapcount = (wlen(l) + lp) // width if wrapcount == 0: screen.append(prompt + l) - screeninfo.append((lp, l2 + [1])) + screeninfo.append((lp, l2)) else: - screen.append(prompt + l[: w - lp] + "\\") - screeninfo.append((lp, l2[: w - lp])) - for i in range(-lp + w, -lp + wrapcount * w, w): - screen.append(l[i : i + w] + "\\") - screeninfo.append((0, l2[i : i + w])) - screen.append(l[wrapcount * w - lp :]) - screeninfo.append((0, l2[wrapcount * w - lp :] + [1])) + for i in range(wrapcount + 1): + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + post = "\\" if i != wrapcount else "" + after = [1] if i != wrapcount else [] + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] self.screeninfo = screeninfo - self.cxy = self.pos2xy(self.pos) + self.cxy = self.pos2xy() if self.msg and self.msg_at_bottom: for mline in self.msg.split("\n"): screen.append(mline) @@ -321,7 +331,7 @@ def process_prompt(self, prompt: str) -> tuple[str, int]: # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) out_prompt = "" - l = len(prompt) + l = wlen(prompt) pos = 0 while True: s = prompt.find("\x01", pos) @@ -333,11 +343,11 @@ def process_prompt(self, prompt: str) -> tuple[str, int]: # Found start and end brackets, subtract from string length l = l - (e - s + 1) keep = prompt[pos:s] - l -= sum(map(len, _r_csi_seq.findall(keep))) + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) out_prompt += keep + prompt[s + 1 : e] pos = e + 1 keep = prompt[pos:] - l -= sum(map(len, _r_csi_seq.findall(keep))) + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) out_prompt += keep return out_prompt, l @@ -412,20 +422,22 @@ def get_prompt(self, lineno, cursor_on_line) -> str: """Return what should be in the left-hand margin for line `lineno'.""" if self.arg is not None and cursor_on_line: - return "(arg: %s) " % self.arg - - if self.paste_mode: - return '(paste) ' - - if "\n" in self.buffer: + prompt = "(arg: %s) " % self.arg + elif self.paste_mode: + prompt = '(paste) ' + elif "\n" in self.buffer: if lineno == 0: - return self.ps2 + prompt = self.ps2 elif lineno == self.buffer.count("\n"): - return self.ps4 + prompt = self.ps4 else: - return self.ps3 + prompt = self.ps3 + else: + prompt = self.ps1 - return self.ps1 + if _can_colorize(): + prompt = f"{_ANSIColors.BOLD_MAGENTA}{prompt}{_ANSIColors.RESET}" + return prompt def push_input_trans(self, itrans) -> None: self.input_trans_stack.append(self.input_trans) @@ -434,31 +446,25 @@ def push_input_trans(self, itrans) -> None: def pop_input_trans(self) -> None: self.input_trans = self.input_trans_stack.pop() - def pos2xy(self, pos: int) -> tuple[int, int]: + def pos2xy(self) -> tuple[int, int]: """Return the x, y coordinates of position 'pos'.""" # this *is* incomprehensible, yes. y = 0 + pos = self.pos assert 0 <= pos <= len(self.buffer) if pos == len(self.buffer): y = len(self.screeninfo) - 1 p, l2 = self.screeninfo[y] - return p + len(l2) - 1, y - else: - for p, l2 in self.screeninfo: - l = l2.count(1) - if l > pos: - break - else: - pos -= l - y += 1 - c = 0 - i = 0 - while c < pos: - c += l2[i] - i += 1 - while l2[i] == 0: - i += 1 - return p + i, y + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + if l >= pos: + break + else: + pos -= l + 1 + y += 1 + return p + sum(l2[:pos]), y def insert(self, text) -> None: """Insert 'text' at the insertion point.""" @@ -468,7 +474,7 @@ def insert(self, text) -> None: def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" - self.cxy = self.pos2xy(self.pos) + self.cxy = self.pos2xy() self.console.move_cursor(*self.cxy) def after_command(self, cmd: Command) -> None: @@ -491,10 +497,6 @@ def prepare(self) -> None: self.dirty = True self.last_command = None self.calc_screen() - # self.screeninfo = [(0, [0])] - # self.cxy = self.pos2xy(self.pos) - # self.lxy = (self.pos, 0) - # XXX should kill ring be here? except BaseException: self.restore() raise diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 1130fd36e2c24f..3688146bc3a578 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -33,6 +33,7 @@ from . import curses from .console import Console, Event from .fancy_termios import tcgetattr, tcsetattr +from .reader import wlen from .trace import trace from .unix_eventqueue import EventQueue @@ -555,56 +556,76 @@ def __setup_movement(self): self.__move = self.__move_short - def __write_changed_line(self, y, oldline, newline, px): + def __write_changed_line(self, y, oldline, newline, px_coord): # this is frustrating; there's no reason to test (say) # self.dch1 inside the loop -- but alternative ways of # structuring this function are equally painful (I'm trying to # avoid writing code generators these days...) - x = 0 - minlen = min(len(oldline), len(newline)) - # + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: break + j += wlen(c) + px_pos += 1 + # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape # sequene - while x < minlen and oldline[x] == newline[x] and newline[x] != "\x1b": - x += 1 - if oldline[x:] == newline[x + 1 :] and self.ich1: + while x_coord < minlen and oldline[x_pos] == newline[x_pos] and newline[x_pos] != "\x1b": + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + character_width = wlen(newline[x_pos]) + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: if ( y == self.__posxy[1] - and x > self.__posxy[0] - and oldline[px:x] == newline[px + 1 : x + 1] + and x_coord > self.__posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] ): - x = px - self.__move(x, y) + x_pos = px_pos + x_coord = px_coord + self.__move(x_coord, y) self.__write_code(self.ich1) - self.__write(newline[x]) - self.__posxy = x + 1, y - elif x < minlen and oldline[x + 1 :] == newline[x + 1 :]: - self.__move(x, y) - self.__write(newline[x]) - self.__posxy = x + 1, y + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif x_coord < minlen and oldline[x_pos + 1 :] == newline[x_pos + 1 :] and wlen(oldline[x_pos]) == wlen(newline[x_pos]): + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line elif ( self.dch1 and self.ich1 - and len(newline) == self.width - and x < len(newline) - 2 - and newline[x + 1 : -1] == oldline[x:-2] + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] ): self.__hide_cursor() self.__move(self.width - 2, y) self.__posxy = self.width - 2, y self.__write_code(self.dch1) - self.__move(x, y) + + self.__move(x_coord, y) self.__write_code(self.ich1) - self.__write(newline[x]) - self.__posxy = x + 1, y + self.__write(newline[x_pos]) + self.__posxy = x_coord + 1, y + else: self.__hide_cursor() - self.__move(x, y) - if len(oldline) > len(newline): + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): self.__write_code(self._el) - self.__write(newline[x:]) - self.__posxy = len(newline), y + self.__write(newline[x_pos:]) + self.__posxy = wlen(newline), y if "\x1b" in newline: # ANSI escape characters are present, so we can't assume diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 04dd5dea271ac8..d95a6f4f7b948e 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -210,7 +210,7 @@ def test_cursor_position_simple_character(self): self.assertEqual(reader.pos, 1) # 1 for simple character - self.assertEqual(reader.pos2xy(reader.pos), (1, 0)) + self.assertEqual(reader.pos2xy(), (1, 0)) def test_cursor_position_double_width_character(self): events = itertools.chain(code_to_events("樂")) @@ -219,7 +219,7 @@ def test_cursor_position_double_width_character(self): self.assertEqual(reader.pos, 1) # 2 for wide character - self.assertEqual(reader.pos2xy(reader.pos), (2, 0)) + self.assertEqual(reader.pos2xy(), (2, 0)) def test_cursor_position_double_width_character_move_left(self): events = itertools.chain(code_to_events("樂"), [ @@ -228,7 +228,7 @@ def test_cursor_position_double_width_character_move_left(self): reader = self.handle_all_events(events) self.assertEqual(reader.pos, 0) - self.assertEqual(reader.pos2xy(reader.pos), (0, 0)) + self.assertEqual(reader.pos2xy(), (0, 0)) def test_cursor_position_double_width_character_move_left_right(self): events = itertools.chain(code_to_events("樂"), [ @@ -240,7 +240,7 @@ def test_cursor_position_double_width_character_move_left_right(self): self.assertEqual(reader.pos, 1) # 2 for wide character - self.assertEqual(reader.pos2xy(reader.pos), (2, 0)) + self.assertEqual(reader.pos2xy(), (2, 0)) def test_cursor_position_double_width_characters_move_up(self): for_loop = "for _ in _:" @@ -252,7 +252,7 @@ def test_cursor_position_double_width_characters_move_up(self): # cursor at end of first line self.assertEqual(reader.pos, len(for_loop)) - self.assertEqual(reader.pos2xy(reader.pos), (len(for_loop), 0)) + self.assertEqual(reader.pos2xy(), (len(for_loop), 0)) def test_cursor_position_double_width_characters_move_up_down(self): for_loop = "for _ in _:" @@ -268,7 +268,7 @@ def test_cursor_position_double_width_characters_move_up_down(self): # < ' 可口可乐; 可口可樂'> # ^ self.assertEqual(reader.pos, 22) - self.assertEqual(reader.pos2xy(reader.pos), (14, 1)) + self.assertEqual(reader.pos2xy(), (14, 1)) def test_cursor_position_multiple_double_width_characters_move_left(self): events = itertools.chain(code_to_events("' 可口可乐; 可口可樂'"), [ @@ -282,7 +282,7 @@ def test_cursor_position_multiple_double_width_characters_move_left(self): # 1 for quote, 1 for space, 2 per wide character, # 1 for semicolon, 1 for space, 2 per wide character - self.assertEqual(reader.pos2xy(reader.pos), (16, 0)) + self.assertEqual(reader.pos2xy(), (16, 0)) def test_cursor_position_move_up_to_eol(self): for_loop = "for _ in _" @@ -300,7 +300,7 @@ def test_cursor_position_move_up_to_eol(self): # h # hel self.assertEqual(reader.pos, len(for_loop)) - self.assertEqual(reader.pos2xy(reader.pos), (len(for_loop), 0)) + self.assertEqual(reader.pos2xy(), (len(for_loop), 0)) def test_cursor_position_move_down_to_eol(self): last_line = " hel" @@ -320,7 +320,7 @@ def test_cursor_position_move_down_to_eol(self): # h # hel self.assertEqual(reader.pos, len(code)) - self.assertEqual(reader.pos2xy(reader.pos), (len(last_line), 3)) + self.assertEqual(reader.pos2xy(), (len(last_line), 3)) class TestPyReplOutput(TestCase): From 691c75ead702462be45ace9855faf43d4e2eb645 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 2 May 2024 11:50:57 +0200 Subject: [PATCH 54/78] Fix vertical navigation with wide characters --- Lib/_pyrepl/commands.py | 53 ++++++++++++++++--------------------- Lib/_pyrepl/reader.py | 43 +++++++++++++++++++----------- Lib/_pyrepl/unix_console.py | 2 +- Lib/_pyrepl/utils.py | 18 +++++++++++++ Lib/test/test_pyrepl.py | 43 ++++++++++++++++++++++-------- 5 files changed, 101 insertions(+), 58 deletions(-) create mode 100644 Lib/_pyrepl/utils.py diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 82f924e8ed8db2..ea953887a789c2 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -236,38 +236,35 @@ class up(MotionCommand): def do(self) -> None: r = self.reader for _ in range(r.get_arg()): - bol1 = r.bol() - eol1 = r.eol() - if bol1 == 0: + x, y = r.pos2xy() + new_y = y - 1 + + if new_y < 0: if r.historyi > 0: r.select_item(r.historyi - 1) return r.pos = 0 r.error("start of buffer") return - bol2 = r.bol(bol1 - 1) - line_pos = r.pos - bol1 - - # Adjusting `pos` instead of (x, y) coordinates means that moving - # between lines that have mixed single-width and double-width - # characters does not align. Most editors do this, so leave it as-is. - if line_pos > bol1 - bol2 - 1 or ( - # If at end-of-non-empty-line, move to end-of-line - r.pos == eol1 and any(not i.isspace() for i in r.buffer[bol1:]) + + if ( + x > (new_x := r.max_column(new_y)) or # we're past the end of the previous line + x == r.max_column(y) and any(not i.isspace() for i in r.buffer[r.bol():]) # move between eols ): - r.pos = bol1 - 1 - else: - r.pos = bol2 + line_pos + x = new_x + + r.setpos_from_xy(x, new_y) class down(MotionCommand): def do(self) -> None: r = self.reader b = r.buffer - for i in range(r.get_arg()): - bol1 = r.bol() - eol1 = r.eol() - if eol1 == len(b): + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if new_y > r.max_row(): if r.historyi < len(r.history): r.select_item(r.historyi + 1) r.pos = r.eol(0) @@ -275,18 +272,14 @@ def do(self) -> None: r.pos = len(b) r.error("end of buffer") return - eol2 = r.eol(eol1 + 1) - - # Adjusting `pos` instead of (x, y) coordinates means that moving - # between lines that have mixed single-width and double-width - # characters does not align. Most editors do this, so leave it as-is. - if r.pos - bol1 > eol2 - eol1 - 1 or ( - # If at end-of-non-empty-line, move to end-of-line - r.pos == eol1 and any(not i.isspace() for i in r.buffer[bol1:]) + + if ( + x > (new_x := r.max_column(new_y)) or # we're past the end of the previous line + x == r.max_column(y) and any(not i.isspace() for i in r.buffer[r.bol():]) # move between eols ): - r.pos = eol2 - else: - r.pos = eol1 + (r.pos - bol1) + 1 + x = new_x + + r.setpos_from_xy(x, new_y) class left(MotionCommand): diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 6af9c437ab20bf..6675664b270531 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -28,8 +28,7 @@ from traceback import _can_colorize, _ANSIColors # type: ignore[attr-defined] from . import commands, console, input - -ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") +from .utils import ANSI_ESCAPE_SEQUENCE, wlen # types @@ -38,20 +37,6 @@ from .types import Callback, SimpleContextManager, KeySpec, CommandName -def str_width(c: str) -> int: - w = unicodedata.east_asian_width(c) - if w in ('N', 'Na', 'H', 'A'): - return 1 - return 2 - - -def wlen(s: str) -> int: - length = sum(str_width(i) for i in s) - - # remove lengths of any escape sequences - return length - sum(len(i) for i in ANSI_ESCAPE_SEQUENCE.findall(s)) - - def disp_str(buffer: str) -> tuple[str, list[int]]: """disp_str(buffer:string) -> (string, [int]) @@ -409,6 +394,13 @@ def eol(self, p: int | None = None) -> int: p += 1 return p + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, returning `default' if there is None. Defaults to 1. @@ -446,6 +438,25 @@ def push_input_trans(self, itrans) -> None: def pop_input_trans(self) -> None: self.input_trans = self.input_trans_stack.pop() + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordincates x, y""" + pos = 0 + i = 0 + while i < y: + pos += len(self.screeninfo[i][1]) - self.screeninfo[i][1].count(0) + 1 + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + def pos2xy(self) -> tuple[int, int]: """Return the x, y coordinates of position 'pos'.""" # this *is* incomprehensible, yes. diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 3688146bc3a578..21dac8d29df8a3 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -33,9 +33,9 @@ from . import curses from .console import Console, Event from .fancy_termios import tcgetattr, tcsetattr -from .reader import wlen from .trace import trace from .unix_eventqueue import EventQueue +from .utils import wlen class InvalidTerminal(RuntimeError): diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 00000000000000..cd1df7c49a216d --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,18 @@ +import re +import unicodedata + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +def str_width(c: str) -> int: + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + length = sum(str_width(i) for i in s) + + # remove lengths of any escape sequences + return length - sum(len(i) for i in ANSI_ESCAPE_SEQUENCE.findall(s)) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index d95a6f4f7b948e..c80a50ef32174e 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -105,8 +105,17 @@ class TestPyReplDriver(TestCase): def prepare_reader(self, events): console = MagicMock() console.get_event.side_effect = events + console.height = 100 + console.width = 80 + reader = ReadlineAlikeReader(console) reader.config = ReadlineConfig() + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + return reader, console def test_up_arrow(self): @@ -124,7 +133,7 @@ def test_up_arrow(self): with suppress(StopIteration): _ = multiline_input(reader) - console.move_cursor.assert_called_with(1, 3) + console.move_cursor.assert_called_with(2, 1) def test_down_arrow(self): code = ( @@ -141,7 +150,7 @@ def test_down_arrow(self): with suppress(StopIteration): _ = multiline_input(reader) - console.move_cursor.assert_called_with(1, 5) + console.move_cursor.assert_called_with(2, 2) def test_left_arrow(self): events = itertools.chain(code_to_events('11+11'), [ @@ -155,7 +164,7 @@ def test_left_arrow(self): console.move_cursor.assert_has_calls( [ - call(3, 1), + call(4, 0), ] ) @@ -171,7 +180,7 @@ def test_right_arrow(self): console.move_cursor.assert_has_calls( [ - call(4, 1), + call(5, 0), ] ) @@ -267,8 +276,8 @@ def test_cursor_position_double_width_characters_move_up_down(self): # cursor here (showing 2nd line only): # < ' 可口可乐; 可口可樂'> # ^ - self.assertEqual(reader.pos, 22) - self.assertEqual(reader.pos2xy(), (14, 1)) + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.pos2xy(), (10, 1)) def test_cursor_position_multiple_double_width_characters_move_left(self): events = itertools.chain(code_to_events("' 可口可乐; 可口可樂'"), [ @@ -285,8 +294,15 @@ def test_cursor_position_multiple_double_width_characters_move_left(self): self.assertEqual(reader.pos2xy(), (16, 0)) def test_cursor_position_move_up_to_eol(self): - for_loop = "for _ in _" - code = f"{for_loop}:\n hello\n h\n hel" + first_line = "for _ in _:" + second_line = " hello" + + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) events = itertools.chain(code_to_events(code), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), @@ -299,12 +315,17 @@ def test_cursor_position_move_up_to_eol(self): # hello # h # hel - self.assertEqual(reader.pos, len(for_loop)) - self.assertEqual(reader.pos2xy(), (len(for_loop), 0)) + self.assertEqual(reader.pos, len(first_line) + len(second_line) + 1) # +1 for newline + self.assertEqual(reader.pos2xy(), (len(second_line), 1)) def test_cursor_position_move_down_to_eol(self): last_line = " hel" - code = f"for _ in _:\n hello\n h\n{last_line}" + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) events = itertools.chain(code_to_events(code), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), From 980407484903a264a1ab99acd62ceb08aa1ef7b5 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 2 May 2024 14:43:45 +0200 Subject: [PATCH 55/78] Write more tests --- Lib/_pyrepl/reader.py | 5 +- Lib/test/test_pyrepl.py | 266 +++++++++++++++++++++++++--------------- 2 files changed, 172 insertions(+), 99 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 6675664b270531..67a1e50dd705fc 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -256,7 +256,6 @@ def calc_screen(self) -> list[str]: lines = self.get_unicode().split("\n") screen: list[str] = [] screeninfo: list[tuple[int, list[int]]] = [] - width = self.console.width - 1 pos = self.pos for ln, line in enumerate(lines): ll = len(line) @@ -274,7 +273,7 @@ def calc_screen(self) -> list[str]: pos -= ll + 1 prompt, lp = self.process_prompt(prompt) l, l2 = disp_str(line) - wrapcount = (wlen(l) + lp) // width + wrapcount = (wlen(l) + lp) // self.console.width if wrapcount == 0: screen.append(prompt + l) screeninfo.append((lp, l2)) @@ -284,7 +283,7 @@ def calc_screen(self) -> list[str]: index_to_wrap_before = 0 column = 0 for character_width in l2: - if column + character_width >= width - prelen: + if column + character_width >= self.console.width - prelen: break index_to_wrap_before += 1 column += character_width diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index c80a50ef32174e..d48bf5190ca4b9 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -3,14 +3,12 @@ import os import rlcompleter import sys +import unittest from code import InteractiveConsole -from contextlib import suppress from functools import partial -from io import BytesIO from unittest import TestCase -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch -import _pyrepl.unix_eventqueue as unix_eventqueue from _pyrepl.console import Console, Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.simple_interact import _strip_final_indent @@ -47,6 +45,54 @@ def code_to_events(code): yield Event(evt='key', data=c, raw=bytearray(c.encode('utf-8'))) +def prepare_mock_console(events, **kwargs): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def prepare_fake_console(**kwargs): + console = FakeConsole() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def prepare_reader(console, **kwargs): + reader = ReadlineAlikeReader(console) + reader.config = ReadlineConfig() + reader.config.readline_completer = None + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def handle_all_events(events, + prepare_console=prepare_mock_console, + prepare_reader=prepare_reader): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + return reader, console + + class FakeConsole(Console): def __init__(self, events, encoding="utf-8"): self.events = iter(events) @@ -101,121 +147,55 @@ def wait(self) -> None: pass -class TestPyReplDriver(TestCase): - def prepare_reader(self, events): - console = MagicMock() - console.get_event.side_effect = events - console.height = 100 - console.width = 80 - - reader = ReadlineAlikeReader(console) - reader.config = ReadlineConfig() - - def get_prompt(lineno, cursor_on_line) -> str: - return "" - - reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) - - return reader, console - - def test_up_arrow(self): +class TestCursorPosition(TestCase): + def test_up_arrow_simple(self): code = ( 'def f():\n' ' ...\n' ) events = itertools.chain(code_to_events(code), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), ]) - reader, console = self.prepare_reader(events) - - with suppress(StopIteration): - _ = multiline_input(reader) - - console.move_cursor.assert_called_with(2, 1) + reader, console = handle_all_events(events) + self.assertEqual(reader.pos2xy(), (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) - def test_down_arrow(self): + def test_down_arrow_end_of_input(self): code = ( 'def f():\n' ' ...\n' ) events = itertools.chain(code_to_events(code), [ Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), ]) - reader, console = self.prepare_reader(events) - - with suppress(StopIteration): - _ = multiline_input(reader) + reader, console = handle_all_events(events) + self.assertEqual(reader.pos2xy(), (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) - console.move_cursor.assert_called_with(2, 2) - - def test_left_arrow(self): + def test_left_arrow_simple(self): events = itertools.chain(code_to_events('11+11'), [ Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), ]) - reader, console = self.prepare_reader(events) + reader, console = handle_all_events(events) + self.assertEqual(reader.pos2xy(), (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) - _ = multiline_input(reader) - - console.move_cursor.assert_has_calls( - [ - call(4, 0), - ] - ) - - def test_right_arrow(self): + def test_right_arrow_end_of_line(self): events = itertools.chain(code_to_events('11+11'), [ Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), ]) - reader, console = self.prepare_reader(events) - - _ = multiline_input(reader) - - console.move_cursor.assert_has_calls( - [ - call(5, 0), - ] - ) - - -class TestCursorPosition(TestCase): - def prepare_reader(self, events): - console = MagicMock() - console.get_event.side_effect = events - console.height = 100 - console.width = 80 - - reader = ReadlineAlikeReader(console) - reader.config = ReadlineConfig() - reader.more_lines = partial(more_lines, namespace=None) - reader.paste_mode = True # Avoid extra indents - - def get_prompt(lineno, cursor_on_line) -> str: - return "" - - reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) - return reader, console - - def handle_all_events(self, events): - reader, _ = self.prepare_reader(events) - try: - while True: - reader.handle1() - except StopIteration: - pass - return reader + reader, console = handle_all_events(events) + self.assertEqual(reader.pos2xy(), (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) def test_cursor_position_simple_character(self): events = itertools.chain(code_to_events("k")) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 1) # 1 for simple character @@ -224,7 +204,7 @@ def test_cursor_position_simple_character(self): def test_cursor_position_double_width_character(self): events = itertools.chain(code_to_events("樂")) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 1) # 2 for wide character @@ -235,7 +215,7 @@ def test_cursor_position_double_width_character_move_left(self): Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 0) self.assertEqual(reader.pos2xy(), (0, 0)) @@ -245,7 +225,7 @@ def test_cursor_position_double_width_character_move_left_right(self): Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 1) # 2 for wide character @@ -257,7 +237,7 @@ def test_cursor_position_double_width_characters_move_up(self): Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) # cursor at end of first line self.assertEqual(reader.pos, len(for_loop)) @@ -271,7 +251,7 @@ def test_cursor_position_double_width_characters_move_up_down(self): Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) # cursor here (showing 2nd line only): # < ' 可口可乐; 可口可樂'> @@ -286,7 +266,7 @@ def test_cursor_position_multiple_double_width_characters_move_left(self): Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 10) # 1 for quote, 1 for space, 2 per wide character, @@ -308,7 +288,7 @@ def test_cursor_position_move_up_to_eol(self): Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) # Cursor should be at end of line 1, even though line 2 is shorter # for _ in _: @@ -333,7 +313,7 @@ def test_cursor_position_move_down_to_eol(self): Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), ]) - reader = self.handle_all_events(events) + reader, _ = handle_all_events(events) # Cursor should be at end of line 3, even though line 2 is shorter # for _ in _: @@ -343,6 +323,29 @@ def test_cursor_position_move_down_to_eol(self): self.assertEqual(reader.pos, len(code)) self.assertEqual(reader.pos2xy(), (len(last_line), 3)) + def test_cursor_position_multiple_mixed_lines_move_up(self): + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.pos2xy(), (15, 1)) + class TestPyReplOutput(TestCase): def prepare_reader(self, events): @@ -677,5 +680,76 @@ def test_paste_not_in_paste_mode(self): self.assertEqual(output, output_code) +class TestReaderScreen(TestCase): + handle_events_test_wrap = partial( + handle_all_events, + prepare_console=partial(prepare_mock_console, width=10) + ) + + def assert_screen_equals(self, reader, expected): + actual = reader.calc_screen() + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_wrap_wide_characters(self): + events = code_to_events(8*"a" + "樂") + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_wrap_three_lines(self): + events = code_to_events(20*"a") + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_wrap_three_lines_mixed_character(self): + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + events = code_to_events(code) + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + + def test_backspace(self): + events = itertools.chain(code_to_events("aaa"), [ + Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), + ]) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_wrap_removes_after_backspace(self): + events = itertools.chain(code_to_events(10 * "a"), [ + Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), + ]) + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, 9*"a") + + def test_wrap_removes_after_backspace(self): + events = itertools.chain(code_to_events(10 * "a"), [ + Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), + ]) + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, 9*"a") + + def test_backspace_in_second_line_after_wrap(self): + events = itertools.chain(code_to_events(11 * "a"), [ + Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), + ]) + reader, _ = self.handle_events_test_wrap(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + if __name__ == "__main__": unittest.main() From 8f3e713d82e8a456522cce08fe5df203cbec4abf Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 2 May 2024 14:54:40 +0200 Subject: [PATCH 56/78] Fix backspace in second line --- Lib/_pyrepl/unix_console.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 21dac8d29df8a3..7ca578dd77d518 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -579,8 +579,6 @@ def __write_changed_line(self, y, oldline, newline, px_coord): x_coord += wlen(newline[x_pos]) x_pos += 1 - character_width = wlen(newline[x_pos]) - # if we need to insert a single character right after the first detected change if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: if ( @@ -590,6 +588,7 @@ def __write_changed_line(self, y, oldline, newline, px_coord): ): x_pos = px_pos x_coord = px_coord + character_width = wlen(newline[x_pos]) self.__move(x_coord, y) self.__write_code(self.ich1) self.__write(newline[x_pos]) @@ -597,6 +596,7 @@ def __write_changed_line(self, y, oldline, newline, px_coord): # if it's a single character change in the middle of the line elif x_coord < minlen and oldline[x_pos + 1 :] == newline[x_pos + 1 :] and wlen(oldline[x_pos]) == wlen(newline[x_pos]): + character_width = wlen(newline[x_pos]) self.__move(x_coord, y) self.__write(newline[x_pos]) self.__posxy = x_coord + character_width, y @@ -614,10 +614,11 @@ def __write_changed_line(self, y, oldline, newline, px_coord): self.__posxy = self.width - 2, y self.__write_code(self.dch1) + character_width = wlen(newline[x_pos]) self.__move(x_coord, y) self.__write_code(self.ich1) self.__write(newline[x_pos]) - self.__posxy = x_coord + 1, y + self.__posxy = character_width + 1, y else: self.__hide_cursor() From abe9fd339af8e8ba2217e391bad1cb8ff71c1199 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 3 May 2024 16:39:22 +0200 Subject: [PATCH 57/78] Fixes for pos2xy & setpos_from_xy when wrapped line --- Lib/_pyrepl/reader.py | 17 +++++- Lib/test/test_pyrepl.py | 125 +++++++++++++++++++++++++++++----------- 2 files changed, 104 insertions(+), 38 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 67a1e50dd705fc..71e21c4b7a668d 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -442,7 +442,13 @@ def setpos_from_xy(self, x: int, y: int) -> None: pos = 0 i = 0 while i < y: - pos += len(self.screeninfo[i][1]) - self.screeninfo[i][1].count(0) + 1 + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer i += 1 j = 0 @@ -469,10 +475,15 @@ def pos2xy(self) -> tuple[int, int]: for p, l2 in self.screeninfo: l = len(l2) - l2.count(0) - if l >= pos: + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: break else: - pos -= l + 1 + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer y += 1 return p + sum(l2[:pos]), y diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index d48bf5190ca4b9..58027f4d029150 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -93,6 +93,12 @@ def handle_all_events(events, return reader, console +handle_events_narrow_console = partial( + handle_all_events, + prepare_console=partial(prepare_mock_console, width=10) + ) + + class FakeConsole(Console): def __init__(self, events, encoding="utf-8"): self.events = iter(events) @@ -158,7 +164,7 @@ def test_up_arrow_simple(self): ]) reader, console = handle_all_events(events) - self.assertEqual(reader.pos2xy(), (0, 1)) + self.assertEqual(reader.cxy, (0, 1)) console.move_cursor.assert_called_once_with(0, 1) def test_down_arrow_end_of_input(self): @@ -171,7 +177,7 @@ def test_down_arrow_end_of_input(self): ]) reader, console = handle_all_events(events) - self.assertEqual(reader.pos2xy(), (0, 2)) + self.assertEqual(reader.cxy, (0, 2)) console.move_cursor.assert_called_once_with(0, 2) def test_left_arrow_simple(self): @@ -180,7 +186,7 @@ def test_left_arrow_simple(self): ]) reader, console = handle_all_events(events) - self.assertEqual(reader.pos2xy(), (4, 0)) + self.assertEqual(reader.cxy, (4, 0)) console.move_cursor.assert_called_once_with(4, 0) def test_right_arrow_end_of_line(self): @@ -189,7 +195,7 @@ def test_right_arrow_end_of_line(self): ]) reader, console = handle_all_events(events) - self.assertEqual(reader.pos2xy(), (5, 0)) + self.assertEqual(reader.cxy, (5, 0)) console.move_cursor.assert_called_once_with(5, 0) def test_cursor_position_simple_character(self): @@ -199,7 +205,7 @@ def test_cursor_position_simple_character(self): self.assertEqual(reader.pos, 1) # 1 for simple character - self.assertEqual(reader.pos2xy(), (1, 0)) + self.assertEqual(reader.cxy, (1, 0)) def test_cursor_position_double_width_character(self): events = itertools.chain(code_to_events("樂")) @@ -208,7 +214,7 @@ def test_cursor_position_double_width_character(self): self.assertEqual(reader.pos, 1) # 2 for wide character - self.assertEqual(reader.pos2xy(), (2, 0)) + self.assertEqual(reader.cxy, (2, 0)) def test_cursor_position_double_width_character_move_left(self): events = itertools.chain(code_to_events("樂"), [ @@ -217,7 +223,7 @@ def test_cursor_position_double_width_character_move_left(self): reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 0) - self.assertEqual(reader.pos2xy(), (0, 0)) + self.assertEqual(reader.cxy, (0, 0)) def test_cursor_position_double_width_character_move_left_right(self): events = itertools.chain(code_to_events("樂"), [ @@ -229,7 +235,7 @@ def test_cursor_position_double_width_character_move_left_right(self): self.assertEqual(reader.pos, 1) # 2 for wide character - self.assertEqual(reader.pos2xy(), (2, 0)) + self.assertEqual(reader.cxy, (2, 0)) def test_cursor_position_double_width_characters_move_up(self): for_loop = "for _ in _:" @@ -241,7 +247,7 @@ def test_cursor_position_double_width_characters_move_up(self): # cursor at end of first line self.assertEqual(reader.pos, len(for_loop)) - self.assertEqual(reader.pos2xy(), (len(for_loop), 0)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) def test_cursor_position_double_width_characters_move_up_down(self): for_loop = "for _ in _:" @@ -257,7 +263,7 @@ def test_cursor_position_double_width_characters_move_up_down(self): # < ' 可口可乐; 可口可樂'> # ^ self.assertEqual(reader.pos, 19) - self.assertEqual(reader.pos2xy(), (10, 1)) + self.assertEqual(reader.cxy, (10, 1)) def test_cursor_position_multiple_double_width_characters_move_left(self): events = itertools.chain(code_to_events("' 可口可乐; 可口可樂'"), [ @@ -271,7 +277,7 @@ def test_cursor_position_multiple_double_width_characters_move_left(self): # 1 for quote, 1 for space, 2 per wide character, # 1 for semicolon, 1 for space, 2 per wide character - self.assertEqual(reader.pos2xy(), (16, 0)) + self.assertEqual(reader.cxy, (16, 0)) def test_cursor_position_move_up_to_eol(self): first_line = "for _ in _:" @@ -296,7 +302,7 @@ def test_cursor_position_move_up_to_eol(self): # h # hel self.assertEqual(reader.pos, len(first_line) + len(second_line) + 1) # +1 for newline - self.assertEqual(reader.pos2xy(), (len(second_line), 1)) + self.assertEqual(reader.cxy, (len(second_line), 1)) def test_cursor_position_move_down_to_eol(self): last_line = " hel" @@ -321,7 +327,7 @@ def test_cursor_position_move_down_to_eol(self): # h # hel self.assertEqual(reader.pos, len(code)) - self.assertEqual(reader.pos2xy(), (len(last_line), 3)) + self.assertEqual(reader.cxy, (len(last_line), 3)) def test_cursor_position_multiple_mixed_lines_move_up(self): code = ( @@ -344,7 +350,25 @@ def test_cursor_position_multiple_mixed_lines_move_up(self): # x = '可口可乐; 可口可樂' # ^ self.assertEqual(reader.pos, 22) - self.assertEqual(reader.pos2xy(), (15, 1)) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + code = ( + "def foo():\n" + " hello" + ) + events = itertools.chain(code_to_events(code), [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ]) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) class TestPyReplOutput(TestCase): @@ -680,40 +704,35 @@ def test_paste_not_in_paste_mode(self): self.assertEqual(output, output_code) -class TestReaderScreen(TestCase): - handle_events_test_wrap = partial( - handle_all_events, - prepare_console=partial(prepare_mock_console, width=10) - ) - +class TestReader(TestCase): def assert_screen_equals(self, reader, expected): actual = reader.calc_screen() expected = expected.split("\n") self.assertListEqual(actual, expected) - def test_wrap_simple(self): + def test_calc_screen_wrap_simple(self): events = code_to_events(10 * "a") - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{9*"a"}\\\na") - def test_wrap_wide_characters(self): + def test_calc_screen_wrap_wide_characters(self): events = code_to_events(8*"a" + "樂") - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") - def test_wrap_three_lines(self): + def test_calc_screen_wrap_three_lines(self): events = code_to_events(20*"a") - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") - def test_wrap_three_lines_mixed_character(self): + def test_calc_screen_wrap_three_lines_mixed_character(self): code = ( "def f():\n" f" {8*"a"}\n" f" {5*"樂"}" ) events = code_to_events(code) - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, ( "def f():\n" f" {7*"a"}\\\n" @@ -722,34 +741,70 @@ def test_wrap_three_lines_mixed_character(self): "樂樂" )) - def test_backspace(self): + def test_calc_screen_backspace(self): events = itertools.chain(code_to_events("aaa"), [ Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), ]) reader, _ = handle_all_events(events) self.assert_screen_equals(reader, "aa") - def test_wrap_removes_after_backspace(self): + def test_calc_screen_wrap_removes_after_backspace(self): events = itertools.chain(code_to_events(10 * "a"), [ Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), ]) - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, 9*"a") - def test_wrap_removes_after_backspace(self): + def test_calc_screen_wrap_removes_after_backspace(self): events = itertools.chain(code_to_events(10 * "a"), [ Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), ]) - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, 9*"a") - def test_backspace_in_second_line_after_wrap(self): + def test_calc_screen_backspace_in_second_line_after_wrap(self): events = itertools.chain(code_to_events(11 * "a"), [ Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), ]) - reader, _ = self.handle_events_test_wrap(events) + reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{9*"a"}\\\na") + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_setpos_from_xy_multiple_lines(self): + code = ( + "def foo():\n" + " return 1" + ) + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + code = ( + "def foo():\n" + " hello" + ) + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + code = ( + "def foo():\n" + " hello" + ) + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + if __name__ == "__main__": unittest.main() From 4f66170495b9bccf448d6768293063590bde313d Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 3 May 2024 17:31:12 +0200 Subject: [PATCH 58/78] Fix linter & run black --- Lib/_pyrepl/commands.py | 22 ++- Lib/_pyrepl/console.py | 20 +-- Lib/_pyrepl/reader.py | 11 +- Lib/test/test_pyrepl.py | 367 +++++++++++++++++++++++++++------------- 4 files changed, 276 insertions(+), 144 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index ea953887a789c2..38253d4742be0a 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -248,8 +248,14 @@ def do(self) -> None: return if ( - x > (new_x := r.max_column(new_y)) or # we're past the end of the previous line - x == r.max_column(y) and any(not i.isspace() for i in r.buffer[r.bol():]) # move between eols + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols ): x = new_x @@ -274,8 +280,14 @@ def do(self) -> None: return if ( - x > (new_x := r.max_column(new_y)) or # we're past the end of the previous line - x == r.max_column(y) and any(not i.isspace() for i in r.buffer[r.bol():]) # move between eols + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols ): x = new_x @@ -410,6 +422,7 @@ def do(self) -> None: class help(Command): def do(self) -> None: import _sitebuiltins + with self.reader.suspend(): self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] @@ -431,6 +444,7 @@ class show_history(Command): def do(self) -> None: from .pager import get_pager from site import gethistoryfile # type: ignore[attr-defined] + history = os.linesep.join(self.reader.history[:]) with self.reader.suspend(): pager = get_pager() diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 7f12f20e13e827..23fb711e2d8e05 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -37,24 +37,19 @@ class Console(ABC): """ @abstractmethod - def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: - ... + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... @abstractmethod - def prepare(self) -> None: - ... + def prepare(self) -> None: ... @abstractmethod - def restore(self) -> None: - ... + def restore(self) -> None: ... @abstractmethod - def move_cursor(self, x: int, y: int) -> None: - ... + def move_cursor(self, x: int, y: int) -> None: ... @abstractmethod - def set_cursor_vis(self, visible: bool) -> None: - ... + def set_cursor_vis(self, visible: bool) -> None: ... @abstractmethod def getheightwidth(self) -> tuple[int, int]: @@ -63,7 +58,7 @@ def getheightwidth(self) -> tuple[int, int]: ... @abstractmethod - def get_event(self, block: bool = True) -> Event: + def get_event(self, block: bool = True) -> Event: """Return an Event instance. Returns None if |block| is false and there is no event pending, otherwise waits for the completion of an event.""" @@ -77,8 +72,7 @@ def push_char(self, char: str) -> None: ... @abstractmethod - def beep(self) -> None: - ... + def beep(self) -> None: ... @abstractmethod def clear(self) -> None: diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 71e21c4b7a668d..ed565e55ed296c 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -78,11 +78,7 @@ def make_default_syntax_table() -> dict[str, int]: def make_default_commands() -> dict[CommandName, type[Command]]: result: dict[CommandName, type[Command]] = {} for v in vars(commands).values(): - if ( - isinstance(v, type) - and issubclass(v, Command) - and v.__name__[0].islower() - ): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): result[v.__name__] = v result[v.__name__.replace("_", "-")] = v return result @@ -152,6 +148,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ] ) + @dataclass(slots=True) class Reader: """The Reader class implements the bare bones of a command reader, @@ -415,7 +412,7 @@ def get_prompt(self, lineno, cursor_on_line) -> str: if self.arg is not None and cursor_on_line: prompt = "(arg: %s) " % self.arg elif self.paste_mode: - prompt = '(paste) ' + prompt = "(paste) " elif "\n" in self.buffer: if lineno == 0: prompt = self.ps2 @@ -539,7 +536,7 @@ def suspend(self) -> SimpleContextManager: self.restore() yield finally: - for arg in ('msg', 'ps1', 'ps2', 'ps3', 'ps4', 'paste_mode'): + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): setattr(self, arg, prev_state[arg]) self.prepare() pass diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 58027f4d029150..aaa126df798fdb 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -42,7 +42,7 @@ def multiline_input(reader, namespace=None): def code_to_events(code): for c in code: - yield Event(evt='key', data=c, raw=bytearray(c.encode('utf-8'))) + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) def prepare_mock_console(events, **kwargs): @@ -80,9 +80,9 @@ def get_prompt(lineno, cursor_on_line) -> str: return reader -def handle_all_events(events, - prepare_console=prepare_mock_console, - prepare_reader=prepare_reader): +def handle_all_events( + events, prepare_console=prepare_mock_console, prepare_reader=prepare_reader +): console = prepare_console(events) reader = prepare_reader(console) try: @@ -94,9 +94,8 @@ def handle_all_events(events, handle_events_narrow_console = partial( - handle_all_events, - prepare_console=partial(prepare_mock_console, width=10) - ) + handle_all_events, prepare_console=partial(prepare_mock_console, width=10) +) class FakeConsole(Console): @@ -155,44 +154,60 @@ def wait(self) -> None: class TestCursorPosition(TestCase): def test_up_arrow_simple(self): + # fmt: off code = ( 'def f():\n' ' ...\n' ) - events = itertools.chain(code_to_events(code), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ]) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) reader, console = handle_all_events(events) self.assertEqual(reader.cxy, (0, 1)) console.move_cursor.assert_called_once_with(0, 1) def test_down_arrow_end_of_input(self): + # fmt: off code = ( 'def f():\n' ' ...\n' ) - events = itertools.chain(code_to_events(code), [ - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ]) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) reader, console = handle_all_events(events) self.assertEqual(reader.cxy, (0, 2)) console.move_cursor.assert_called_once_with(0, 2) def test_left_arrow_simple(self): - events = itertools.chain(code_to_events('11+11'), [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - ]) + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) reader, console = handle_all_events(events) self.assertEqual(reader.cxy, (4, 0)) console.move_cursor.assert_called_once_with(4, 0) def test_right_arrow_end_of_line(self): - events = itertools.chain(code_to_events('11+11'), [ - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - ]) + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) reader, console = handle_all_events(events) self.assertEqual(reader.cxy, (5, 0)) @@ -217,19 +232,25 @@ def test_cursor_position_double_width_character(self): self.assertEqual(reader.cxy, (2, 0)) def test_cursor_position_double_width_character_move_left(self): - events = itertools.chain(code_to_events("樂"), [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - ]) + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 0) self.assertEqual(reader.cxy, (0, 0)) def test_cursor_position_double_width_character_move_left_right(self): - events = itertools.chain(code_to_events("樂"), [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - ]) + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 1) @@ -239,9 +260,20 @@ def test_cursor_position_double_width_character_move_left_right(self): def test_cursor_position_double_width_characters_move_up(self): for_loop = "for _ in _:" - events = itertools.chain(code_to_events(f"{for_loop}\n ' 可口可乐; 可口可樂'"), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ]) + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) reader, _ = handle_all_events(events) @@ -251,11 +283,22 @@ def test_cursor_position_double_width_characters_move_up(self): def test_cursor_position_double_width_characters_move_up_down(self): for_loop = "for _ in _:" - events = itertools.chain(code_to_events(f"{for_loop}\n ' 可口可乐; 可口可樂'"), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ]) + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) reader, _ = handle_all_events(events) @@ -266,11 +309,14 @@ def test_cursor_position_double_width_characters_move_up_down(self): self.assertEqual(reader.cxy, (10, 1)) def test_cursor_position_multiple_double_width_characters_move_left(self): - events = itertools.chain(code_to_events("' 可口可乐; 可口可樂'"), [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - ]) + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) reader, _ = handle_all_events(events) self.assertEqual(reader.pos, 10) @@ -282,17 +328,23 @@ def test_cursor_position_multiple_double_width_characters_move_left(self): def test_cursor_position_move_up_to_eol(self): first_line = "for _ in _:" second_line = " hello" - + + # fmt: off code = ( f"{first_line}\n" f"{second_line}\n" " h\n" " hel" ) - events = itertools.chain(code_to_events(code), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ]) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) reader, _ = handle_all_events(events) @@ -301,23 +353,32 @@ def test_cursor_position_move_up_to_eol(self): # hello # h # hel - self.assertEqual(reader.pos, len(first_line) + len(second_line) + 1) # +1 for newline + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline self.assertEqual(reader.cxy, (len(second_line), 1)) def test_cursor_position_move_down_to_eol(self): last_line = " hel" + + # fmt: off code = ( "for _ in _:\n" " hello\n" " h\n" f"{last_line}" ) - events = itertools.chain(code_to_events(code), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ]) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) reader, _ = handle_all_events(events) @@ -330,11 +391,14 @@ def test_cursor_position_move_down_to_eol(self): self.assertEqual(reader.cxy, (len(last_line), 3)) def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off code = ( "def foo():\n" " x = '可口可乐; 可口可樂'\n" " y = 'abckdfjskldfjslkdjf'" ) + # fmt: on + events = itertools.chain( code_to_events(code), 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], @@ -344,7 +408,7 @@ def test_cursor_position_multiple_mixed_lines_move_up(self): reader, _ = handle_all_events(events) # By moving left, we're before the s: - # y = 'abckdfjskldfjslkdjf' + # y = 'abckdfjskldfjslkdjf' # ^ # And we should move before the semi-colon despite the different offset # x = '可口可乐; 可口可樂' @@ -353,13 +417,19 @@ def test_cursor_position_multiple_mixed_lines_move_up(self): self.assertEqual(reader.cxy, (15, 1)) def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off code = ( "def foo():\n" " hello" ) - events = itertools.chain(code_to_events(code), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ]) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) reader, _ = handle_events_narrow_console(events) # The code looks like this: @@ -380,25 +450,28 @@ def prepare_reader(self, events): return reader def test_basic(self): - reader = self.prepare_reader(code_to_events('1+1\n')) + reader = self.prepare_reader(code_to_events("1+1\n")) output = multiline_input(reader) self.assertEqual(output, "1+1") def test_multiline_edit(self): - events = itertools.chain(code_to_events('def f():\n ...\n\n'), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), - Event(evt="key", data="g", raw=bytearray(b"g")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ]) + events = itertools.chain( + code_to_events("def f():\n ...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) reader = self.prepare_reader(events) output = multiline_input(reader) @@ -407,14 +480,17 @@ def test_multiline_edit(self): self.assertEqual(output, "def g():\n ...\n ") def test_history_navigation_with_up_arrow(self): - events = itertools.chain(code_to_events('1+1\n2+2\n'), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ]) + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) reader = self.prepare_reader(events) @@ -428,13 +504,16 @@ def test_history_navigation_with_up_arrow(self): self.assertEqual(output, "1+1") def test_history_navigation_with_down_arrow(self): - events = itertools.chain(code_to_events('1+1\n2+2\n'), [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ]) + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) reader = self.prepare_reader(events) @@ -442,12 +521,15 @@ def test_history_navigation_with_down_arrow(self): self.assertEqual(output, "1+1") def test_history_search(self): - events = itertools.chain(code_to_events('1+1\n2+2\n3+3\n'), [ - Event(evt="key", data="\x12", raw=bytearray(b"\x12")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ]) + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) reader = self.prepare_reader(events) @@ -476,7 +558,7 @@ def prepare_reader(self, events, namespace): return reader def test_simple_completion(self): - events = code_to_events('os.geten\t\n') + events = code_to_events("os.geten\t\n") namespace = {"os": os} reader = self.prepare_reader(events, namespace) @@ -485,7 +567,7 @@ def test_simple_completion(self): self.assertEqual(output, "os.getenv") def test_completion_with_many_options(self): - events = code_to_events('os.\t\tO_AS\t\n') + events = code_to_events("os.\t\tO_AS\t\n") namespace = {"os": os} reader = self.prepare_reader(events, namespace) @@ -494,7 +576,7 @@ def test_completion_with_many_options(self): self.assertEqual(output, "os.O_ASYNC") def test_empty_namespace_completion(self): - events = code_to_events('os.geten\t\n') + events = code_to_events("os.geten\t\n") namespace = {} reader = self.prepare_reader(events, namespace) @@ -502,7 +584,7 @@ def test_empty_namespace_completion(self): self.assertEqual(output, "os.geten") def test_global_namespace_completion(self): - events = code_to_events('py\t\n') + events = code_to_events("py\t\n") namespace = {"python": None} reader = self.prepare_reader(events, namespace) output = multiline_input(reader, namespace) @@ -626,6 +708,7 @@ def prepare_reader(self, events): return reader def test_paste(self): + # fmt: off code = ( 'def a():\n' ' for x in range(10):\n' @@ -634,34 +717,48 @@ def test_paste(self): ' else:\n' ' pass\n' ) + # fmt: on - events = itertools.chain([ - Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), - ], code_to_events(code), [ - Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), - ], code_to_events("\n")) + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, code) def test_paste_mid_newlines(self): + # fmt: off code = ( 'def f():\n' ' x = y\n' ' \n' ' y = z\n' ) + # fmt: on - events = itertools.chain([ - Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), - ], code_to_events(code), [ - Event(evt='key', data='f3', raw=bytearray(b'\x1bOR')), - ], code_to_events("\n")) + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, code) def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off code = ( 'def f():\n' ' x = y\n' @@ -674,6 +771,7 @@ def test_paste_mid_newlines_not_in_paste_mode(self): ' x = y\n' ' ' ) + # fmt: on events = code_to_events(code) reader = self.prepare_reader(events) @@ -681,6 +779,7 @@ def test_paste_mid_newlines_not_in_paste_mode(self): self.assertEqual(output, expected) def test_paste_not_in_paste_mode(self): + # fmt: off input_code = ( 'def a():\n' ' for x in range(10):\n' @@ -697,6 +796,7 @@ def test_paste_not_in_paste_mode(self): ' print(x)\n' ' else:' ) + # fmt: on events = code_to_events(input_code) reader = self.prepare_reader(events) @@ -716,23 +816,28 @@ def test_calc_screen_wrap_simple(self): self.assert_screen_equals(reader, f"{9*"a"}\\\na") def test_calc_screen_wrap_wide_characters(self): - events = code_to_events(8*"a" + "樂") + events = code_to_events(8 * "a" + "樂") reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") def test_calc_screen_wrap_three_lines(self): - events = code_to_events(20*"a") + events = code_to_events(20 * "a") reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off code = ( "def f():\n" f" {8*"a"}\n" f" {5*"樂"}" ) + # fmt: on + events = code_to_events(code) reader, _ = handle_events_narrow_console(events) + + # fmt: off self.assert_screen_equals(reader, ( "def f():\n" f" {7*"a"}\\\n" @@ -740,32 +845,45 @@ def test_calc_screen_wrap_three_lines_mixed_character(self): f" {3*"樂"}\\\n" "樂樂" )) + # fmt: on def test_calc_screen_backspace(self): - events = itertools.chain(code_to_events("aaa"), [ - Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), - ]) + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) reader, _ = handle_all_events(events) self.assert_screen_equals(reader, "aa") def test_calc_screen_wrap_removes_after_backspace(self): - events = itertools.chain(code_to_events(10 * "a"), [ - Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), - ]) + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, 9*"a") + self.assert_screen_equals(reader, 9 * "a") def test_calc_screen_wrap_removes_after_backspace(self): - events = itertools.chain(code_to_events(10 * "a"), [ - Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), - ]) + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, 9*"a") + self.assert_screen_equals(reader, 9 * "a") def test_calc_screen_backspace_in_second_line_after_wrap(self): - events = itertools.chain(code_to_events(11 * "a"), [ - Event(evt='key', data='backspace', raw=bytearray(b'\x7f')), - ]) + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, f"{9*"a"}\\\na") @@ -776,30 +894,39 @@ def test_setpos_for_xy_simple(self): self.assertEqual(reader.pos, 0) def test_setpos_from_xy_multiple_lines(self): + # fmt: off code = ( "def foo():\n" " return 1" ) + # fmt: on + events = code_to_events(code) reader, _ = handle_all_events(events) reader.setpos_from_xy(2, 1) self.assertEqual(reader.pos, 13) def test_setpos_from_xy_after_wrap(self): + # fmt: off code = ( "def foo():\n" " hello" ) + # fmt: on + events = code_to_events(code) reader, _ = handle_events_narrow_console(events) reader.setpos_from_xy(2, 2) self.assertEqual(reader.pos, 13) def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off code = ( "def foo():\n" " hello" ) + # fmt: on + events = code_to_events(code) reader, _ = handle_events_narrow_console(events) reader.setpos_from_xy(0, 1) From 4bc36ab0c31a0d1bd73f63de8c2a81ea8f7806ef Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 3 May 2024 17:35:50 +0200 Subject: [PATCH 59/78] Update Lib/_pyrepl/reader.py --- Lib/_pyrepl/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index ed565e55ed296c..e57bc263546451 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -435,7 +435,7 @@ def pop_input_trans(self) -> None: self.input_trans = self.input_trans_stack.pop() def setpos_from_xy(self, x: int, y: int) -> None: - """Set pos according to coordincates x, y""" + """Set pos according to coordinates x, y""" pos = 0 i = 0 while i < y: From b5168312a8600e2996dc01847c775edf6cc78138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 13:24:31 +0200 Subject: [PATCH 60/78] Add typing to consoles --- .github/workflows/mypy.yml | 4 ++- Lib/_pyrepl/_minimal_curses.py | 2 +- Lib/_pyrepl/commands.py | 8 +++-- Lib/_pyrepl/console.py | 24 ++++++++------- Lib/_pyrepl/curses.py | 7 ++--- Lib/_pyrepl/mypy.ini | 2 -- Lib/_pyrepl/reader.py | 30 +++++++++---------- Lib/_pyrepl/trace.py | 14 +++++---- Lib/_pyrepl/unix_console.py | 53 +++++++++++++++++++--------------- Lib/_pyrepl/unix_eventqueue.py | 22 +++++++------- Lib/test/test_pyrepl.py | 7 +++-- 11 files changed, 94 insertions(+), 79 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b766785de405d2..35996f237814ba 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_pyrepl/**" - "Lib/test/libregrtest/**" - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" @@ -35,8 +36,9 @@ jobs: strategy: matrix: target: [ + "Lib/_pyrepl", "Lib/test/libregrtest", - "Tools/build/", + "Tools/build", "Tools/cases_generator", "Tools/clinic", "Tools/jit", diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py index 7cff7d7ca2ea51..0f067b52095461 100644 --- a/Lib/_pyrepl/_minimal_curses.py +++ b/Lib/_pyrepl/_minimal_curses.py @@ -36,7 +36,7 @@ def _find_clib(): clib.tigetstr.argtypes = [ctypes.c_char_p] clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] clib.tparm.restype = ctypes.c_char_p OK = 0 diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 38253d4742be0a..1e785fee473fdb 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -44,6 +44,10 @@ class Command: kills_digit_arg: bool = True def __init__(self, reader: HistoricalReader, event_name: str, event: str) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + self.reader = reader self.event = event self.event_name = event_name @@ -137,7 +141,7 @@ def do(self) -> None: class repaint(Command): def do(self) -> None: self.reader.dirty = True - self.reader.console.repaint_prep() + self.reader.console.repaint() class kill_line(KillCommand): @@ -227,7 +231,7 @@ def do(self) -> None: ## in a handler for SIGCONT? r.console.prepare() r.pos = p - r.posxy = 0, 0 # XXX this is invalid + # r.posxy = 0, 0 # XXX this is invalid r.dirty = True r.console.screen = [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 23fb711e2d8e05..d7e86e768671dc 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -17,24 +17,24 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import annotations + from abc import ABC, abstractmethod -import dataclasses +from dataclasses import dataclass, field -@dataclasses.dataclass +@dataclass class Event: evt: str data: str raw: bytes = b"" +@dataclass class Console(ABC): - """Attributes: - - screen, - height, - width, - """ + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 @abstractmethod def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... @@ -58,14 +58,14 @@ def getheightwidth(self) -> tuple[int, int]: ... @abstractmethod - def get_event(self, block: bool = True) -> Event: + def get_event(self, block: bool = True) -> Event | None: """Return an Event instance. Returns None if |block| is false and there is no event pending, otherwise waits for the completion of an event.""" ... @abstractmethod - def push_char(self, char: str) -> None: + def push_char(self, char: int | bytes) -> None: """ Push a character to the console event queue. """ @@ -106,3 +106,7 @@ def getpending(self) -> Event: def wait(self) -> None: """Wait for an event.""" ... + + @abstractmethod + def repaint(self) -> None: + ... diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py index aedf46222d27d0..3a624d9f6835d1 100644 --- a/Lib/_pyrepl/curses.py +++ b/Lib/_pyrepl/curses.py @@ -18,17 +18,14 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# If we are running on top of pypy, we import only _minimal_curses. -# Don't try to fall back to _curses, because that's going to use cffi -# and fall again more loudly. try: import _curses except ImportError: try: - import curses as _curses + import curses as _curses # type: ignore[no-redef] except ImportError: - import _curses + from . import _minimal_curses as _curses # type: ignore[no-redef] setupterm = _curses.setupterm tigetstr = _curses.tigetstr diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini index 92617569e5e22a..ecd03094dbf538 100644 --- a/Lib/_pyrepl/mypy.ini +++ b/Lib/_pyrepl/mypy.ini @@ -16,11 +16,9 @@ strict = True # Various stricter settings that we can't yet enable # Try to enable these in the following order: disallow_any_generics = False -disallow_incomplete_defs = False disallow_untyped_calls = False disallow_untyped_defs = False check_untyped_defs = False -warn_return_any = False disable_error_code = return diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index e57bc263546451..924ef3528ae0ac 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -212,7 +212,7 @@ class Reader: ps2: str = "/>> " ps3: str = "|.. " ps4: str = R"\__ " - kill_ring: list = field(default_factory=list) + kill_ring: list[list[str]] = field(default_factory=list) msg: str = "" arg: int | None = None dirty: bool = False @@ -406,7 +406,7 @@ def get_arg(self, default: int = 1) -> int: else: return self.arg - def get_prompt(self, lineno, cursor_on_line) -> str: + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: """Return what should be in the left-hand margin for line `lineno'.""" if self.arg is not None and cursor_on_line: @@ -427,7 +427,7 @@ def get_prompt(self, lineno, cursor_on_line) -> str: prompt = f"{_ANSIColors.BOLD_MAGENTA}{prompt}{_ANSIColors.RESET}" return prompt - def push_input_trans(self, itrans) -> None: + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: self.input_trans_stack.append(self.input_trans) self.input_trans = itrans @@ -484,7 +484,7 @@ def pos2xy(self) -> tuple[int, int]: y += 1 return p + sum(l2[:pos]), y - def insert(self, text) -> None: + def insert(self, text: str | list[str]) -> None: """Insert 'text' at the insertion point.""" self.buffer[self.pos : self.pos] = list(text) self.pos += len(text) @@ -561,17 +561,15 @@ def refresh(self) -> None: self.console.refresh(screen, self.cxy) self.dirty = False - def do_cmd(self, cmd) -> None: - if isinstance(cmd[0], str): - cmd = self.commands.get(cmd[0], commands.invalid_command)(self, *cmd) - elif isinstance(cmd[0], type) and issubclass(cmd[0], commands.Command): - cmd = cmd[0](self, *cmd) - else: - return # nothing to do + def do_cmd(self, cmd: list[str]) -> None: + assert isinstance(cmd[0], str) + + command_type = self.commands.get(cmd[0], commands.invalid_command) + command = command_type(self, *cmd) # type: ignore[arg-type] - cmd.do() + command.do() - self.after_command(cmd) + self.after_command(command) if self.dirty: self.refresh() @@ -579,9 +577,9 @@ def do_cmd(self, cmd) -> None: self.update_cursor() if not isinstance(cmd, commands.digit_arg): - self.last_command = cmd.__class__ + self.last_command = command_type - self.finished = bool(cmd.finish) + self.finished = bool(command.finish) if self.finished: self.console.finish() self.finish() @@ -625,7 +623,7 @@ def handle1(self, block: bool = True) -> bool: self.do_cmd(cmd) return True - def push_char(self, char) -> None: + def push_char(self, char: int | bytes) -> None: self.console.push_char(char) self.handle1(block=False) diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index f141660220bf00..a8eb2433cd3cce 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,14 +1,18 @@ +from __future__ import annotations + import os -trace_filename = os.environ.get("PYREPL_TRACE") +# types +if False: + from typing import IO + -if trace_filename is not None: +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): trace_file = open(trace_filename, "a") -else: - trace_file = None -def trace(line, *k, **kw): +def trace(line: str, *k: object, **kw: object) -> None: if trace_file is None: return if k or kw: diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 7ca578dd77d518..c22b1d5b5bc290 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -19,6 +19,8 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import annotations + import errno import os import re @@ -38,6 +40,11 @@ from .utils import wlen +# types +if False: + from typing import IO + + class InvalidTerminal(RuntimeError): pass @@ -54,7 +61,7 @@ class InvalidTerminal(RuntimeError): # Add (possibly) missing baudrates (check termios man page) to termios -def add_supported_baudrates(dictionary, rate): +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: baudrate_name = "B%d" % rate if hasattr(termios, baudrate_name): dictionary[getattr(termios, baudrate_name)] = rate @@ -62,7 +69,7 @@ def add_supported_baudrates(dictionary, rate): # Check the termios man page (Line speed) to know where these # values come from. -supported_baudrates = [ +potential_baudrates = [ 0, 110, 115200, @@ -85,19 +92,19 @@ def add_supported_baudrates(dictionary, rate): 9600, ] -ratedict = {} -for rate in supported_baudrates: - add_supported_baudrates(ratedict, rate) +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) # Clean up variables to avoid unintended usage -del rate, add_supported_baudrates +del rate, add_baudrate_if_supported # ------------ end of baudrate definitions ------------ delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") try: - poll = select.poll + poll: type[select.poll] = select.poll except AttributeError: # this is exactly the minumum necessary to support what we # do with poll objects @@ -112,14 +119,17 @@ def poll(self): # note: a 'timeout' argument would be *milliseconds* r, w, e = select.select([self.fd], [], []) return r - poll = MinimalPoll - - -POLLIN = getattr(select, "POLLIN", None) + poll = MinimalPoll # type: ignore[assignment] class UnixConsole(Console): - def __init__(self, f_in=0, f_out=1, term=None, encoding=None): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): """ Initialize the UnixConsole. @@ -129,10 +139,8 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): - term (str): Terminal name. - encoding (str): Encoding to use for I/O operations. """ - if encoding is None: - encoding = sys.getdefaultencoding() - self.encoding = encoding + self.encoding = encoding or sys.getdefaultencoding() if isinstance(f_in, int): self.input_fd = f_in @@ -145,8 +153,8 @@ def __init__(self, f_in=0, f_out=1, term=None, encoding=None): self.output_fd = f_out.fileno() self.pollob = poll() - self.pollob.register(self.input_fd, POLLIN) - curses.setupterm(term, self.output_fd) + self.pollob.register(self.input_fd, select.POLLIN) + curses.setupterm(term or None, self.output_fd) self.term = term def _my_getstr(cap, optional=0): @@ -187,7 +195,7 @@ def _my_getstr(cap, optional=0): self.event_queue = EventQueue(self.input_fd, self.encoding) self.cursor_visible = 1 - def change_encoding(self, encoding): + def change_encoding(self, encoding: str) -> None: """ Change the encoding used for I/O operations. @@ -340,17 +348,14 @@ def restore(self): signal.signal(signal.SIGWINCH, self.old_sigwinch) del self.old_sigwinch - def push_char(self, char): + def push_char(self, char: int | bytes) -> None: """ Push a character to the console event queue. - - Parameters: - - char (str): Character to push. """ trace("push char {char!r}", char=char) self.event_queue.push(char) - def get_event(self, block: bool = True) -> Event: + def get_event(self, block: bool = True) -> Event | None: """ Get an event from the console event queue. @@ -698,7 +703,7 @@ def __show_cursor(self): self.__maybe_write_code(self._cnorm) self.cursor_visible = 1 - def repaint_prep(self): + def repaint(self): if not self.__gone_tall: self.__posxy = 0, self.__posxy[1] self.__write("\r") diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 73e133de2dcab1..b179796444ac05 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -62,16 +62,16 @@ def get_terminal_keycodes(): Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} - for key, terminal_code in TERMINAL_KEYNAMES.items(): + for key, terminal_code in TERMINAL_KEYNAMES.items(): keycode = curses.tigetstr(terminal_code) - trace('key {key} tiname {tiname} keycode {keycode!r}', **locals()) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes class EventQueue: - def __init__(self, fd, encoding): + def __init__(self, fd: int, encoding: str) -> None: self.keycodes = get_terminal_keycodes() if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] @@ -80,10 +80,10 @@ def __init__(self, fd, encoding): self.keymap = self.compiled_keymap trace("keymap {k!r}", k=self.keymap) self.encoding = encoding - self.events = deque() + self.events: deque[Event] = deque() self.buf = bytearray() - def get(self): + def get(self) -> Event | None: """ Retrieves the next event from the queue. """ @@ -92,13 +92,13 @@ def get(self): else: return None - def empty(self): + def empty(self) -> bool: """ Checks if the queue is empty. """ return not self.events - def flush_buf(self): + def flush_buf(self) -> bytearray: """ Flushes the buffer and returns its contents. """ @@ -106,14 +106,14 @@ def flush_buf(self): self.buf = bytearray() return old - def insert(self, event): + def insert(self, event: Event) -> None: """ Inserts an event into the queue. """ trace('added event {event}', event=event) self.events.append(event) - def push(self, char): + def push(self, char: int | bytes) -> None: """ Processes a character by updating the buffer and handling special key mappings. """ @@ -139,8 +139,8 @@ def push(self, char): trace('unrecognized escape sequence, propagating...') self.keymap = self.compiled_keymap self.insert(Event('key', '\033', bytearray(b'\033'))) - for c in self.flush_buf()[1:]: - self.push(chr(c)) + for _c in self.flush_buf()[1:]: + self.push(_c) else: try: diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index aaa126df798fdb..7539b7c3bacc8c 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -106,7 +106,7 @@ def __init__(self, events, encoding="utf-8"): self.height = 100 self.width = 80 - def get_event(self, block: bool = True) -> Event: + def get_event(self, block: bool = True) -> Event | None: return next(self.events) def getpending(self) -> Event: @@ -130,7 +130,7 @@ def move_cursor(self, x: int, y: int) -> None: def set_cursor_vis(self, visible: bool) -> None: pass - def push_char(self, char: str) -> None: + def push_char(self, char: int | bytes) -> None: pass def beep(self) -> None: @@ -151,6 +151,9 @@ def forgetinput(self) -> None: def wait(self) -> None: pass + def repaint(self) -> None: + pass + class TestCursorPosition(TestCase): def test_up_arrow_simple(self): From 18cd2cf5fce4b1e35410c79beb932f1d8471e9fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 15:24:36 +0200 Subject: [PATCH 61/78] Work around mypy being unhappy with _colorize --- Lib/_pyrepl/reader.py | 2 +- Lib/_pyrepl/simple_interact.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index c03073b76283c0..d12db13e1b6444 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -25,7 +25,7 @@ from dataclasses import dataclass, field, fields import re import unicodedata -from _colorize import can_colorize, ANSIColors +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] from . import commands, console, input diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 0e6fdb8550f7bf..16cac678ca0993 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -23,7 +23,7 @@ allowing multiline input and multiline history entries. """ -import _colorize +import _colorize # type: ignore[import-not-found] import _sitebuiltins import linecache import sys From afe2513c7f75bd840996e883df6fb60dd1ea3083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 15:27:48 +0200 Subject: [PATCH 62/78] Remove duplicate test --- Lib/test/test_pyrepl.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 7539b7c3bacc8c..5293b3eb08f51a 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -870,16 +870,6 @@ def test_calc_screen_wrap_removes_after_backspace(self): reader, _ = handle_events_narrow_console(events) self.assert_screen_equals(reader, 9 * "a") - def test_calc_screen_wrap_removes_after_backspace(self): - events = itertools.chain( - code_to_events(10 * "a"), - [ - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), - ], - ) - reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, 9 * "a") - def test_calc_screen_backspace_in_second_line_after_wrap(self): events = itertools.chain( code_to_events(11 * "a"), From 864ecb87769cd0cf8ca5959308cc21bb9ebb3961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 16:37:25 +0200 Subject: [PATCH 63/78] Add types to HistoricalReader and CompletingReader --- Lib/_pyrepl/completing_reader.py | 115 +++++++++++++++--------------- Lib/_pyrepl/historical_reader.py | 118 +++++++++++++++---------------- Lib/_pyrepl/reader.py | 2 +- 3 files changed, 118 insertions(+), 117 deletions(-) diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index ad29c1c0fe9a18..19fc06feaf3ced 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -18,12 +18,22 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import annotations + +from dataclasses import dataclass, field + import re -from . import commands, reader +from . import commands, console, reader from .reader import Reader -def prefix(wordlist, j=0): +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: d = {} i = j try: @@ -36,19 +46,20 @@ def prefix(wordlist, j=0): d = {} except IndexError: return wordlist[0][j:i] + return "" STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") -def stripcolor(s): +def stripcolor(s: str) -> str: return STRIPCOLOR_REGEX.sub('', s) -def real_len(s): +def real_len(s: str) -> int: return len(stripcolor(s)) -def left_align(s, maxlen): +def left_align(s: str, maxlen: int) -> str: stripped = stripcolor(s) if len(stripped) > maxlen: # too bad, we remove the color @@ -57,7 +68,13 @@ def left_align(s, maxlen): return s + ' '*padding -def build_menu(cons, wordlist, start, use_brackets, sort_in_column): +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: if use_brackets: item = "[ %s ]" padding = 4 @@ -147,8 +164,9 @@ def build_menu(cons, wordlist, start, use_brackets, sort_in_column): class complete(commands.Command): - def do(self): - r = self.reader + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] last_is_completer = r.last_command_is(self.__class__) immutable_completions = r.assume_immutable_completions completions_unchangable = last_is_completer and immutable_completions @@ -162,7 +180,7 @@ def do(self): elif len(completions) == 1: if completions_unchangable and len(completions[0]) == len(stem): r.msg = "[ sole completion ]" - r.dirty = 1 + r.dirty = True r.insert(completions[0][len(stem):]) else: p = prefix(completions, len(stem)) @@ -174,19 +192,22 @@ def do(self): r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, r.cmpltn_menu_end, r.use_brackets, r.sort_in_column) - r.dirty = 1 + r.dirty = True elif stem + p in completions: r.msg = "[ complete but not unique ]" - r.dirty = 1 + r.dirty = True else: r.msg = "[ not unique ]" - r.dirty = 1 + r.dirty = True class self_insert(commands.self_insert): - def do(self): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + commands.self_insert.do(self) - r = self.reader + if r.cmpltn_menu_vis: stem = r.get_stem() if len(stem) < 1: @@ -202,38 +223,40 @@ def do(self): r.cmpltn_reset() +@dataclass class CompletingReader(Reader): - """Adds completion support + """Adds completion support""" - Adds instance variables: - * cmpltn_menu, cmpltn_menu_vis, cmpltn_menu_end, cmpltn_choices: - * - """ + ### Class variables # see the comment for the complete command assume_immutable_completions = True use_brackets = True # display completions inside [] sort_in_column = False - def collect_keymap(self): - return super(CompletingReader, self).collect_keymap() + ( - (r'\t', 'complete'),) + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_vis: int = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) - def __init__(self, console): - super(CompletingReader, self).__init__(console) - self.cmpltn_menu = ["[ menu 1 ]", "[ menu 2 ]"] - self.cmpltn_menu_vis = 0 - self.cmpltn_menu_end = 0 + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() for c in (complete, self_insert): self.commands[c.__name__] = c self.commands[c.__name__.replace('_', '-')] = c - def after_command(self, cmd): - super(CompletingReader, self).after_command(cmd) + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() - def calc_screen(self): - screen = super(CompletingReader, self).calc_screen() + def calc_screen(self) -> list[str]: + screen = super().calc_screen() if self.cmpltn_menu_vis: ly = self.lxy[1] screen[ly:ly] = self.cmpltn_menu @@ -241,17 +264,17 @@ def calc_screen(self): self.cxy = self.cxy[0], self.cxy[1] + len(self.cmpltn_menu) return screen - def finish(self): - super(CompletingReader, self).finish() + def finish(self) -> None: + super().finish() self.cmpltn_reset() - def cmpltn_reset(self): + def cmpltn_reset(self) -> None: self.cmpltn_menu = [] self.cmpltn_menu_vis = 0 self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] - def get_stem(self): + def get_stem(self) -> str: st = self.syntax_table SW = reader.SYNTAX_WORD b = self.buffer @@ -260,25 +283,5 @@ def get_stem(self): p -= 1 return ''.join(b[p+1:self.pos]) - def get_completions(self, stem): + def get_completions(self, stem: str) -> list[str]: return [] - - -def test(): - class TestReader(CompletingReader): - def get_completions(self, stem): - return [s for l in self.history - for s in l.split() - if s and s.startswith(stem)] - - reader = TestReader() - reader.ps1 = "c**> " - reader.ps2 = "c/*> " - reader.ps3 = "c|*> " - reader.ps4 = r"c\*> " - while reader.readline(): - pass - - -if __name__ == '__main__': - test() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index 05a86d92377a09..eef7d901b083ef 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -20,6 +20,7 @@ from __future__ import annotations from contextlib import contextmanager +from dataclasses import dataclass, field from . import commands, input from .reader import Reader @@ -62,7 +63,7 @@ def do(self) -> None: class previous_history(commands.Command): - def do(self): + def do(self) -> None: r = self.reader if r.historyi == 0: r.error("start of history list") @@ -71,32 +72,32 @@ def do(self): class restore_history(commands.Command): - def do(self): + def do(self) -> None: r = self.reader if r.historyi != len(r.history): if r.get_unicode() != r.history[r.historyi]: r.buffer = list(r.history[r.historyi]) r.pos = len(r.buffer) - r.dirty = 1 + r.dirty = True class first_history(commands.Command): - def do(self): + def do(self) -> None: self.reader.select_item(0) class last_history(commands.Command): - def do(self): + def do(self) -> None: self.reader.select_item(len(self.reader.history)) class operate_and_get_next(commands.FinishCommand): - def do(self): + def do(self) -> None: self.reader.next_history = self.reader.historyi + 1 class yank_arg(commands.Command): - def do(self): + def do(self) -> None: r = self.reader if r.last_command is self.__class__: r.yank_arg_i += 1 @@ -120,119 +121,102 @@ def do(self): b[r.pos - o : r.pos] = list(w) r.yank_arg_yanked = w r.pos += len(w) - o - r.dirty = 1 + r.dirty = True class forward_history_isearch(commands.Command): - def do(self): + def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_FORWARDS r.isearch_start = r.historyi, r.pos r.isearch_term = "" - r.dirty = 1 + r.dirty = True r.push_input_trans(r.isearch_trans) class reverse_history_isearch(commands.Command): - def do(self): + def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS - r.dirty = 1 + r.dirty = True r.isearch_term = "" r.push_input_trans(r.isearch_trans) r.isearch_start = r.historyi, r.pos class isearch_cancel(commands.Command): - def do(self): + def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_NONE r.pop_input_trans() r.select_item(r.isearch_start[0]) r.pos = r.isearch_start[1] - r.dirty = 1 + r.dirty = True class isearch_add_character(commands.Command): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer r.isearch_term += self.event[-1] - r.dirty = 1 + r.dirty = True p = r.pos + len(r.isearch_term) - 1 if b[p : p + 1] != [r.isearch_term[-1]]: r.isearch_next() class isearch_backspace(commands.Command): - def do(self): + def do(self) -> None: r = self.reader if len(r.isearch_term) > 0: r.isearch_term = r.isearch_term[:-1] - r.dirty = 1 + r.dirty = True else: r.error("nothing to rubout") class isearch_forwards(commands.Command): - def do(self): + def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_FORWARDS r.isearch_next() class isearch_backwards(commands.Command): - def do(self): + def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS r.isearch_next() class isearch_end(commands.Command): - def do(self): + def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_NONE r.console.forgetinput() r.pop_input_trans() - r.dirty = 1 + r.dirty = True +@dataclass class HistoricalReader(Reader): """Adds history support (with incremental history searching) to the Reader class. - - Adds the following instance variables: - * history: - a list of strings - * historyi: - * transient_history: - * next_history: - * isearch_direction, isearch_term, isearch_start: - * yank_arg_i, yank_arg_yanked: - used by the yank-arg command; not actually manipulated by any - HistoricalReader instance methods. """ - def collect_keymap(self): - return super().collect_keymap() + ( - (r"\C-n", "next-history"), - (r"\C-p", "previous-history"), - (r"\C-o", "operate-and-get-next"), - (r"\C-r", "reverse-history-isearch"), - (r"\C-s", "forward-history-isearch"), - (r"\M-r", "restore-history"), - (r"\M-.", "yank-arg"), - (r"\", "last-history"), - (r"\", "first-history"), - ) - - def __init__(self, console): - super().__init__(console) - self.history = [] - self.historyi = 0 - self.transient_history = {} - self.next_history = None - self.isearch_direction = ISEARCH_DIRECTION_NONE + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() for c in [ next_history, previous_history, @@ -253,11 +237,25 @@ def __init__(self, console): ]: self.commands[c.__name__] = c self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos self.isearch_trans = input.KeymapTranslator( isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character ) - def select_item(self, i): + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "last-history"), + (r"\", "first-history"), + ) + + def select_item(self, i: int) -> None: self.transient_history[self.historyi] = self.get_unicode() buf = self.transient_history.get(i) if buf is None: @@ -267,14 +265,14 @@ def select_item(self, i): self.pos = len(self.buffer) self.dirty = True - def get_item(self, i): + def get_item(self, i: int) -> str: if i != len(self.history): return self.transient_history.get(i, self.history[i]) else: return self.transient_history.get(i, self.get_unicode()) @contextmanager - def suspend(self): + def suspend(self) -> SimpleContextManager: with super().suspend(): try: old_history = self.history[:] @@ -283,7 +281,7 @@ def suspend(self): finally: self.history[:] = old_history - def prepare(self): + def prepare(self) -> None: super().prepare() try: self.transient_history = {} @@ -299,14 +297,14 @@ def prepare(self): self.restore() raise - def get_prompt(self, lineno, cursor_on_line): + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] return "(%s-search `%s') " % (d, self.isearch_term) else: return super().get_prompt(lineno, cursor_on_line) - def isearch_next(self): + def isearch_next(self) -> None: st = self.isearch_term p = self.pos i = self.historyi @@ -334,7 +332,7 @@ def isearch_next(self): s = self.get_item(i) p = len(s) - def finish(self): + def finish(self) -> None: super().finish() ret = self.get_unicode() for i, t in self.transient_history.items(): diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index d12db13e1b6444..70101315c97e26 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -242,7 +242,7 @@ def __post_init__(self) -> None: self.cxy = self.pos2xy() self.lxy = (self.pos, 0) - def collect_keymap(self): + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: return default_keymap def calc_screen(self) -> list[str]: From b5f889503efba6ec980ec88dfaee510f05790f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 18:57:19 +0200 Subject: [PATCH 64/78] Add typing to readline.py --- Lib/_pyrepl/commands.py | 4 +- Lib/_pyrepl/reader.py | 9 +- Lib/_pyrepl/readline.py | 198 +++++++++++++++++---------------- Lib/_pyrepl/simple_interact.py | 30 +++-- Lib/_pyrepl/types.py | 5 +- Lib/_pyrepl/unix_eventqueue.py | 2 +- Lib/code.py | 2 +- Lib/test/test_pyrepl.py | 21 ++-- 8 files changed, 148 insertions(+), 123 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 1e785fee473fdb..60ceb30d2cd77d 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -43,7 +43,9 @@ class Command: finish: bool = False kills_digit_arg: bool = True - def __init__(self, reader: HistoricalReader, event_name: str, event: str) -> None: + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: # Reader should really be "any reader" but there's too much usage of # HistoricalReader methods and fields in the code below for us to # refactor at the moment. diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 70101315c97e26..a7ef988da12a6a 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -30,6 +30,7 @@ from . import commands, console, input from .utils import ANSI_ESCAPE_SEQUENCE, wlen +from .trace import trace # types @@ -562,9 +563,13 @@ def refresh(self) -> None: self.console.refresh(screen, self.cxy) self.dirty = False - def do_cmd(self, cmd: list[str]) -> None: + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" assert isinstance(cmd[0], str) + trace("received command {cmd}", cmd=cmd) command_type = self.commands.get(cmd[0], commands.invalid_command) command = command_type(self, *cmd) # type: ignore[arg-type] @@ -613,7 +618,7 @@ def handle1(self, block: bool = True) -> bool: if translate: cmd = self.input_trans.get() else: - cmd = event.evt, event.data + cmd = [event.evt, event.data] if cmd is None: if block: diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index bbc0cf69ff4a6e..b9725bc11ed53b 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -26,6 +26,10 @@ extensions for multiline input. """ +from __future__ import annotations + +from dataclasses import dataclass, field + import os import readline from site import gethistoryfile # type: ignore[attr-defined] @@ -35,7 +39,14 @@ from .completing_reader import CompletingReader from .unix_console import UnixConsole, _error -ENCODING = sys.getfilesystemencoding() or "latin1" # XXX review +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + __all__ = [ "add_history", @@ -68,20 +79,34 @@ # ____________________________________________________________ +@dataclass class ReadlineConfig: - readline_completer = readline.get_completer() - completer_delims = dict.fromkeys(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + readline_completer: Completer | None = readline.get_completer() + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") +@dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields assume_immutable_completions = False use_brackets = False sort_in_column = True - def error(self, msg="none"): + # Instance fields + config: ReadlineConfig + more_lines: Callable[[str], bool] | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: pass # don't show error messages by default - def get_stem(self): + def get_stem(self) -> str: b = self.buffer p = self.pos - 1 completer_delims = self.config.completer_delims @@ -89,7 +114,7 @@ def get_stem(self): p -= 1 return "".join(b[p + 1 : self.pos]) - def get_completions(self, stem): + def get_completions(self, stem: str) -> list[str]: if len(stem) == 0 and self.more_lines is not None: b = self.buffer p = self.pos @@ -119,7 +144,7 @@ def get_completions(self, stem): result.sort() return result - def get_trimmed_history(self, maxlength): + def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: cut = len(self.history) - maxlength if cut < 0: @@ -130,32 +155,13 @@ def get_trimmed_history(self, maxlength): # --- simplified support for reading multiline Python statements --- - # This duplicates small parts of pyrepl.python_reader. I'm not - # reusing the PythonicReader class directly for two reasons. One is - # to try to keep as close as possible to CPython's prompt. The - # other is that it is the readline module that we are ultimately - # implementing here, and I don't want the built-in raw_input() to - # start trying to read multiline inputs just because what the user - # typed look like valid but incomplete Python code. So we get the - # multiline feature only when using the multiline_input() function - # directly (see _pypy_interact.py). - - more_lines = None - - def collect_keymap(self): + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: return super().collect_keymap() + ( (r"\n", "maybe-accept"), (r"\", "backspace-dedent"), ) - def __init__(self, console): - super().__init__(console) - self.commands["maybe_accept"] = maybe_accept - self.commands["maybe-accept"] = maybe_accept - self.commands["backspace_dedent"] = backspace_dedent - self.commands["backspace-dedent"] = backspace_dedent - - def after_command(self, cmd): + def after_command(self, cmd: Command) -> None: super().after_command(cmd) if self.more_lines is None: # Force single-line input if we are in raw_input() mode. @@ -172,12 +178,12 @@ def after_command(self, cmd): self.pos = len(self.buffer) -def set_auto_history(_should_auto_add_history): +def set_auto_history(_should_auto_add_history: bool) -> None: """Enable or disable automatic history""" historical_reader.should_auto_add_history = bool(_should_auto_add_history) -def _get_this_line_indent(buffer, pos): +def _get_this_line_indent(buffer: list[str], pos: int) -> int: indent = 0 while pos > 0 and buffer[pos - 1] in " \t": indent += 1 @@ -187,7 +193,7 @@ def _get_this_line_indent(buffer, pos): return 0 -def _get_previous_line_indent(buffer, pos): +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: prevlinestart = pos while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": prevlinestart -= 1 @@ -202,9 +208,10 @@ def _get_previous_line_indent(buffer, pos): class maybe_accept(commands.Command): - def do(self): - r = self.reader - r.dirty = 1 # this is needed to hide the completion menu, if visible + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible # # if there are already several lines and the cursor # is not on the last one, always insert a new \n. @@ -220,13 +227,13 @@ def do(self): for i in range(prevlinestart, prevlinestart + indent): r.insert(r.buffer[i]) elif not self.reader.paste_mode: - self.finish = 1 + self.finish = True else: r.insert("\n") class backspace_dedent(commands.Command): - def do(self): + def do(self) -> None: r = self.reader b = r.buffer if r.pos > 0: @@ -242,7 +249,7 @@ def do(self): break r.pos -= repeat del b[r.pos : r.pos + repeat] - r.dirty = 1 + r.dirty = True else: self.reader.error("can't backspace at start") @@ -250,29 +257,34 @@ def do(self): # ____________________________________________________________ +@dataclass(slots=True) class _ReadlineWrapper: - reader = None - saved_history_length = -1 - startup_hook = None - config = ReadlineConfig() - - def __init__(self, f_in=None, f_out=None): - self.f_in = f_in if f_in is not None else os.dup(0) - self.f_out = f_out if f_out is not None else os.dup(1) - - def get_reader(self): + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = None + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: if self.reader is None: console = UnixConsole(self.f_in, self.f_out, encoding=ENCODING) - self.reader = ReadlineAlikeReader(console) - self.reader.config = self.config + self.reader = ReadlineAlikeReader(console=console, config=self.config) return self.reader - def raw_input(self, prompt=""): + def input(self, prompt: object = "") -> str: try: reader = self.get_reader() except _error: - return _old_raw_input(prompt) - reader.ps1 = prompt + assert raw_input is not None + return raw_input(prompt) + reader.ps1 = str(prompt) return reader.readline(startup_hook=self.startup_hook) def multiline_input(self, more_lines, ps1, ps2): @@ -291,39 +303,35 @@ def multiline_input(self, more_lines, ps1, ps2): reader.more_lines = saved reader.paste_mode = False - def parse_and_bind(self, string): + def parse_and_bind(self, string: str) -> None: pass # XXX we don't support parsing GNU-readline-style init files - def set_completer(self, function=None): + def set_completer(self, function: Completer | None = None) -> None: self.config.readline_completer = function - def get_completer(self): + def get_completer(self) -> Completer | None: return self.config.readline_completer - def set_completer_delims(self, string): - self.config.completer_delims = dict.fromkeys(string) + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) - def get_completer_delims(self): - chars = list(self.config.completer_delims.keys()) - chars.sort() - return "".join(chars) + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) - def _histline(self, line): + def _histline(self, line: str) -> str: line = line.rstrip("\n") - if isinstance(line, str): - return line # on py3k - return str(line, "utf-8", "replace") + return line - def get_history_length(self): + def get_history_length(self) -> int: return self.saved_history_length - def set_history_length(self, length): + def set_history_length(self, length: int) -> None: self.saved_history_length = length - def get_current_history_length(self): + def get_current_history_length(self) -> int: return len(self.get_reader().history) - def read_history_file(self, filename=gethistoryfile()): + def read_history_file(self, filename: str = gethistoryfile()) -> None: # multiline extension (really a hack) for the end of lines that # are actually continuations inside a single multiline_input() # history item: we use \r\n instead of just \n. If the history @@ -347,7 +355,7 @@ def read_history_file(self, filename=gethistoryfile()): if line: history.append(line) - def write_history_file(self, filename=gethistoryfile()): + def write_history_file(self, filename: str = gethistoryfile()) -> None: maxlength = self.saved_history_length history = self.get_reader().get_trimmed_history(maxlength) with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: @@ -355,43 +363,43 @@ def write_history_file(self, filename=gethistoryfile()): entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") - def clear_history(self): + def clear_history(self) -> None: del self.get_reader().history[:] - def get_history_item(self, index): + def get_history_item(self, index: int) -> str | None: history = self.get_reader().history if 1 <= index <= len(history): return history[index - 1] else: - return None # blame readline.c for not raising + return None # like readline.c - def remove_history_item(self, index): + def remove_history_item(self, index: int) -> None: history = self.get_reader().history if 0 <= index < len(history): del history[index] else: raise ValueError("No history item at position %d" % index) - # blame readline.c for raising ValueError + # like readline.c - def replace_history_item(self, index, line): + def replace_history_item(self, index: int, line: str) -> None: history = self.get_reader().history if 0 <= index < len(history): history[index] = self._histline(line) else: raise ValueError("No history item at position %d" % index) - # blame readline.c for raising ValueError + # like readline.c - def add_history(self, line): + def add_history(self, line: str) -> None: self.get_reader().history.append(self._histline(line)) - def set_startup_hook(self, function=None): + def set_startup_hook(self, function: Callback | None = None) -> None: self.startup_hook = function - def get_line_buffer(self): + def get_line_buffer(self) -> bytes: buf_str = self.get_reader().get_unicode() - return buf_str.encode() + return buf_str.encode(ENCODING) - def _get_idxs(self): + def _get_idxs(self) -> tuple[int, int]: start = cursor = self.get_reader().pos buf = self.get_line_buffer() for i in range(cursor - 1, -1, -1): @@ -400,14 +408,14 @@ def _get_idxs(self): start = i return start, cursor - def get_begidx(self): + def get_begidx(self) -> int: return self._get_idxs()[0] - def get_endidx(self): + def get_endidx(self) -> int: return self._get_idxs()[1] - def insert_text(self, text): - return self.get_reader().insert(text) + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) _wrapper = _ReadlineWrapper() @@ -446,13 +454,13 @@ def insert_text(self, text): # Stubs -def _make_stub(_name, _ret): - def stub(*args, **kwds): +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: import warnings warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) - stub.func_name = _name + stub.__name__ = _name globals()[_name] = stub @@ -467,9 +475,9 @@ def stub(*args, **kwds): # ____________________________________________________________ -def _setup(): - global _old_raw_input - if _old_raw_input is not None: +def _setup() -> None: + global raw_input + if raw_input is not None: return # don't run _setup twice try: @@ -486,9 +494,9 @@ def _setup(): # this is not really what readline.c does. Better than nothing I guess import builtins - _old_raw_input = builtins.input - builtins.input = _wrapper.raw_input + raw_input = builtins.input + builtins.input = _wrapper.input -_old_raw_input = None +raw_input: Callable[[object], str] | None = None _setup() diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 16cac678ca0993..22c8e854bebc9c 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -23,18 +23,22 @@ allowing multiline input and multiline history entries. """ +from __future__ import annotations + import _colorize # type: ignore[import-not-found] import _sitebuiltins import linecache import sys import code +from types import ModuleType from .console import Event from .readline import _get_reader, multiline_input from .unix_console import _error -def check(): # returns False if there is a problem initializing the state +def check() -> bool: + """Returns False if there is a problem initializing the state.""" try: _get_reader() except _error: @@ -42,7 +46,7 @@ def check(): # returns False if there is a problem initializing the state return True -def _strip_final_indent(text): +def _strip_final_indent(text: str) -> str: # kill spaces and tabs at the end, but only if they follow '\n'. # meant to remove the auto-indentation only (although it would of # course also remove explicitly-added indentation). @@ -61,15 +65,23 @@ def _strip_final_indent(text): } class InteractiveColoredConsole(code.InteractiveConsole): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] self.can_colorize = _colorize.can_colorize() def showtraceback(self): super().showtraceback(colorize=self.can_colorize) -def run_multiline_interactive_console(mainmodule=None, future_flags=0): +def run_multiline_interactive_console( + mainmodule: ModuleType | None= None, future_flags: int = 0 +) -> None: import code import __main__ @@ -97,13 +109,13 @@ def maybe_run_command(statement: str) -> bool: # inside multiline_input. reader.prepare() reader.refresh() - reader.do_cmd([command, Event(evt=command, data=command)]) + reader.do_cmd((command, [statement])) reader.restore() return True return False - def more_lines(unicodetext): + def more_lines(unicodetext: str) -> bool: # ooh, look at the hack: src = _strip_final_indent(unicodetext) try: @@ -131,8 +143,8 @@ def more_lines(unicodetext): continue input_name = f"" - linecache._register_code(input_name, statement, "") - more = console.push(_strip_final_indent(statement), filename=input_name) + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name) # type: ignore[call-arg] assert not more input_n += 1 except KeyboardInterrupt: diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index 21921627871064..f9d48b828c720b 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -1,7 +1,8 @@ -from typing import Any, Callable, Iterator +from collections.abc import Callable, Iterator -Callback = Callable[[], Any] +Callback = Callable[[], object] SimpleContextManager = Iterator[None] KeySpec = str # like r"\C-c" CommandName = str # like "interrupt" EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index b179796444ac05..70cfade26e23b1 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -57,7 +57,7 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes(): +def get_terminal_keycodes() -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ diff --git a/Lib/code.py b/Lib/code.py index 7a5ee6002395ef..1ee1ad62ff4506 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -171,7 +171,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename="", local_exit=False): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 5293b3eb08f51a..564b234084687e 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -63,9 +63,8 @@ def prepare_fake_console(**kwargs): def prepare_reader(console, **kwargs): - reader = ReadlineAlikeReader(console) - reader.config = ReadlineConfig() - reader.config.readline_completer = None + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) reader.more_lines = partial(more_lines, namespace=None) reader.paste_mode = True # Avoid extra indents @@ -447,9 +446,8 @@ def test_cursor_position_after_wrap_and_move_up(self): class TestPyReplOutput(TestCase): def prepare_reader(self, events): console = FakeConsole(events) - reader = ReadlineAlikeReader(console) - reader.config = ReadlineConfig() - reader.config.readline_completer = None + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) return reader def test_basic(self): @@ -555,9 +553,9 @@ def test_control_character(self): class TestPyReplCompleter(TestCase): def prepare_reader(self, events, namespace): console = FakeConsole(events) - reader = ReadlineAlikeReader(console) - reader.config = ReadlineConfig() - reader.config.readline_completer = rlcompleter.Completer(namespace).complete + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) return reader def test_simple_completion(self): @@ -705,9 +703,8 @@ def setUp(self) -> None: def prepare_reader(self, events): console = FakeConsole(events) - reader = ReadlineAlikeReader(console) - reader.config = ReadlineConfig() - reader.config.readline_completer = None + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) return reader def test_paste(self): From 45537bb3a2484b0e79186681331091cd611bb0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 19:28:01 +0200 Subject: [PATCH 65/78] Pin test_pyrepl to the `curses` test resource --- Lib/test/test_pyrepl.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 564b234084687e..9d1754631d8467 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -9,6 +9,13 @@ from unittest import TestCase from unittest.mock import MagicMock, patch +from test.support import requires + +# Optionally test pyrepl. This currently requires that the +# 'curses' resource be given on the regrtest command line using the -u +# option. If not available, nothing after this line will be executed. +requires('curses') + from _pyrepl.console import Console, Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.simple_interact import _strip_final_indent From 22c4d3fb55f8181ea619fd9b2ca5da61880a7a6a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 4 May 2024 20:16:34 +0200 Subject: [PATCH 66/78] Don't call anything curses in test_repl --- Lib/test/test_pyrepl.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 9d1754631d8467..b411f4be1ebc43 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -1,4 +1,3 @@ -import curses import itertools import os import rlcompleter @@ -599,11 +598,8 @@ def test_global_namespace_completion(self): self.assertEqual(output, "python") +@patch("_pyrepl.curses.tigetstr", MagicMock(return_value=b"")) class TestUnivEventQueue(TestCase): - def setUp(self) -> None: - curses.setupterm() - return super().setUp() - def test_get(self): eq = EventQueue(sys.stdout.fileno(), "utf-8") event = Event("key", "a", b"a") @@ -704,10 +700,6 @@ def test_push_unrecognized_escape_sequence(self): class TestPasteEvent(TestCase): - def setUp(self) -> None: - curses.setupterm() - return super().setUp() - def prepare_reader(self, events): console = FakeConsole(events) config = ReadlineConfig(readline_completer=None) From 6a0fcefe2060d76d0bf84c26b9c63aa243abb774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 20:46:46 +0200 Subject: [PATCH 67/78] Many machines on CI run with -uall; skip the test if curses import fails --- Lib/test/test_pyrepl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index b411f4be1ebc43..38f93c3633f32a 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -9,11 +9,14 @@ from unittest.mock import MagicMock, patch from test.support import requires +from test.support.import_helper import import_module # Optionally test pyrepl. This currently requires that the # 'curses' resource be given on the regrtest command line using the -u -# option. If not available, nothing after this line will be executed. +# option. Additionally, we need to attempt to import curses and readline. requires('curses') +curses = import_module('curses') +readline = import_module('readline') from _pyrepl.console import Console, Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig From b4362b50672d7387c6b2774221f3b286083540da Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 4 May 2024 21:28:36 +0200 Subject: [PATCH 68/78] Fix refleaks --- Lib/test/test_pyrepl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 38f93c3633f32a..c53bdef0ca9cb1 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -601,7 +601,7 @@ def test_global_namespace_completion(self): self.assertEqual(output, "python") -@patch("_pyrepl.curses.tigetstr", MagicMock(return_value=b"")) +@patch("_pyrepl.curses.tigetstr", lambda x: b"") class TestUnivEventQueue(TestCase): def test_get(self): eq = EventQueue(sys.stdout.fileno(), "utf-8") From a900ccdee5af37d1d0cc6bfe32764af43e729cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 21:41:37 +0200 Subject: [PATCH 69/78] Document PYTHON_BASIC_REPL --- Doc/glossary.rst | 4 ++++ Doc/using/cmdline.rst | 9 +++++++++ Lib/site.py | 4 ++-- Modules/main.c | 4 ++-- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 05ac3edb63b65d..3ba0d5f732c113 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1084,6 +1084,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 1f5d02d1ea54a2..a81941c2d39054 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -1158,6 +1158,15 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_REPL + + If this variable is set to ``1``, the interpreter will not attempt to + load the Python-based :term:`REPL` that requires :mod:`curses` and + :mod:`readline`, and will instead use the traditional parser-based + :term:`REPL`. + + .. versionadded:: 3.13 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a diff --git a/Lib/site.py b/Lib/site.py index 3d6477e9f65ea8..b63447d6673f68 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -515,7 +515,7 @@ def register_readline(): # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() try: - if os.getenv("PYTHON_OLD_REPL"): + if os.getenv("PYTHON_BASIC_REPL"): readline.read_history_file(history) else: _pyrepl.readline.read_history_file(history) @@ -524,7 +524,7 @@ def register_readline(): def write_history(): try: - if os.getenv("PYTHON_OLD_REPL"): + if os.getenv("PYTHON_BASIC_REPL"): readline.write_history_file(history) else: _pyrepl.readline.write_history_file(history) diff --git a/Modules/main.c b/Modules/main.c index e33d3c64a8e33a..8eded2639ad90a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -513,8 +513,8 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - if (!isatty(fileno(stdin)) || _Py_GetEnv(config->use_environment, "PYTHON_OLD_REPL")) - { + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { PyCompilerFlags cf = _PyCompilerFlags_INIT; int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); return (run != 0); From 13fda4ee2b6ed1c4e3cdb66df769b08d8bded277 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 4 May 2024 21:55:59 +0200 Subject: [PATCH 70/78] Do not call _setup() on import --- Lib/_pyrepl/readline.py | 1 - Lib/_pyrepl/simple_interact.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index b9725bc11ed53b..37ba98d4c8c87a 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -499,4 +499,3 @@ def _setup() -> None: raw_input: Callable[[object], str] | None = None -_setup() diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 22c8e854bebc9c..70f9e009d840e5 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -84,6 +84,8 @@ def run_multiline_interactive_console( ) -> None: import code import __main__ + from .readline import _setup + _setup() mainmodule = mainmodule or __main__ console = InteractiveColoredConsole(mainmodule.__dict__, filename="") From 6ebf89cc5cca33e1aa567edb09e31c00c6e60e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 4 May 2024 22:19:20 +0200 Subject: [PATCH 71/78] Document pyrepl --- Doc/glossary.rst | 10 ++++++---- Doc/tutorial/appendix.rst | 22 ++++++++++++++++++++++ Doc/using/cmdline.rst | 1 + 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 3ba0d5f732c113..2846f77feb112d 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -620,7 +621,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 4bea0d8a49ce20..812f2473327377 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -10,6 +10,28 @@ Appendix Interactive Mode ================ +There are two variants of the interactive :term:`REPL`. The classic +basic interpreter is supported on all platforms with minimal line +control capabilities. + +On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and +:mod:`readline` support, a new interactive shell is used by default. +This one supports color, multiline editing, history browsing, and +paste mode. To disable color, see :ref:`using-on-controlling-color` for +details. Function keys provide some additional functionality. +``F1`` enters the interactive help browser :mod:`pydoc`. +``F2`` allows for browsing command-line history without output nor the +:term:`>>>` and :term:`...` prompts. ``F3`` enters "paste mode", which +makes pasting larger blocks of code easier. Press ``F3`` to return to +the regular prompt. + +When using the new interactive shell, exit the shell by typing ``exit`` +or ``quit``. Adding call parentheses after those commands is not +required. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + .. _tut-error: Error Handling diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index a81941c2d39054..bdd9fe3f2fcd31 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -42,6 +42,7 @@ additional methods of invocation: * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + For more on interactive mode, see :ref:`tut-interac`. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an From 058bc7ff80b2309c8870beb4fcd30244c081af2b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 5 May 2024 12:20:24 +0200 Subject: [PATCH 72/78] Apply suggestions from code review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/tutorial/appendix.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 812f2473327377..10eb1432175a84 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -19,14 +19,14 @@ On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. -``F1`` enters the interactive help browser :mod:`pydoc`. -``F2`` allows for browsing command-line history without output nor the -:term:`>>>` and :term:`...` prompts. ``F3`` enters "paste mode", which -makes pasting larger blocks of code easier. Press ``F3`` to return to +:kbd:`F1` enters the interactive help browser :mod:`pydoc`. +:kbd:`F2` allows for browsing command-line history without output nor the +:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which +makes pasting larger blocks of code easier. Press :kbd:`F3` to return to the regular prompt. -When using the new interactive shell, exit the shell by typing ``exit`` -or ``quit``. Adding call parentheses after those commands is not +When using the new interactive shell, exit the shell by typing :kbd:`exit` +or :kbd:`quit`. Adding call parentheses after those commands is not required. If the new interactive shell is not desired, it can be disabled via From 7ac9af85b262c9246020eff67bbad3d37fa95f9d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 5 May 2024 12:21:07 +0200 Subject: [PATCH 73/78] Update Lib/_pyrepl/_minimal_curses.py Co-authored-by: Jelle Zijlstra --- Lib/_pyrepl/_minimal_curses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py index 0f067b52095461..0757fb2c664add 100644 --- a/Lib/_pyrepl/_minimal_curses.py +++ b/Lib/_pyrepl/_minimal_curses.py @@ -24,7 +24,7 @@ def _find_clib(): path = ctypes.util.find_library(lib) if path: return path - raise ModuleNotFoundError("curses library not found", name="_minimal_curses") + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") _clibpath = _find_clib() From 7422f1c73d31e22f487f09898af3870c9bcc81d4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 5 May 2024 12:24:01 +0200 Subject: [PATCH 74/78] Use more specific exceptions --- Lib/_pyrepl/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index fd7314544b4757..52455dd5f3692f 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -25,10 +25,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): run_interactive = None try: if not os.isatty(sys.stdin.fileno()): - raise ImportError + raise RuntimeError("pyrepl cannot work if stdin it's not a tty") from .simple_interact import check if not check(): - raise ImportError + raise RuntimeError("pyrepl checks failed") from .simple_interact import run_multiline_interactive_console run_interactive = run_multiline_interactive_console except Exception as e: From d2de559f971500bb24ca7897a0773317c7bc169c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 5 May 2024 17:11:23 +0200 Subject: [PATCH 75/78] Use better error messages --- Lib/_pyrepl/__main__.py | 9 +++++---- Lib/_pyrepl/simple_interact.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 52455dd5f3692f..417ee17adc83d3 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -24,15 +24,16 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): # run_interactive = None try: + import errno if not os.isatty(sys.stdin.fileno()): - raise RuntimeError("pyrepl cannot work if stdin it's not a tty") + raise OSError(errno.ENOTTY, "tty required", "stdin") from .simple_interact import check - if not check(): - raise RuntimeError("pyrepl checks failed") + if err := check(): + raise RuntimeError(err) from .simple_interact import run_multiline_interactive_console run_interactive = run_multiline_interactive_console except Exception as e: - print(f"Warning: 'import _pyrepl' failed with '{e}'", file=sys.stderr) + print(f"warning: can't use pyrepl: {e}", file=sys.stderr) CAN_USE_PYREPL = False if run_interactive is None: return sys._baserepl() diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 70f9e009d840e5..ce79d0d547ebce 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -37,13 +37,13 @@ from .unix_console import _error -def check() -> bool: - """Returns False if there is a problem initializing the state.""" +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" try: _get_reader() - except _error: - return False - return True + except _error as e: + return str(e) or repr(e) or "unknown error" + return "" def _strip_final_indent(text: str) -> str: From 3053b39e6b0274c354c3f56338dba1a5f9abef8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 5 May 2024 17:32:30 +0200 Subject: [PATCH 76/78] Add to What's New --- Doc/whatsnew/3.13.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 269a7cc985ad19..9d8d89c5f53214 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -97,6 +97,34 @@ New typing features: New Features ============ +A Better Interactive Interpreter +-------------------------------- + +On Unix-like systems like Linux or macOS, Python now uses a new +:term:`interactive` shell. When the user starts the :term:`REPL` +from a tty, and both :mod:`curses` and :mod:`readline` are available, +the interactive shell now supports the following new features: + +* colorized prompts; +* multiline editing with history preservation; +* interactive help browsing using :kbd:`F1` with a separate command + history; +* history browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts; +* "paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt); +* ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit` without the need to use call parentheses after the + command name. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + +For more on interactive mode, see :ref:`tut-interac`. + +(Contributed by Pablo Galindo Galgado, Łukasz Langa, and +Lysandros Nikolaou in :gh:`111201`.) + Improved Error Messages ----------------------- From 149658bf22b89c5298a9237e13dbd09a2cc168ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 5 May 2024 17:43:26 +0200 Subject: [PATCH 77/78] Forgot the PyPy attribution in "What's New" --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9d8d89c5f53214..5288221f6199f7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -123,7 +123,7 @@ the :envvar:`PYTHON_BASIC_REPL` environment variable. For more on interactive mode, see :ref:`tut-interac`. (Contributed by Pablo Galindo Galgado, Łukasz Langa, and -Lysandros Nikolaou in :gh:`111201`.) +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project.) Improved Error Messages ----------------------- From be2a0c92f44dd721b790c015ce7bca451280fed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 5 May 2024 19:49:40 +0200 Subject: [PATCH 78/78] let's never talk about this Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5288221f6199f7..7bd9a7804253dc 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -122,7 +122,7 @@ the :envvar:`PYTHON_BASIC_REPL` environment variable. For more on interactive mode, see :ref:`tut-interac`. -(Contributed by Pablo Galindo Galgado, Łukasz Langa, and +(Contributed by Pablo Galindo Salgado, Łukasz Langa, and Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project.) Improved Error Messages