Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion Doc/library/ast.rst
Original file line number Diff line number Diff line change
Expand Up @@ -2480,7 +2480,7 @@ and classes for traversing abstract syntax trees:
node = YourTransformer().visit(node)


.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False)
.. function:: dump(node, annotate_fields=True, include_attributes=False, *, color=False, indent=None, show_empty=False)

Return a formatted dump of the tree in *node*. This is mainly useful for
debugging purposes. If *annotate_fields* is true (by default),
Expand All @@ -2490,6 +2490,10 @@ and classes for traversing abstract syntax trees:
numbers and column offsets are not dumped by default. If this is wanted,
*include_attributes* can be set to true.

If *color* is ``True``, the returned string is syntax highlighted using
ANSI escape sequences.
If ``False`` (the default), colored output is always disabled.

If *indent* is a non-negative integer or string, then the tree will be
pretty-printed with that indent level. An indent level
of 0, negative, or ``""`` will only insert newlines. ``None`` (the default)
Expand Down Expand Up @@ -2527,6 +2531,9 @@ and classes for traversing abstract syntax trees:
.. versionchanged:: 3.15
Omit optional ``Load()`` values by default.

.. versionchanged:: next
Added the *color* parameter.


.. _ast-compiler-flags:

Expand Down Expand Up @@ -2584,6 +2591,10 @@ Command-line usage

.. versionadded:: 3.9

.. versionchanged:: next
The output is now syntax highlighted by default. This can be
:ref:`controlled using environment variables <using-on-controlling-color>`.

The :mod:`!ast` module can be executed as a script from the command line.
It is as simple as:

Expand Down
14 changes: 14 additions & 0 deletions Doc/whatsnew/3.15.rst
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,20 @@ array
(Contributed by Sergey B Kirpichev in :gh:`146238`.)


ast
---

* Add *color* parameter to :func:`~ast.dump`.
If ``True``, the returned string is syntax highlighted using ANSI escape
sequences.
If ``False`` (the default), colored output is always disabled.
(Contributed by Stan Ulbrych in :gh:`148981`.)

* The :ref:`command-line <ast-cli>` output is now syntax highlighted by default.
This can be :ref:`controlled using environment variables <using-on-controlling-color>`.
(Contributed by Stan Ulbrych in :gh:`148981`.)


base64
------

Expand Down
15 changes: 15 additions & 0 deletions Lib/_colorize.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,17 @@ class Argparse(ThemeSection):
message: str = ANSIColors.MAGENTA


@dataclass(frozen=True, kw_only=True)
class Ast(ThemeSection):
node: str = ANSIColors.CYAN
field: str = ANSIColors.BLUE
attribute: str = ANSIColors.GREY
string: str = ANSIColors.GREEN
number: str = ANSIColors.YELLOW
keyword: str = ANSIColors.BOLD_BLUE
reset: str = ANSIColors.RESET


@dataclass(frozen=True, kw_only=True)
class Difflib(ThemeSection):
"""A 'git diff'-like theme for `difflib.unified_diff`."""
Expand Down Expand Up @@ -405,6 +416,7 @@ class Theme:
below.
"""
argparse: Argparse = field(default_factory=Argparse)
ast: Ast = field(default_factory=Ast)
difflib: Difflib = field(default_factory=Difflib)
fancycompleter: FancyCompleter = field(default_factory=FancyCompleter)
http_server: HttpServer = field(default_factory=HttpServer)
Expand All @@ -418,6 +430,7 @@ def copy_with(
self,
*,
argparse: Argparse | None = None,
ast: Ast | None = None,
difflib: Difflib | None = None,
fancycompleter: FancyCompleter | None = None,
http_server: HttpServer | None = None,
Expand All @@ -434,6 +447,7 @@ def copy_with(
"""
return type(self)(
argparse=argparse or self.argparse,
ast=ast or self.ast,
difflib=difflib or self.difflib,
fancycompleter=fancycompleter or self.fancycompleter,
http_server=http_server or self.http_server,
Expand All @@ -454,6 +468,7 @@ def no_colors(cls) -> Self:
"""
return cls(
argparse=Argparse.no_colors(),
ast=Ast.no_colors(),
difflib=Difflib.no_colors(),
fancycompleter=FancyCompleter.no_colors(),
http_server=HttpServer.no_colors(),
Expand Down
54 changes: 38 additions & 16 deletions Lib/ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
:license: Python License.
"""
from _ast import *
lazy from _colorize import can_colorize, get_theme


def parse(source, filename='<unknown>', mode='exec', *,
Expand Down Expand Up @@ -117,21 +118,32 @@ def _convert_literal(node):
def dump(
node, annotate_fields=True, include_attributes=False,
*,
indent=None, show_empty=False,
color=False, indent=None, show_empty=False,
):
"""
Return a formatted dump of the tree in node. This is mainly useful for
debugging purposes. If annotate_fields is true (by default),
the returned string will show the names and the values for fields.
If annotate_fields is false, the result string will be more compact by
omitting unambiguous field names. Attributes such as line
numbers and column offsets are not dumped by default. If this is wanted,
include_attributes can be set to true. If indent is a non-negative
integer or string, then the tree will be pretty-printed with that indent
level. None (the default) selects the single line representation.
debugging purposes.

If annotate_fields is true (by default), the returned string will show the
names and the values for fields. If annotate_fields is false, the result
string will be more compact by omitting unambiguous field names.

Attributes such as line numbers and column offsets are not dumped by default.
If this is wanted, include_attributes can be set to true.

If color is true, the returned string is syntax highlighted using ANSI
escape sequences. If color is false (the default), colored output is always
disabled.

If indent is a non-negative integer or string, then the tree will be
pretty-printed with that indent level. If indent is None (the default),
the tree is dumped on a single line.

If show_empty is False, then empty lists and fields that are None
will be omitted from the output for better readability.
"""
t = get_theme(force_color=color, force_no_color=not color).ast

def _format(node, level=0):
if indent is not None:
level += 1
Expand Down Expand Up @@ -166,15 +178,17 @@ def _format(node, level=0):
field_type = cls._field_types.get(name, object)
if field_type is expr_context:
if not keywords:
args_buffer.append(repr(value))
args_buffer.append(
f'{t.node}{type(value).__name__}'
f'{t.reset}()')
continue
if not keywords:
args.extend(args_buffer)
args_buffer = []
value, simple = _format(value, level)
allsimple = allsimple and simple
if keywords:
args.append('%s=%s' % (name, value))
args.append(f'{t.field}{name}{t.reset}={value}')
else:
args.append(value)
if include_attributes and node._attributes:
Expand All @@ -187,14 +201,21 @@ def _format(node, level=0):
continue
value, simple = _format(value, level)
allsimple = allsimple and simple
args.append('%s=%s' % (name, value))
args.append(f'{t.attribute}{name}{t.reset}={value}')
cls_name = f'{t.node}{cls.__name__}{t.reset}'
if allsimple and len(args) <= 3:
return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args
return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False
return f'{cls_name}({", ".join(args)})', not args
return f'{cls_name}({prefix}{sep.join(args)})', False
elif isinstance(node, list):
if not node:
return '[]', True
return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False
if isinstance(node, bool) or node is None or node is Ellipsis:
return f'{t.keyword}{node!r}{t.reset}', True
if isinstance(node, (int, float, complex)):
return f'{t.number}{node!r}{t.reset}', True
if isinstance(node, (str, bytes)):
return f'{t.string}{node!r}{t.reset}', True
return repr(node), True

if not isinstance(node, AST):
Expand Down Expand Up @@ -642,7 +663,7 @@ def main(args=None):
import argparse
import sys

parser = argparse.ArgumentParser(color=True)
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('infile', nargs='?', default='-',
help='the file to parse; defaults to stdin')
parser.add_argument('-m', '--mode', default='exec',
Expand All @@ -661,7 +682,7 @@ def main(args=None):
'(for example, 3.10)')
parser.add_argument('-O', '--optimize',
type=int, default=-1, metavar='LEVEL',
help='optimization level for parser (default -1)')
help='optimization level for parser')
parser.add_argument('--show-empty', default=False, action='store_true',
help='show empty lists and fields in dump output')
args = parser.parse_args(args)
Expand All @@ -688,6 +709,7 @@ def main(args=None):
tree = parse(source, name, args.mode, type_comments=args.no_type_comments,
feature_version=feature_version, optimize=args.optimize)
print(dump(tree, include_attributes=args.include_attributes,
color=can_colorize(file=sys.stdout),
indent=args.indent, show_empty=args.show_empty))

if __name__ == '__main__':
Expand Down
46 changes: 30 additions & 16 deletions Lib/dataclasses.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import sys
import types
import inspect
import keyword
import itertools
import annotationlib
import abc
from reprlib import recursive_repr
lazy import copy
lazy import inspect
lazy import re


Expand Down Expand Up @@ -988,6 +988,28 @@ def _hash_exception(cls, fields, func_builder):
# See https://bugs.python.org/issue32929#msg312829 for an if-statement
# version of this table.

# A non-data descriptor to autogenerate class docstring
# from the signature of its __init__ method on demand.
# The primary reason is to be able to lazy import `inspect` module.
class _AutoDocstring:

def __get__(self, _obj, cls):
try:
# In some cases fetching a signature is not possible.
# But, we surely should not fail in this case.
text_sig = str(inspect.signature(
cls,
annotation_format=annotationlib.Format.FORWARDREF,
)).replace(' -> None', '')
except TypeError, ValueError:
text_sig = ''

doc = cls.__name__ + text_sig
setattr(cls, '__doc__', doc)
return doc

_auto_docstring = _AutoDocstring()


def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen,
match_args, kw_only, slots, weakref_slot):
Expand Down Expand Up @@ -1215,23 +1237,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen,
if hash_action:
cls.__hash__ = hash_action(cls, field_list, func_builder)

# Generate the methods and add them to the class. This needs to be done
# before the __doc__ logic below, since inspect will look at the __init__
# signature.
# Generate the methods and add them to the class.
func_builder.add_fns_to_class(cls)

if not getattr(cls, '__doc__'):
# Create a class doc-string.
try:
# In some cases fetching a signature is not possible.
# But, we surely should not fail in this case.
text_sig = str(inspect.signature(
cls,
annotation_format=annotationlib.Format.FORWARDREF,
)).replace(' -> None', '')
except (TypeError, ValueError):
text_sig = ''
cls.__doc__ = (cls.__name__ + text_sig)
# Create a class doc-string lazily via descriptor protocol
# to avoid importing `inspect` module.
cls.__doc__ = _auto_docstring

if match_args:
# I could probably compute this once.
Expand Down Expand Up @@ -1391,8 +1403,10 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields):
# make an update, since all closures for a class will share a
# given cell.
for member in newcls.__dict__.values():

# If this is a wrapped function, unwrap it.
member = inspect.unwrap(member)
if not isinstance(member, type) and hasattr(member, '__wrapped__'):
member = inspect.unwrap(member)

if isinstance(member, types.FunctionType):
if _update_func_cell_for__class__(member, cls, newcls):
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test__colorize.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class TestImportTime(unittest.TestCase):
@cpython_only
def test_lazy_import(self):
import_helper.ensure_lazy_imports(
"_colorize", {"copy", "re"}
"_colorize", {"copy", "re", "inspect"}
)


Expand Down
11 changes: 11 additions & 0 deletions Lib/test/test_ast/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -1705,6 +1705,16 @@ def check_text(code, empty, full, **kwargs):
full="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)], type_ignores=[])",
)

def test_dump_with_color(self):
node = ast.parse("x = 1")
self.assertNotIn("\x1b[", ast.dump(node))
self.assertNotIn("\x1b[", ast.dump(node, color=False))
self.assertIn("\x1b[", ast.dump(node, color=True))

node = ast.Constant(value="\x1b[31m")
self.assertEqual(ast.dump(node), "Constant(value='\\x1b[31m')")
self.assertIn("'\\x1b[31m'", ast.dump(node, color=True))

def test_copy_location(self):
src = ast.parse('1 + 1', mode='eval')
src.body.right = ast.copy_location(ast.Constant(2), src.body.right)
Expand Down Expand Up @@ -3415,6 +3425,7 @@ def test_subinterpreter(self):
self.assertEqual(res, 0)


@support.force_not_colorized_test_class
class CommandLineTests(unittest.TestCase):
def setUp(self):
self.filename = tempfile.mktemp()
Expand Down
26 changes: 25 additions & 1 deletion Lib/test/test_dataclasses/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,21 @@
import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation.

from test import support
from test.support import import_helper
from test.support import cpython_only, import_helper

# Just any custom exception we can catch.
class CustomError(Exception): pass


class TestImportTime(unittest.TestCase):

@cpython_only
def test_lazy_import(self):
import_helper.ensure_lazy_imports(
"dataclasses", {"inspect", "re", "copy"}
)


class TestCase(unittest.TestCase):
def test_no_fields(self):
@dataclass
Expand Down Expand Up @@ -2309,6 +2319,20 @@ class C:

self.assertDocStrEqual(C.__doc__, "C()")

def test_docstring_slotted(self):
@dataclass(slots=True)
class C:
x: int

self.assertDocStrEqual(C.__doc__, "C(x:int)")

def test_docstring_recursive(self):
@dataclass()
class C:
x: list[C]

self.assertDocStrEqual(C.__doc__, "C(x:list[test.test_dataclasses.TestDocString.test_docstring_recursive.<locals>.C])")

def test_docstring_one_field(self):
@dataclass
class C:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Reduce the import time of :mod:`dataclasses` module by ~20%.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add *color* parameter to :func:`ast.dump`.
Loading