diff --git a/.gitignore b/.gitignore index 52736b3..05a7ff2 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ poetry.lock docs/_build/ # sphinxnotes-any >= 2.5 docs/.any* +.worktrees/ diff --git a/docs/api.rst b/docs/api.rst index c9e9742..270f453 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -55,22 +55,22 @@ Context refers to the dynamic content of a Jinja template. It can be: Our dedicated data type (:py:class:`sphinxnotes.render.ParsedData`), or any Python ``dict``. -:py:class:`~sphinxnotes.render.PendingContext`: +:py:class:`~sphinxnotes.render.UnresolvedContext`: Context that is not yet available. For example, it may contain :py:class:`unparsed data `, remote data, and more. - :py:class:`PendingContext` can be resolved to + :py:class:`UnresolvedContext` can be resolved to :py:class:`~sphinxnotes.render.ResolvedContext` by calling - :py:meth:`~sphinxnotes.render.PendingContext.resolve`. + :py:meth:`~sphinxnotes.render.UnresolvedContext.resolve`. .. autotype:: sphinxnotes.render.ResolvedContext -.. autoclass:: sphinxnotes.render.PendingContext +.. autoclass:: sphinxnotes.render.UnresolvedContext :members: resolve -``PendingContext`` Implementations ----------------------------------- +``UnresolvedContext`` Implementations +------------------------------------- .. autoclass:: sphinxnotes.render.UnparsedData :show-inheritance: diff --git a/src/sphinxnotes/render/__init__.py b/src/sphinxnotes/render/__init__.py index ef98a23..e66cc0e 100644 --- a/src/sphinxnotes/render/__init__.py +++ b/src/sphinxnotes/render/__init__.py @@ -22,7 +22,7 @@ Schema, ) from .template import Phase, Template -from .ctx import PendingContext, ResolvedContext +from .ctx import UnresolvedContext, ResolvedContext from .ctxnodes import pending_node from .extractx import ( extra_context, @@ -56,7 +56,7 @@ 'Schema', 'Phase', 'Template', - 'PendingContext', + 'UnresolvedContext', 'ResolvedContext', 'ParsingPhaseExtraContext', 'ParsedPhaseExtraContext', diff --git a/src/sphinxnotes/render/ctx.py b/src/sphinxnotes/render/ctx.py index 2da40ac..32f8c41 100644 --- a/src/sphinxnotes/render/ctx.py +++ b/src/sphinxnotes/render/ctx.py @@ -7,63 +7,17 @@ from typing import Any from abc import ABC, abstractmethod from collections.abc import Hashable -from dataclasses import dataclass - from .data import ParsedData -from .utils import Unpicklable type ResolvedContext = ParsedData | dict[str, Any] """Resolved context types used by template rendering.""" -@dataclass -class PendingContextRef: - """An abstract reference to :class:`PendingContext`.""" - - ref: int - chksum: int - - def __hash__(self) -> int: - return hash((self.ref, self.chksum)) - - -class PendingContext(ABC, Unpicklable, Hashable): - """An abstract representation of context that is not currently available.""" +class UnresolvedContext(ABC, Hashable): + """An abstract representation of context that will be resolved later.""" @abstractmethod def resolve(self) -> ResolvedContext: """This method will be called when rendering to get the available :py:type:`ResolvedContext`.""" ... - - -class PendingContextStorage: - """Area for temporarily storing :py:class:`PendingContext`. - - This class is intended to solve the problem that: - - Some :class:`PendingContext` objects are :class:`Unpicklable`, so they cannot be held - by :class:`pending_node` (as ``pending_node`` will be pickled along with - the docutils doctree) - - This class maintains a mapping from :class:`PendingContextRef` -> :class:`PendingContext`. - ``pending_node`` owns the ``PendingContextRef``, and can retrieve the context - by calling :py:meth:`retrieve`. - """ - - _next_id: int - _data: dict[PendingContextRef, PendingContext] = {} - - def __init__(self) -> None: - self._next_id = 0 - self._data = {} - - def stash(self, pending: PendingContext) -> PendingContextRef: - ref = PendingContextRef(self._next_id, hash(pending)) - self._next_id += 1 - self._data[ref] = pending - return ref - - def retrieve(self, ref: PendingContextRef) -> PendingContext | None: - data = self._data.pop(ref, None) - return data diff --git a/src/sphinxnotes/render/ctxnodes.py b/src/sphinxnotes/render/ctxnodes.py index 6f1a4fc..a7ade16 100644 --- a/src/sphinxnotes/render/ctxnodes.py +++ b/src/sphinxnotes/render/ctxnodes.py @@ -1,15 +1,15 @@ from __future__ import annotations from typing import TYPE_CHECKING, override +import pickle from pprint import pformat from docutils import nodes from docutils.parsers.rst.states import Inliner -from .template import Template +from .data import ValueWrapper, ParsedData +from .template import Template, Phase from .ctx import ( - PendingContextRef, - PendingContext, - PendingContextStorage, + UnresolvedContext, ResolvedContext, ) from .markup import MarkupRenderer @@ -21,7 +21,7 @@ ) if TYPE_CHECKING: - from typing import Any, Callable, ClassVar + from typing import Any, Callable from .markup import Host from .ctx import ResolvedContext @@ -30,7 +30,7 @@ class pending_node(nodes.Element): """A docutils node to be rendered.""" # The context to be rendered by Jinja template. - ctx: PendingContextRef | ResolvedContext + ctx: UnresolvedContext | ResolvedContext # The extra context as supplement to ctx. extra: dict[str, Any] #: Jinja template for rendering the context. @@ -39,16 +39,12 @@ class pending_node(nodes.Element): inline: bool #: Whether the rendering pipeline is finished (failed is also finished). rendered: bool - - #: Mapping of PendingContextRef -> PendingContext. - #: - #: NOTE: ``PendingContextStorage`` holds Unpicklable data (``PendingContext``) - #: but it is doesn't matters :-), cause pickle doesn't deal with ClassVar. - _PENDING_CONTEXTS: ClassVar[PendingContextStorage] = PendingContextStorage() + #: Stored pickling error for later-phase unresolved context. + _ctx_pickle_error: Exception | None def __init__( self, - ctx: PendingContext | ResolvedContext, + ctx: UnresolvedContext | ResolvedContext, tmpl: Template, inline: bool = False, rawsource='', @@ -56,17 +52,20 @@ def __init__( **attributes, ) -> None: super().__init__(rawsource, *children, **attributes) - if not isinstance(ctx, PendingContext): - self.ctx = ctx - else: - self.ctx = self._PENDING_CONTEXTS.stash(ctx) + self._ctx_pickle_error = None + if isinstance(ctx, UnresolvedContext) and tmpl.phase != Phase.Parsing: + try: + pickle.dumps(ctx) + except Exception as exc: + self._ctx_pickle_error = exc + self.ctx = ctx self.extra = {} self.template = tmpl self.inline = inline self.rendered = False # Init hook lists. - self._pending_context_hooks = [] + self._unresolved_context_hooks = [] self._resolved_data_hooks = [] self._markup_text_hooks = [] self._rendered_nodes_hooks = [] @@ -75,7 +74,7 @@ def render(self, host: Host) -> None: """ The core function for rendering context and template to docutils nodes. - 1. PendingContextRef -> PendingContext -> ResolvedContext + 1. UnresolvedContext -> ResolvedContext 2. TemplateRenderer.render(ResolvedContext) -> Markup Text (``str``) 3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node]) """ @@ -97,29 +96,30 @@ def err_report() -> Report: return report return Report('Render Report', 'ERROR', source=self.source, line=self.line) - # 1. Prepare context for Jinja template. - if isinstance(self.ctx, PendingContextRef): - report.text('Pending context ref:') - report.code(pformat(self.ctx), lang='python') - - pdata = self._PENDING_CONTEXTS.retrieve(self.ctx) - if pdata is None: - report = err_report() - report.text(f'Failed to retrieve pending context from ref {self.ctx}') - self += report - return None + if self._ctx_pickle_error is not None: + report = err_report() + report.text( + f'UnresolvedContext used by {self.template.phase} phase templates ' + 'must be picklable:' + ) + report.exception(self._ctx_pickle_error) + self += report + return None - report.text('Pending context:') + # 1. Prepare context for Jinja template. + if isinstance(self.ctx, UnresolvedContext): + pdata = self.ctx + report.text('Unresolved context:') report.code(pformat(pdata), lang='python') - for hook in self._pending_context_hooks: + for hook in self._unresolved_context_hooks: hook(self, pdata) try: ctx = self.ctx = pdata.resolve() except Exception as e: report = err_report() - report.text('Failed to resolve pending context:') + report.text('Failed to resolve unresolved context:') report.exception(e) self += report return None @@ -221,18 +221,18 @@ def unwrap_and_replace_self_inline(self, inliner: Report.Inliner) -> None: """Hooks for processing render intermediate products.""" - type PendingContextHook = Callable[[pending_node, PendingContext], None] + type UnresolvedContextHook = Callable[[pending_node, UnresolvedContext], None] type ResolvedContextHook = Callable[[pending_node, ResolvedContext], None] type MarkupTextHook = Callable[[pending_node, str], str] type RenderedNodesHook = Callable[[pending_node, list[nodes.Node]], None] - _pending_context_hooks: list[PendingContextHook] + _unresolved_context_hooks: list[UnresolvedContextHook] _resolved_data_hooks: list[ResolvedContextHook] _markup_text_hooks: list[MarkupTextHook] _rendered_nodes_hooks: list[RenderedNodesHook] - def hook_pending_context(self, hook: PendingContextHook) -> None: - self._pending_context_hooks.append(hook) + def hook_unresolved_context(self, hook: UnresolvedContextHook) -> None: + self._unresolved_context_hooks.append(hook) def hook_resolved_context(self, hook: ResolvedContextHook) -> None: self._resolved_data_hooks.append(hook) @@ -259,3 +259,16 @@ def copy(self) -> Any: def deepcopy(self) -> Any: # NOTE: Same to :meth:`copy`. return self.copy() + + @override + def astext(self) -> str: + ctx = self.ctx + if isinstance(ctx, UnresolvedContext): + try: + ctx = ctx.resolve() + except Exception: + return '' + if isinstance(ctx, ParsedData): + return ValueWrapper(ctx.content).as_str() or '' + else: + return '' diff --git a/src/sphinxnotes/render/data.py b/src/sphinxnotes/render/data.py index 6f47e3f..0897fec 100644 --- a/src/sphinxnotes/render/data.py +++ b/src/sphinxnotes/render/data.py @@ -14,8 +14,6 @@ from dataclasses import dataclass, asdict, field as dataclass_field from ast import literal_eval -from .utils import Unpicklable - if TYPE_CHECKING: from typing import Any, Callable, Generator, Self @@ -287,7 +285,7 @@ def asdict(self) -> dict[str, Any]: @dataclass -class Field(Unpicklable): +class Field: #: Type of element. etype: type = str #: Type of container (if the field holds multiple values). @@ -374,8 +372,9 @@ def parse(self, rawval: str | None) -> Value: raise ValueError(f"Failed to parse '{rawval}' as {self.etype}: {e}") from e def __getattr__(self, name: str) -> Value: - if name in self.flags: - return self.flags[name] + flags = self.__dict__.get('flags') + if flags is not None and name in flags: + return flags[name] raise AttributeError(name) @@ -491,7 +490,7 @@ def by_option_store_value_error(opt: ByOption) -> ValueError: @dataclass(frozen=True) -class Schema(Unpicklable): +class Schema: name: Field | None attrs: dict[str, Field] | Field content: Field | None diff --git a/src/sphinxnotes/render/ext/adhoc.py b/src/sphinxnotes/render/ext/adhoc.py index 67e05c9..00e3f2a 100644 --- a/src/sphinxnotes/render/ext/adhoc.py +++ b/src/sphinxnotes/render/ext/adhoc.py @@ -33,7 +33,7 @@ from types import ModuleType from docutils.utils import Reporter from sphinx.util.typing import RoleFunction - from .. import PendingContext, ResolvedContext + from .. import UnresolvedContext, ResolvedContext # Keys of env.temp_data. @@ -140,7 +140,7 @@ class DataRenderDirective(BaseContextDirective): has_content = True @override - def current_context(self) -> PendingContext | ResolvedContext: + def current_context(self) -> UnresolvedContext | ResolvedContext: return {} @override diff --git a/src/sphinxnotes/render/pipeline.py b/src/sphinxnotes/render/pipeline.py index 3a53114..0bc886b 100644 --- a/src/sphinxnotes/render/pipeline.py +++ b/src/sphinxnotes/render/pipeline.py @@ -11,7 +11,7 @@ from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver from .template import HostWrapper, Phase, Template, Host -from .ctx import PendingContext, ResolvedContext +from .ctx import UnresolvedContext, ResolvedContext from .ctxnodes import pending_node from .extractx import ExtraContextGenerator @@ -83,7 +83,7 @@ def queue_pending_node(self, n: pending_node) -> None: @final def queue_context( - self, ctx: PendingContext | ResolvedContext, tmpl: Template + self, ctx: UnresolvedContext | ResolvedContext, tmpl: Template ) -> pending_node: """A helper method for ``queue_pending_node``.""" pending = pending_node(ctx, tmpl) @@ -162,7 +162,7 @@ class BaseContextSource(Pipeline): """Methods to be implemented.""" @abstractmethod - def current_context(self) -> PendingContext | ResolvedContext: + def current_context(self) -> UnresolvedContext | ResolvedContext: """Return the context to be rendered.""" ... diff --git a/src/sphinxnotes/render/sources.py b/src/sphinxnotes/render/sources.py index d941b10..161bc91 100644 --- a/src/sphinxnotes/render/sources.py +++ b/src/sphinxnotes/render/sources.py @@ -16,14 +16,14 @@ from docutils.parsers.rst import directives from .data import Field, RawData, Schema -from .ctx import PendingContext, ResolvedContext +from .ctx import UnresolvedContext, ResolvedContext from .template import Template from .pipeline import BaseContextSource, BaseContextDirective, BaseContextRole @dataclass -class UnparsedData(PendingContext): - """A pending context which contains raw data and its schema. +class UnparsedData(UnresolvedContext): + """An unresolved context which contains raw data and its schema. Raw data will be parsed when calling ``resolve``. """ @@ -60,7 +60,7 @@ def current_schema(self) -> Schema: @final @override - def current_context(self) -> PendingContext | ResolvedContext: + def current_context(self) -> UnresolvedContext | ResolvedContext: return UnparsedData(self.current_raw_data(), self.current_schema()) diff --git a/tests/conftest.py b/tests/conftest.py index 016532e..281148f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,11 @@ from pathlib import Path +import sys import pytest pytest_plugins = 'sphinx.testing.fixtures' +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) + @pytest.fixture(scope='session') def rootdir() -> Path: diff --git a/tests/test_ctx.py b/tests/test_ctx.py new file mode 100644 index 0000000..8335884 --- /dev/null +++ b/tests/test_ctx.py @@ -0,0 +1,60 @@ +import pickle + +from sphinxnotes.render.ctx import UnresolvedContext +from sphinxnotes.render.ctxnodes import pending_node +from sphinxnotes.render.data import RawData, Schema +from sphinxnotes.render.sources import UnparsedData +from sphinxnotes.render.template import Phase, Template +from sphinxnotes.render.utils import Report, Unpicklable + + +class UnpicklableUnresolvedContext(UnresolvedContext, Unpicklable): + def resolve(self): + return {} + + def __hash__(self) -> int: + return 0 + + +def test_schema_and_unparsed_data_are_picklable(): + schema = Schema.from_dsl( + name='str', + attrs={'age': 'int', 'tags': 'words of str'}, + content='str', + ) + pending = UnparsedData( + RawData('mimi', {'age': '2', 'tags': 'cat cute'}, 'hello'), + schema, + ) + + restored = pickle.loads(pickle.dumps(pending)) + + assert restored.resolve().name == 'mimi' + assert restored.resolve().attrs == {'age': 2, 'tags': ['cat', 'cute']} + assert restored.resolve().content == 'hello' + + +def test_pending_node_reports_unpicklable_unresolved_context_for_later_phase(): + node = pending_node( + UnpicklableUnresolvedContext(), + Template('ignored', phase=Phase.Parsed), + ) + + node.rendered = False + node.render(host=None) + + reports = [child for child in node.children if isinstance(child, Report)] + + assert len(reports) == 1 + assert reports[0].is_error() + assert 'UnresolvedContext' in reports[0].astext() + assert 'picklable' in reports[0].astext() + + +def test_pending_node_allows_unpicklable_unresolved_context_for_parsing_phase(): + node = pending_node( + UnpicklableUnresolvedContext(), + Template('ignored', phase=Phase.Parsing), + ) + + assert isinstance(node.ctx, UnpicklableUnresolvedContext)