From 097164a15faa0123fc37b12f4c918f08f6b43398 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 21:08:36 +1100 Subject: [PATCH 01/10] Install pdflatex in Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index add28760..04691e8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ cache: directories: - $HOME/.cache/pip before_install: + - sudo apt-get install texlive-latex-base - pip install --upgrade pip setuptools # Upgrade pip and setuptools to get ones with `wheel` support - pip install --find-links http://wheels.astropy.org/ --find-links http://wheels2.astropy.org/ --trusted-host wheels.astropy.org --trusted-host wheels2.astropy.org --use-wheel nose numpy matplotlib ${SPHINX_SPEC} script: From f4f65ed90ea4435e0a025aac294fffad79c1f8e8 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 20:27:11 +1100 Subject: [PATCH 02/10] Ensure reference renaming is parallel-safe This prefixes each reference with a token indicative of which docstring it belongs to and then relabels the text of the reference once the doctree is compiled --- numpydoc/numpydoc.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py index c14fb0e4..d45f198c 100644 --- a/numpydoc/numpydoc.py +++ b/numpydoc/numpydoc.py @@ -24,6 +24,7 @@ import sphinx import inspect import collections +import hashlib if sphinx.__version__ < '1.0.1': raise RuntimeError("Sphinx 1.0.1 or newer is required") @@ -36,9 +37,9 @@ sixu = lambda s: unicode(s, 'unicode_escape') -def rename_references(app, what, name, obj, options, lines, - reference_offset=[0]): - # replace reference numbers so that there are no duplicates +def rename_references(app, what, name, obj, options, lines): + # decorate reference numbers so that there are no duplicates + # these are later undecorated in the doctree, in relabel_references references = set() for line in lines: line = line.strip() @@ -48,19 +49,41 @@ def rename_references(app, what, name, obj, options, lines, references.add(m.group(1)) if references: - for r in references: - if r.isdigit(): - new_r = sixu("R%d") % (reference_offset[0] + int(r)) - else: - new_r = sixu("%s%d") % (r, reference_offset[0]) + # we use a hash to mangle the reference name to avoid invalid names + sha = hashlib.sha256() + sha.update(name.encode('utf8')) + prefix = 'R' + sha.hexdigest() + for r in references: + new_r = prefix + '-' + r for i, line in enumerate(lines): lines[i] = lines[i].replace(sixu('[%s]_') % r, sixu('[%s]_') % new_r) lines[i] = lines[i].replace(sixu('.. [%s]') % r, sixu('.. [%s]') % new_r) - reference_offset[0] += len(references) + +def relabel_references(app, doc): + # Change name_ref to ref in label text + from docutils.nodes import citation, Text + from sphinx.addnodes import pending_xref + for citation_node in doc.traverse(citation): + label_node = citation_node[0] + new_text = Text(citation_node['names'][0].split('-')[-1]) + label_node.replace(label_node[0], new_text) + + for id in citation_node['backrefs']: + ref = doc.ids[id] + ref_text = ref[0] + + # Sphinx has created pending_xref nodes with [reftext] text. + def matching_pending_xref(node): + return (isinstance(node, pending_xref) and + node[0].astext() == '[%s]' % ref_text) + + for xref_node in ref.parent.traverse(): + xref_node.replace(xref_node[0], Text('[%s]' % new_text)) + ref.replace(ref_text, new_text.copy()) DEDUPLICATION_TAG = ' !! processed by numpydoc !!' @@ -137,6 +160,7 @@ def setup(app, get_doc_object_=get_doc_object): app.connect('autodoc-process-docstring', mangle_docstrings) app.connect('autodoc-process-signature', mangle_signature) + app.connect('doctree-read', relabel_references) app.add_config_value('numpydoc_edit_link', None, False) app.add_config_value('numpydoc_use_plots', None, False) app.add_config_value('numpydoc_show_class_members', True, True) From 105319d8a9260fb948c9447face8c9e4d3dfeede Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 20:54:48 +1100 Subject: [PATCH 03/10] Fix missing argument --- numpydoc/numpydoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py index d45f198c..0cfb0ca4 100644 --- a/numpydoc/numpydoc.py +++ b/numpydoc/numpydoc.py @@ -81,7 +81,7 @@ def matching_pending_xref(node): return (isinstance(node, pending_xref) and node[0].astext() == '[%s]' % ref_text) - for xref_node in ref.parent.traverse(): + for xref_node in ref.parent.traverse(matching_pending_xref): xref_node.replace(xref_node[0], Text('[%s]' % new_text)) ref.replace(ref_text, new_text.copy()) From d0a433bdc31003f17db8e5ff9dda71d4d3553ab3 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 21:20:20 +1100 Subject: [PATCH 04/10] More TeX packages --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 04691e8f..b02facfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ cache: directories: - $HOME/.cache/pip before_install: - - sudo apt-get install texlive-latex-base + - sudo apt-get install texlive texlive-latex-extra - pip install --upgrade pip setuptools # Upgrade pip and setuptools to get ones with `wheel` support - pip install --find-links http://wheels.astropy.org/ --find-links http://wheels2.astropy.org/ --trusted-host wheels.astropy.org --trusted-host wheels2.astropy.org --use-wheel nose numpy matplotlib ${SPHINX_SPEC} script: From 81986b0b5d94747f2f5bddbb05673d762cd1ee44 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 21:28:25 +1100 Subject: [PATCH 05/10] Some sphinx require latexmk --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b02facfd..fda4f5cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ cache: directories: - $HOME/.cache/pip before_install: - - sudo apt-get install texlive texlive-latex-extra + - sudo apt-get install texlive texlive-latex-extra latexmk - pip install --upgrade pip setuptools # Upgrade pip and setuptools to get ones with `wheel` support - pip install --find-links http://wheels.astropy.org/ --find-links http://wheels2.astropy.org/ --trusted-host wheels.astropy.org --trusted-host wheels2.astropy.org --use-wheel nose numpy matplotlib ${SPHINX_SPEC} script: From edfc03656827603cc9b497af78a9414d873e2396 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 22:26:04 +1100 Subject: [PATCH 06/10] Move imports --- numpydoc/numpydoc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py index 0cfb0ca4..74f1daa9 100644 --- a/numpydoc/numpydoc.py +++ b/numpydoc/numpydoc.py @@ -21,11 +21,14 @@ import sys import re import pydoc -import sphinx import inspect import collections import hashlib +from docutils.nodes import citation, Text +import sphinx +from sphinx.addnodes import pending_xref + if sphinx.__version__ < '1.0.1': raise RuntimeError("Sphinx 1.0.1 or newer is required") @@ -65,8 +68,6 @@ def rename_references(app, what, name, obj, options, lines): def relabel_references(app, doc): # Change name_ref to ref in label text - from docutils.nodes import citation, Text - from sphinx.addnodes import pending_xref for citation_node in doc.traverse(citation): label_node = citation_node[0] new_text = Text(citation_node['names'][0].split('-')[-1]) From 810ddd8539932c2fc777bddf8409e3c8618c8258 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 22:29:16 +1100 Subject: [PATCH 07/10] Fix comment --- numpydoc/numpydoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py index 74f1daa9..db7ea853 100644 --- a/numpydoc/numpydoc.py +++ b/numpydoc/numpydoc.py @@ -67,7 +67,7 @@ def rename_references(app, what, name, obj, options, lines): def relabel_references(app, doc): - # Change name_ref to ref in label text + # Change 'hash-ref' to 'ref' in label text for citation_node in doc.traverse(citation): label_node = citation_node[0] new_text = Text(citation_node['names'][0].split('-')[-1]) From c410820832a471042c8d2da22ef5ec7635ed20bc Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 22:53:32 +1100 Subject: [PATCH 08/10] Warn about #130 --- doc/format.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/format.rst b/doc/format.rst index d5fc0b69..5e59ec17 100644 --- a/doc/format.rst +++ b/doc/format.rst @@ -373,6 +373,12 @@ The sections of the docstring are: should not be required to understand it. References are numbered, starting from one, in the order in which they are cited. + .. warning:: **References will break tables** + + Where references like [1] appear in a tables within a numpydoc + docstring, the table markup will be broken by numpydoc processing. See + `numpydoc issue #130 `_ + 12. **Examples** An optional section for examples, using the `doctest From c2d0ebdb7ce2fea5d7f7ba2f582f8a4233bbe66d Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 22:55:51 +1100 Subject: [PATCH 09/10] Fixes to relabelling of references: * Use label text not normalised text * Split at first - not last * Do not relabel non-docstring content --- numpydoc/numpydoc.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py index db7ea853..ae4b5f4b 100644 --- a/numpydoc/numpydoc.py +++ b/numpydoc/numpydoc.py @@ -27,7 +27,7 @@ from docutils.nodes import citation, Text import sphinx -from sphinx.addnodes import pending_xref +from sphinx.addnodes import pending_xref, desc if sphinx.__version__ < '1.0.1': raise RuntimeError("Sphinx 1.0.1 or newer is required") @@ -40,6 +40,9 @@ sixu = lambda s: unicode(s, 'unicode_escape') +HASH_LEN = 12 + + def rename_references(app, what, name, obj, options, lines): # decorate reference numbers so that there are no duplicates # these are later undecorated in the doctree, in relabel_references @@ -55,7 +58,7 @@ def rename_references(app, what, name, obj, options, lines): # we use a hash to mangle the reference name to avoid invalid names sha = hashlib.sha256() sha.update(name.encode('utf8')) - prefix = 'R' + sha.hexdigest() + prefix = 'R' + sha.hexdigest()[:HASH_LEN] for r in references: new_r = prefix + '-' + r @@ -66,11 +69,23 @@ def rename_references(app, what, name, obj, options, lines): sixu('.. [%s]') % new_r) +def _ascend(node, cls): + while not isinstance(node, cls) and node.parent: + node = node.parent + return node + + def relabel_references(app, doc): # Change 'hash-ref' to 'ref' in label text for citation_node in doc.traverse(citation): + if _ascend(citation_node, desc) is None: + # no desc node in ancestry -> not in a docstring + # XXX: should we also somehow check it's in a References section? + continue label_node = citation_node[0] - new_text = Text(citation_node['names'][0].split('-')[-1]) + prefix, _, new_label = label_node[0].astext().partition('-') + assert len(prefix) == HASH_LEN + 1 + new_text = Text(new_label) label_node.replace(label_node[0], new_text) for id in citation_node['backrefs']: From a8f413faf11e345341d045e3bac3d3587c4419bc Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 1 Nov 2017 23:32:33 +1100 Subject: [PATCH 10/10] Fix logic for identifying non-docscring --- numpydoc/numpydoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py index ae4b5f4b..cba7d567 100644 --- a/numpydoc/numpydoc.py +++ b/numpydoc/numpydoc.py @@ -27,7 +27,7 @@ from docutils.nodes import citation, Text import sphinx -from sphinx.addnodes import pending_xref, desc +from sphinx.addnodes import pending_xref, desc_content if sphinx.__version__ < '1.0.1': raise RuntimeError("Sphinx 1.0.1 or newer is required") @@ -70,7 +70,7 @@ def rename_references(app, what, name, obj, options, lines): def _ascend(node, cls): - while not isinstance(node, cls) and node.parent: + while node and not isinstance(node, cls): node = node.parent return node @@ -78,7 +78,7 @@ def _ascend(node, cls): def relabel_references(app, doc): # Change 'hash-ref' to 'ref' in label text for citation_node in doc.traverse(citation): - if _ascend(citation_node, desc) is None: + if _ascend(citation_node, desc_content) is None: # no desc node in ancestry -> not in a docstring # XXX: should we also somehow check it's in a References section? continue