Browse Source

releaseok build now ensures README.rst is ASCII. Closes #1889

Ask Solem 11 years ago
parent
commit
a727a66208
3 changed files with 59 additions and 16 deletions
  1. 5 5
      README.rst
  2. 34 10
      extra/release/sphinx-to-rst.py
  3. 20 1
      pavement.py

+ 5 - 5
README.rst

@@ -81,8 +81,8 @@ getting started tutorials:
 .. _`Next steps`:
 .. _`Next steps`:
     http://docs.celeryproject.org/en/latest/getting-started/next-steps.html
     http://docs.celeryproject.org/en/latest/getting-started/next-steps.html
 
 
-Celery is
-==========
+Celery is...
+============
 
 
 - **Simple**
 - **Simple**
 
 
@@ -119,8 +119,8 @@ Celery is…
     Custom pool implementations, serializers, compression schemes, logging,
     Custom pool implementations, serializers, compression schemes, logging,
     schedulers, consumers, producers, autoscalers, broker transports and much more.
     schedulers, consumers, producers, autoscalers, broker transports and much more.
 
 
-It supports
-============
+It supports...
+==============
 
 
     - **Message Transports**
     - **Message Transports**
 
 
@@ -128,7 +128,7 @@ It supports…
         - MongoDB_ (experimental), Amazon SQS (experimental),
         - MongoDB_ (experimental), Amazon SQS (experimental),
         - CouchDB_ (experimental), SQLAlchemy_ (experimental),
         - CouchDB_ (experimental), SQLAlchemy_ (experimental),
         - Django ORM (experimental), `IronMQ`_
         - Django ORM (experimental), `IronMQ`_
-        - and more
+        - and more...
 
 
     - **Concurrency**
     - **Concurrency**
 
 

+ 34 - 10
extra/release/sphinx-to-rst.py

@@ -1,22 +1,35 @@
 #!/usr/bin/env python
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+import codecs
 import os
 import os
 import re
 import re
 import sys
 import sys
 
 
 from collections import Callable
 from collections import Callable
 
 
-dirname = ""
+dirname = ''
 
 
 RE_CODE_BLOCK = re.compile(r'.. code-block:: (.+?)\s*$')
 RE_CODE_BLOCK = re.compile(r'.. code-block:: (.+?)\s*$')
 RE_INCLUDE = re.compile(r'.. include:: (.+?)\s*$')
 RE_INCLUDE = re.compile(r'.. include:: (.+?)\s*$')
 RE_REFERENCE = re.compile(r':(.+?):`(.+?)`')
 RE_REFERENCE = re.compile(r':(.+?):`(.+?)`')
+UNITABLE = {'…': '...'}
+X = re.compile(re.escape('…'))
+HEADER = re.compile('^[\=\~\-]+$')
+UNIRE = re.compile('|'.join(re.escape(p) for p in UNITABLE),
+                   re.UNICODE)
+
+
+def _replace_handler(match, key=UNITABLE.__getitem__):
+    return key(match.group(0))
 
 
 
 
 def include_file(lines, pos, match):
 def include_file(lines, pos, match):
     global dirname
     global dirname
     orig_filename = match.groups()[0]
     orig_filename = match.groups()[0]
     filename = os.path.join(dirname, orig_filename)
     filename = os.path.join(dirname, orig_filename)
-    fh = open(filename)
+    fh = codecs.open(filename, encoding='utf-8')
     try:
     try:
         old_dirname = dirname
         old_dirname = dirname
         dirname = os.path.dirname(orig_filename)
         dirname = os.path.dirname(orig_filename)
@@ -28,8 +41,19 @@ def include_file(lines, pos, match):
         fh.close()
         fh.close()
 
 
 
 
+def asciify(lines):
+    prev_diff = None
+    for line in lines:
+        new_line = UNIRE.sub(_replace_handler, line)
+        if prev_diff and HEADER.match(new_line):
+            new_line = ''.join([
+                new_line.rstrip(), new_line[0] * prev_diff, '\n'])
+        prev_diff = len(new_line) - len(line)
+        yield new_line.encode('ascii')
+
+
 def replace_code_block(lines, pos, match):
 def replace_code_block(lines, pos, match):
-    lines[pos] = ""
+    lines[pos] = ''
     curpos = pos - 1
     curpos = pos - 1
     # Find the first previous line with text to append "::" to it.
     # Find the first previous line with text to append "::" to it.
     while True:
     while True:
@@ -39,10 +63,10 @@ def replace_code_block(lines, pos, match):
             break
             break
         curpos -= 1
         curpos -= 1
 
 
-    if lines[prev_line_with_text].endswith(":"):
-        lines[prev_line_with_text] += ":"
+    if lines[prev_line_with_text].endswith(':'):
+        lines[prev_line_with_text] += ':'
     else:
     else:
-        lines[prev_line_with_text] += "::"
+        lines[prev_line_with_text] += '::'
 
 
 TO_RST_MAP = {RE_CODE_BLOCK: replace_code_block,
 TO_RST_MAP = {RE_CODE_BLOCK: replace_code_block,
               RE_REFERENCE: r'``\2``',
               RE_REFERENCE: r'``\2``',
@@ -60,17 +84,17 @@ def _process(lines):
                     line = lines[i]
                     line = lines[i]
             else:
             else:
                 lines[i] = regex.sub(alt, line)
                 lines[i] = regex.sub(alt, line)
-    return lines
+    return asciify(lines)
 
 
 
 
 def sphinx_to_rst(fh):
 def sphinx_to_rst(fh):
-    return "".join(_process(fh))
+    return ''.join(_process(fh))
 
 
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     global dirname
     global dirname
     dirname = os.path.dirname(sys.argv[1])
     dirname = os.path.dirname(sys.argv[1])
-    fh = open(sys.argv[1])
+    fh = codecs.open(sys.argv[1], encoding='utf-8')
     try:
     try:
         print(sphinx_to_rst(fh))
         print(sphinx_to_rst(fh))
     finally:
     finally:

+ 20 - 1
pavement.py

@@ -1,4 +1,8 @@
+from __future__ import print_function
+
 import sys
 import sys
+import traceback
+
 from paver.easy import task, sh, cmdopts, path, needs, options, Bunch
 from paver.easy import task, sh, cmdopts, path, needs, options, Bunch
 from paver import doctools  # noqa
 from paver import doctools  # noqa
 from paver.setuputils import setup  # noqa
 from paver.setuputils import setup  # noqa
@@ -95,11 +99,25 @@ def clean_contributing(options):
     path('CONTRIBUTING.rst').unlink_p()
     path('CONTRIBUTING.rst').unlink_p()
 
 
 
 
+@task
+def verify_readme(options):
+    with open('README.rst') as fp:
+        try:
+            fp.read().encode('ascii')
+        except Exception:
+            print('README contains non-ascii characters', file=sys.stderr)
+            print('Original exception below...', file=sys.stderr)
+            traceback.print_stack(file=sys.stderr)
+            sh('false')
+
+
 @task
 @task
 @needs('clean_readme')
 @needs('clean_readme')
 def readme(options):
 def readme(options):
     sh('{0} extra/release/sphinx-to-rst.py docs/templates/readme.txt \
     sh('{0} extra/release/sphinx-to-rst.py docs/templates/readme.txt \
             > README.rst'.format(sys.executable))
             > README.rst'.format(sys.executable))
+    verify_readme()
+
 
 
 @task
 @task
 @needs('clean_contributing')
 @needs('clean_contributing')
@@ -107,6 +125,7 @@ def contributing(options):
     sh('{0} extra/release/sphinx-to-rst.py docs/contributing.rst \
     sh('{0} extra/release/sphinx-to-rst.py docs/contributing.rst \
             > CONTRIBUTING.rst'.format(sys.executable))
             > CONTRIBUTING.rst'.format(sys.executable))
 
 
+
 @task
 @task
 def bump(options):
 def bump(options):
     sh("extra/release/bump_version.py \
     sh("extra/release/bump_version.py \
@@ -166,7 +185,7 @@ def gitcleanforce(options):
 
 
 @task
 @task
 @needs('flakes', 'autodoc', 'verifyindex',
 @needs('flakes', 'autodoc', 'verifyindex',
-       'verifyconfigref', 'test', 'gitclean')
+       'verifyconfigref', 'verify_readme', 'test', 'gitclean')
 def releaseok(options):
 def releaseok(options):
     pass
     pass