releaseok build now ensures README.rst is ASCII. Closes #1889

README.rst

@@ -81,8 +81,8 @@ getting started tutorials:
 .. _`Next steps`:
     http://docs.celeryproject.org/en/latest/getting-started/next-steps.html
 
-Celery is…
-==========
+Celery is...
+============
 
 - **Simple**
 
@@ -119,8 +119,8 @@ Celery is…
     Custom pool implementations, serializers, compression schemes, logging,
     schedulers, consumers, producers, autoscalers, broker transports and much more.
 
-It supports…
-============
+It supports...
+==============
 
     - **Message Transports**
 
@@ -128,7 +128,7 @@ It supports…
         - MongoDB_ (experimental), Amazon SQS (experimental),
         - CouchDB_ (experimental), SQLAlchemy_ (experimental),
         - Django ORM (experimental), `IronMQ`_
-        - and more…
+        - and more...
 
     - **Concurrency**
 

extra/release/sphinx-to-rst.py

@@ -1,22 +1,35 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+import codecs
 import os
 import re
 import sys
 
 from collections import Callable
 
-dirname = ""
+dirname = ''
 
 RE_CODE_BLOCK = re.compile(r'.. code-block:: (.+?)\s*$')
 RE_INCLUDE = re.compile(r'.. include:: (.+?)\s*$')
 RE_REFERENCE = re.compile(r':(.+?):`(.+?)`')
+UNITABLE = {'…': '...'}
+X = re.compile(re.escape('…'))
+HEADER = re.compile('^[\=\~\-]+$')
+UNIRE = re.compile('|'.join(re.escape(p) for p in UNITABLE),
+                   re.UNICODE)
+
+
+def _replace_handler(match, key=UNITABLE.__getitem__):
+    return key(match.group(0))
 
 
 def include_file(lines, pos, match):
     global dirname
     orig_filename = match.groups()[0]
     filename = os.path.join(dirname, orig_filename)
-    fh = open(filename)
+    fh = codecs.open(filename, encoding='utf-8')
     try:
         old_dirname = dirname
         dirname = os.path.dirname(orig_filename)
@@ -28,8 +41,19 @@ def include_file(lines, pos, match):
         fh.close()
 
 
+def asciify(lines):
+    prev_diff = None
+    for line in lines:
+        new_line = UNIRE.sub(_replace_handler, line)
+        if prev_diff and HEADER.match(new_line):
+            new_line = ''.join([
+                new_line.rstrip(), new_line[0] * prev_diff, '\n'])
+        prev_diff = len(new_line) - len(line)
+        yield new_line.encode('ascii')
+
+
 def replace_code_block(lines, pos, match):
-    lines[pos] = ""
+    lines[pos] = ''
     curpos = pos - 1
     # Find the first previous line with text to append "::" to it.
     while True:
@@ -39,10 +63,10 @@ def replace_code_block(lines, pos, match):
             break
         curpos -= 1
 
-    if lines[prev_line_with_text].endswith(":"):
-        lines[prev_line_with_text] += ":"
+    if lines[prev_line_with_text].endswith(':'):
+        lines[prev_line_with_text] += ':'
     else:
-        lines[prev_line_with_text] += "::"
+        lines[prev_line_with_text] += '::'
 
 TO_RST_MAP = {RE_CODE_BLOCK: replace_code_block,
               RE_REFERENCE: r'``\2``',
@@ -60,17 +84,17 @@ def _process(lines):
                     line = lines[i]
             else:
                 lines[i] = regex.sub(alt, line)
-    return lines
+    return asciify(lines)
 
 
 def sphinx_to_rst(fh):
-    return "".join(_process(fh))
+    return ''.join(_process(fh))
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     global dirname
     dirname = os.path.dirname(sys.argv[1])
-    fh = open(sys.argv[1])
+    fh = codecs.open(sys.argv[1], encoding='utf-8')
     try:
         print(sphinx_to_rst(fh))
     finally:

pavement.py

@@ -1,4 +1,8 @@
+from __future__ import print_function
+
 import sys
+import traceback
+
 from paver.easy import task, sh, cmdopts, path, needs, options, Bunch
 from paver import doctools  # noqa
 from paver.setuputils import setup  # noqa
@@ -95,11 +99,25 @@ def clean_contributing(options):
     path('CONTRIBUTING.rst').unlink_p()
 
 
+@task
+def verify_readme(options):
+    with open('README.rst') as fp:
+        try:
+            fp.read().encode('ascii')
+        except Exception:
+            print('README contains non-ascii characters', file=sys.stderr)
+            print('Original exception below...', file=sys.stderr)
+            traceback.print_stack(file=sys.stderr)
+            sh('false')
+
+
 @task
 @needs('clean_readme')
 def readme(options):
     sh('{0} extra/release/sphinx-to-rst.py docs/templates/readme.txt \
             > README.rst'.format(sys.executable))
+    verify_readme()
+
 
 @task
 @needs('clean_contributing')
@@ -107,6 +125,7 @@ def contributing(options):
     sh('{0} extra/release/sphinx-to-rst.py docs/contributing.rst \
             > CONTRIBUTING.rst'.format(sys.executable))
 
+
 @task
 def bump(options):
     sh("extra/release/bump_version.py \
@@ -166,7 +185,7 @@ def gitcleanforce(options):
 
 @task
 @needs('flakes', 'autodoc', 'verifyindex',
-       'verifyconfigref', 'test', 'gitclean')
+       'verifyconfigref', 'verify_readme', 'test', 'gitclean')
 def releaseok(options):
     pass