Documents celery.utils.debug

celery/utils/debug.py

@@ -1,41 +1,42 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
 import os
 
 from .compat import format_d
 
+try:
+    from psutil import Process
+except ImportError:
+    Process = None  # noqa
+
 _process = None
 _mem_sample = []
 
-def ps():
-    global _process
-    if _process is None:
-        try:
-            from psutil import Process
-        except ImportError:
-            return None
-        _process = Process(os.getpid())
-    return _process
 
-def mem_rss():
-    p = ps()
-    if p is not None:
-        return "%sMB" % (format_d(p.get_memory_info().rss // 1024), )
+def sample_mem():
+    """Sample RSS memory usage.
 
-def sample(x, n=10, k=0):
-    j = len(x) // n
-    for _ in xrange(n):
-        yield x[k]
-        k += j
+    Statistics can then be output by calling :func:`memdump`.
+
+    """
+    _mem_sample.append(mem_rss())
+
+
+def memdump(samples=10):
+    """Dump memory statistics.
 
+    Will print a sample of all RSS memory samples added by
+    calling :func:`sample_mem`, and in addition print
+    used RSS memory after :func:`gc.collect`.
 
-def memdump():
+    """
     if ps() is None:
         print("- rss: (psutil not installed).")
         return
     if filter(None, _mem_sample):
         print("- rss (sample):")
-        for mem in sample(_mem_sample):
+        for mem in sample(_mem_sample, samples):
             print("-    > %s," % mem)
         _mem_sample[:] = []
     import gc
@@ -43,5 +44,32 @@ def memdump():
     print("- rss (end): %s." % (mem_rss()))
 
 
-def sample_mem():
-    _mem_sample.append(mem_rss())
+def sample(x, n, k=0):
+    """Given a list `x` a sample of length ``n`` of that list is returned.
+
+    E.g. if `n` is 10, and `x` has 100 items, a list of every 10th
+    item is returned.
+
+    ``k`` can be used as offset.
+
+    """
+    j = len(x) // n
+    for _ in xrange(n):
+        yield x[k]
+        k += j
+
+
+def mem_rss():
+    """Returns RSS memory usage as a humanized string."""
+    p = ps()
+    if p is not None:
+        return "%sMB" % (format_d(p.get_memory_info().rss // 1024), )
+
+
+def ps():
+    """Returns the global :class:`psutil.Process` instance,
+    or :const:`None` if :mod:`psutil` is not installed."""
+    global _process
+    if _process is None and Process is not None:
+        _process = Process(os.getpid())
+    return _process

docs/reference/celery.utils.debug.rst

@@ -0,0 +1,48 @@
+====================================
+ celery.utils.debug
+====================================
+
+.. contents::
+    :local:
+
+Sampling Memory Usage
+=====================
+
+This module can be used to diagnose and sample the memory usage
+used by parts of your application.
+
+E.g to sample the memory usage of applying tasks you can do this:
+
+.. code-block:: python
+
+
+    from celery.utils.debug import sample_mem, memdump
+
+    from tasks import add
+
+
+    try:
+        for i in range(100):
+            for j in range(100):
+                add.delay(i, j)
+            sample_mem()
+    finally:
+        memdump()
+
+
+API Reference
+=============
+
+.. currentmodule:: celery.utils.debug
+
+.. automodule:: celery.utils.debug
+
+    .. autofunction:: sample_mem
+
+    .. autofunction:: memdump
+
+    .. autofunction:: sample
+
+    .. autofunction:: mem_rss
+
+    .. autofunction:: ps

docs/reference/index.rst

@@ -26,6 +26,7 @@
     celery.schedules
     celery.signals
     celery.security
+    celery.utils.debug
     celery.utils.mail
     celery.exceptions
     celery.loaders