Merge branch 'leobantech/master'

celery/app/defaults.py

@@ -91,6 +91,7 @@ NAMESPACES = {
         "REDIS_DB": Option(None, type="int"),
         "REDIS_PASSWORD": Option(None, type="string"),
         "RESULT_BACKEND": Option(None, type="string"),
+        "RESULT_DB_SHORT_LIVED_SESSIONS": Option(False, type="bool"),
         "RESULT_DBURI": Option(),
         "RESULT_ENGINE_OPTIONS": Option(None, type="dict"),
         "RESULT_EXCHANGE": Option("celeryresults"),

celery/backends/database.py

@@ -29,13 +29,17 @@ class DatabaseBackend(BaseDictBackend):
         self.dburi = dburi or self.app.conf.CELERY_RESULT_DBURI
         self.engine_options = dict(engine_options or {},
                         **self.app.conf.CELERY_RESULT_ENGINE_OPTIONS or {})
+        self.short_lived_sessions = self.app.conf.CELERY_RESULT_DB_SHORT_LIVED_SESSIONS
         if not self.dburi:
             raise ImproperlyConfigured(
                     "Missing connection string! Do you have "
                     "CELERY_RESULT_DBURI set to a real value?")
 
     def ResultSession(self):
-        return ResultSession(dburi=self.dburi, **self.engine_options)
+        return ResultSession(
+                    dburi=self.dburi, 
+                    short_lived_sessions=self.short_lived_sessions, 
+                    **self.engine_options)
 
     def _store_result(self, task_id, result, status, traceback=None):
         """Store return value and status of an executed task."""

celery/db/session.py

@@ -8,7 +8,7 @@ ResultModelBase = declarative_base()
 
 _SETUP = defaultdict(lambda: False)
 _ENGINES = {}
-_MAKERS = {}
+_SESSIONS = {}
 
 
 def get_engine(dburi, **kwargs):
@@ -17,11 +17,11 @@ def get_engine(dburi, **kwargs):
     return _ENGINES[dburi]
 
 
-def create_session(dburi, **kwargs):
+def create_session(dburi, short_lived_sessions=False, **kwargs):
     engine = get_engine(dburi, **kwargs)
-    if dburi not in _MAKERS:
-        _MAKERS[dburi] = sessionmaker(bind=engine)
-    return engine, _MAKERS[dburi]
+    if short_lived_sessions or dburi not in _SESSIONS:
+        _SESSIONS[dburi] = sessionmaker(bind=engine)
+    return engine, _SESSIONS[dburi]
 
 
 def setup_results(engine):

docs/configuration.rst

@@ -166,6 +166,17 @@ the :setting:`CELERY_RESULT_ENGINE_OPTIONS` setting::
     # echo enables verbose logging from SQLAlchemy.
     CELERY_RESULT_ENGINE_OPTIONS = {"echo": True}
 
+
+.. setting:: CELERY_RESULT_DB_SHORT_LIVED_SESSIONS
+    CELERY_RESULT_DB_SHORT_LIVED_SESSIONS = True
+
+Short lived sessions are disabled by default.  If enabled they can drastically reduce 
+performance, especially on systems processing lots of tasks.  This option is useful 
+on low-traffic workers that experience errors as a result of cached database connections 
+going stale through inactivity.  For example, intermittent errors like 
+`(OperationalError) (2006, 'MySQL server has gone away')` can be fixed by enabling
+short lived sessions.  This option only affects the database backend.
+
 .. _`Supported Databases`:
     http://www.sqlalchemy.org/docs/core/engines.html#supported-databases