|
|
@@ -1,35 +1,41 @@
|
|
|
# -* coding: utf-8 -*-
|
|
|
"""
|
|
|
celery.backends.elasticsearch
|
|
|
- ~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
|
|
Elasticsearch result store backend.
|
|
|
- Based on CouchDB backend.
|
|
|
|
|
|
"""
|
|
|
-from __future__ import absolute_import
|
|
|
+from __future__ import absolute_import, unicode_literals
|
|
|
|
|
|
-try:
|
|
|
- import elasticsearch
|
|
|
-except ImportError:
|
|
|
- elasticsearch = None # noqa
|
|
|
-
|
|
|
-from .base import KeyValueStoreBackend
|
|
|
-
|
|
|
-import datetime
|
|
|
+from datetime import datetime
|
|
|
|
|
|
from kombu.utils.url import _parse_url
|
|
|
|
|
|
from celery.exceptions import ImproperlyConfigured
|
|
|
|
|
|
+from .base import KeyValueStoreBackend
|
|
|
+
|
|
|
+try:
|
|
|
+ import elasticsearch
|
|
|
+except ImportError:
|
|
|
+ elasticsearch = None # noqa
|
|
|
+
|
|
|
__all__ = ['ElasticsearchBackend']
|
|
|
|
|
|
-ERR_LIB_MISSING = """\
|
|
|
+E_LIB_MISSING = """\
|
|
|
You need to install the elasticsearch library to use the Elasticsearch \
|
|
|
-result backend\
|
|
|
+result backend.\
|
|
|
"""
|
|
|
|
|
|
+
|
|
|
class ElasticsearchBackend(KeyValueStoreBackend):
|
|
|
+ """Elasticsearch Backend.
|
|
|
+
|
|
|
+ :raises celery.exceptions.ImproperlyConfigured: if
|
|
|
+ module :mod:`elasticsearch` is not available.
|
|
|
+
|
|
|
+ """
|
|
|
|
|
|
index = 'celery'
|
|
|
doc_type = 'backend'
|
|
|
@@ -37,84 +43,79 @@ class ElasticsearchBackend(KeyValueStoreBackend):
|
|
|
host = 'localhost'
|
|
|
port = 9200
|
|
|
|
|
|
-
|
|
|
def __init__(self, url=None, *args, **kwargs):
|
|
|
- """Initialize Elasticsearch backend instance.
|
|
|
-
|
|
|
- :raises celery.exceptions.ImproperlyConfigured: if
|
|
|
- module :mod:`elasticsearch` is not available.
|
|
|
-
|
|
|
- """
|
|
|
super(ElasticsearchBackend, self).__init__(*args, **kwargs)
|
|
|
|
|
|
if elasticsearch is None:
|
|
|
- raise ImproperlyConfigured(ERR_LIB_MISSING)
|
|
|
-
|
|
|
- uindex = udoc_type = uscheme = uhost = uport = None
|
|
|
-
|
|
|
- if url:
|
|
|
- uscheme, uhost, uport, _, _, uuri, _ = _parse_url(url) # noqa
|
|
|
- uuri = uuri.strip('/') if uuri else None
|
|
|
- uuris = uuri.split("/")
|
|
|
- uindex = uuris[0] if len(uuris) > 0 else None
|
|
|
- udoc_type = uuris[1] if len(uuris) > 1 else None
|
|
|
-
|
|
|
- self.index = uindex or self.index
|
|
|
- self.doc_type = udoc_type or self.doc_type
|
|
|
- self.scheme = uscheme or self.scheme
|
|
|
- self.host = uhost or self.host
|
|
|
- self.port = uport or self.port
|
|
|
-
|
|
|
- self._server = None
|
|
|
+ raise ImproperlyConfigured(E_LIB_MISSING)
|
|
|
|
|
|
+ index = doc_type = scheme = host = port = None
|
|
|
|
|
|
- def _get_server(self):
|
|
|
- """Connect to the Elasticsearch server."""
|
|
|
- return elasticsearch.Elasticsearch(self.host)
|
|
|
-
|
|
|
+ if url:
|
|
|
+ scheme, host, port, _, _, path, _ = _parse_url(url) # noqa
|
|
|
+ if path:
|
|
|
+ path = path.strip('/')
|
|
|
+ index, _, doc_type = path.partition('/')
|
|
|
|
|
|
- @property
|
|
|
- def server(self):
|
|
|
- if self._server is None:
|
|
|
- self._server = self._get_server()
|
|
|
- return self._server
|
|
|
+ self.index = index or self.index
|
|
|
+ self.doc_type = doc_type or self.doc_type
|
|
|
+ self.scheme = scheme or self.scheme
|
|
|
+ self.host = host or self.host
|
|
|
+ self.port = port or self.port
|
|
|
|
|
|
+ self._server = None
|
|
|
|
|
|
def get(self, key):
|
|
|
try:
|
|
|
- out = self.server.get(index=self.index,\
|
|
|
- doc_type=self.doc_type,\
|
|
|
- id=key)
|
|
|
- if isinstance(out, dict) \
|
|
|
- and "found" in out and out["found"] \
|
|
|
- and "_source" in out and key in out["_source"]:
|
|
|
- return out["_source"][key]
|
|
|
- else:
|
|
|
- return None
|
|
|
+ res = self.server.get(
|
|
|
+ index=self.index,
|
|
|
+ doc_type=self.doc_type,
|
|
|
+ id=key,
|
|
|
+ )
|
|
|
+ try:
|
|
|
+ if res['found']:
|
|
|
+ return res['_source'][key]
|
|
|
+ except (TypeError, KeyError):
|
|
|
+ pass
|
|
|
except elasticsearch.exceptions.NotFoundError:
|
|
|
- return None
|
|
|
-
|
|
|
+ pass
|
|
|
|
|
|
def set(self, key, value):
|
|
|
try:
|
|
|
- data = {}
|
|
|
- data['@timestamp'] = "{0}Z".format(datetime.datetime.utcnow()\
|
|
|
- .strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
|
|
|
- data[key] = value
|
|
|
- self.server.index(index=self.index, doc_type=self.doc_type,\
|
|
|
- id=key, body=data)
|
|
|
+ self._index(
|
|
|
+ id=key,
|
|
|
+ body={
|
|
|
+ key: value,
|
|
|
+ '@timestamp': '{0}Z'.format(
|
|
|
+ datetime.utcnow().isoformat()[:-3]
|
|
|
+ ),
|
|
|
+ },
|
|
|
+ )
|
|
|
except elasticsearch.exceptions.ConflictError:
|
|
|
# document already exists, update it
|
|
|
data = self.get(key)
|
|
|
data[key] = value
|
|
|
- self.server.index(index=self.index, doc_type=self.doc_type,\
|
|
|
- id=key, body=data, refresh=True)
|
|
|
+ self._index(key, data, refresh=True)
|
|
|
|
|
|
+ def _index(self, id, body, **kwargs):
|
|
|
+ return self.server.index(
|
|
|
+ index=self.index,
|
|
|
+ doc_type=self.doc_type,
|
|
|
+ **kwargs
|
|
|
+ )
|
|
|
|
|
|
def mget(self, keys):
|
|
|
return [self.get(key) for key in keys]
|
|
|
|
|
|
-
|
|
|
def delete(self, key):
|
|
|
self.server.delete(index=self.index, doc_type=self.doc_type, id=key)
|
|
|
|
|
|
+ def _get_server(self):
|
|
|
+ """Connect to the Elasticsearch server."""
|
|
|
+ return elasticsearch.Elasticsearch(self.host)
|
|
|
+
|
|
|
+ @property
|
|
|
+ def server(self):
|
|
|
+ if self._server is None:
|
|
|
+ self._server = self._get_server()
|
|
|
+ return self._server
|
Ask Solem