cache.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913
  1. # -*- coding: utf-8 -*-
  2. """
  3. werkzeug.contrib.cache
  4. ~~~~~~~~~~~~~~~~~~~~~~
  5. The main problem with dynamic Web sites is, well, they're dynamic. Each
  6. time a user requests a page, the webserver executes a lot of code, queries
  7. the database, renders templates until the visitor gets the page he sees.
  8. This is a lot more expensive than just loading a file from the file system
  9. and sending it to the visitor.
  10. For most Web applications, this overhead isn't a big deal but once it
  11. becomes, you will be glad to have a cache system in place.
  12. How Caching Works
  13. =================
  14. Caching is pretty simple. Basically you have a cache object lurking around
  15. somewhere that is connected to a remote cache or the file system or
  16. something else. When the request comes in you check if the current page
  17. is already in the cache and if so, you're returning it from the cache.
  18. Otherwise you generate the page and put it into the cache. (Or a fragment
  19. of the page, you don't have to cache the full thing)
  20. Here is a simple example of how to cache a sidebar for 5 minutes::
  21. def get_sidebar(user):
  22. identifier = 'sidebar_for/user%d' % user.id
  23. value = cache.get(identifier)
  24. if value is not None:
  25. return value
  26. value = generate_sidebar_for(user=user)
  27. cache.set(identifier, value, timeout=60 * 5)
  28. return value
  29. Creating a Cache Object
  30. =======================
  31. To create a cache object you just import the cache system of your choice
  32. from the cache module and instantiate it. Then you can start working
  33. with that object:
  34. >>> from werkzeug.contrib.cache import SimpleCache
  35. >>> c = SimpleCache()
  36. >>> c.set("foo", "value")
  37. >>> c.get("foo")
  38. 'value'
  39. >>> c.get("missing") is None
  40. True
  41. Please keep in mind that you have to create the cache and put it somewhere
  42. you have access to it (either as a module global you can import or you just
  43. put it into your WSGI application).
  44. :copyright: (c) 2014 by the Werkzeug Team, see AUTHORS for more details.
  45. :license: BSD, see LICENSE for more details.
  46. """
  47. import os
  48. import re
  49. import errno
  50. import tempfile
  51. import platform
  52. from hashlib import md5
  53. from time import time
  54. try:
  55. import cPickle as pickle
  56. except ImportError: # pragma: no cover
  57. import pickle
  58. from werkzeug._compat import iteritems, string_types, text_type, \
  59. integer_types, to_native
  60. from werkzeug.posixemulation import rename
  61. def _items(mappingorseq):
  62. """Wrapper for efficient iteration over mappings represented by dicts
  63. or sequences::
  64. >>> for k, v in _items((i, i*i) for i in xrange(5)):
  65. ... assert k*k == v
  66. >>> for k, v in _items(dict((i, i*i) for i in xrange(5))):
  67. ... assert k*k == v
  68. """
  69. if hasattr(mappingorseq, 'items'):
  70. return iteritems(mappingorseq)
  71. return mappingorseq
  72. class BaseCache(object):
  73. """Baseclass for the cache systems. All the cache systems implement this
  74. API or a superset of it.
  75. :param default_timeout: the default timeout (in seconds) that is used if
  76. no timeout is specified on :meth:`set`. A timeout
  77. of 0 indicates that the cache never expires.
  78. """
  79. def __init__(self, default_timeout=300):
  80. self.default_timeout = default_timeout
  81. def _normalize_timeout(self, timeout):
  82. if timeout is None:
  83. timeout = self.default_timeout
  84. return timeout
  85. def get(self, key):
  86. """Look up key in the cache and return the value for it.
  87. :param key: the key to be looked up.
  88. :returns: The value if it exists and is readable, else ``None``.
  89. """
  90. return None
  91. def delete(self, key):
  92. """Delete `key` from the cache.
  93. :param key: the key to delete.
  94. :returns: Whether the key existed and has been deleted.
  95. :rtype: boolean
  96. """
  97. return True
  98. def get_many(self, *keys):
  99. """Returns a list of values for the given keys.
  100. For each key an item in the list is created::
  101. foo, bar = cache.get_many("foo", "bar")
  102. Has the same error handling as :meth:`get`.
  103. :param keys: The function accepts multiple keys as positional
  104. arguments.
  105. """
  106. return [self.get(k) for k in keys]
  107. def get_dict(self, *keys):
  108. """Like :meth:`get_many` but return a dict::
  109. d = cache.get_dict("foo", "bar")
  110. foo = d["foo"]
  111. bar = d["bar"]
  112. :param keys: The function accepts multiple keys as positional
  113. arguments.
  114. """
  115. return dict(zip(keys, self.get_many(*keys)))
  116. def set(self, key, value, timeout=None):
  117. """Add a new key/value to the cache (overwrites value, if key already
  118. exists in the cache).
  119. :param key: the key to set
  120. :param value: the value for the key
  121. :param timeout: the cache timeout for the key in seconds (if not
  122. specified, it uses the default timeout). A timeout of
  123. 0 idicates that the cache never expires.
  124. :returns: ``True`` if key has been updated, ``False`` for backend
  125. errors. Pickling errors, however, will raise a subclass of
  126. ``pickle.PickleError``.
  127. :rtype: boolean
  128. """
  129. return True
  130. def add(self, key, value, timeout=None):
  131. """Works like :meth:`set` but does not overwrite the values of already
  132. existing keys.
  133. :param key: the key to set
  134. :param value: the value for the key
  135. :param timeout: the cache timeout for the key in seconds (if not
  136. specified, it uses the default timeout). A timeout of
  137. 0 idicates that the cache never expires.
  138. :returns: Same as :meth:`set`, but also ``False`` for already
  139. existing keys.
  140. :rtype: boolean
  141. """
  142. return True
  143. def set_many(self, mapping, timeout=None):
  144. """Sets multiple keys and values from a mapping.
  145. :param mapping: a mapping with the keys/values to set.
  146. :param timeout: the cache timeout for the key in seconds (if not
  147. specified, it uses the default timeout). A timeout of
  148. 0 idicates that the cache never expires.
  149. :returns: Whether all given keys have been set.
  150. :rtype: boolean
  151. """
  152. rv = True
  153. for key, value in _items(mapping):
  154. if not self.set(key, value, timeout):
  155. rv = False
  156. return rv
  157. def delete_many(self, *keys):
  158. """Deletes multiple keys at once.
  159. :param keys: The function accepts multiple keys as positional
  160. arguments.
  161. :returns: Whether all given keys have been deleted.
  162. :rtype: boolean
  163. """
  164. return all(self.delete(key) for key in keys)
  165. def has(self, key):
  166. """Checks if a key exists in the cache without returning it. This is a
  167. cheap operation that bypasses loading the actual data on the backend.
  168. This method is optional and may not be implemented on all caches.
  169. :param key: the key to check
  170. """
  171. raise NotImplementedError(
  172. '%s doesn\'t have an efficient implementation of `has`. That '
  173. 'means it is impossible to check whether a key exists without '
  174. 'fully loading the key\'s data. Consider using `self.get` '
  175. 'explicitly if you don\'t care about performance.'
  176. )
  177. def clear(self):
  178. """Clears the cache. Keep in mind that not all caches support
  179. completely clearing the cache.
  180. :returns: Whether the cache has been cleared.
  181. :rtype: boolean
  182. """
  183. return True
  184. def inc(self, key, delta=1):
  185. """Increments the value of a key by `delta`. If the key does
  186. not yet exist it is initialized with `delta`.
  187. For supporting caches this is an atomic operation.
  188. :param key: the key to increment.
  189. :param delta: the delta to add.
  190. :returns: The new value or ``None`` for backend errors.
  191. """
  192. value = (self.get(key) or 0) + delta
  193. return value if self.set(key, value) else None
  194. def dec(self, key, delta=1):
  195. """Decrements the value of a key by `delta`. If the key does
  196. not yet exist it is initialized with `-delta`.
  197. For supporting caches this is an atomic operation.
  198. :param key: the key to increment.
  199. :param delta: the delta to subtract.
  200. :returns: The new value or `None` for backend errors.
  201. """
  202. value = (self.get(key) or 0) - delta
  203. return value if self.set(key, value) else None
  204. class NullCache(BaseCache):
  205. """A cache that doesn't cache. This can be useful for unit testing.
  206. :param default_timeout: a dummy parameter that is ignored but exists
  207. for API compatibility with other caches.
  208. """
  209. def has(self, key):
  210. return False
  211. class SimpleCache(BaseCache):
  212. """Simple memory cache for single process environments. This class exists
  213. mainly for the development server and is not 100% thread safe. It tries
  214. to use as many atomic operations as possible and no locks for simplicity
  215. but it could happen under heavy load that keys are added multiple times.
  216. :param threshold: the maximum number of items the cache stores before
  217. it starts deleting some.
  218. :param default_timeout: the default timeout that is used if no timeout is
  219. specified on :meth:`~BaseCache.set`. A timeout of
  220. 0 indicates that the cache never expires.
  221. """
  222. def __init__(self, threshold=500, default_timeout=300):
  223. BaseCache.__init__(self, default_timeout)
  224. self._cache = {}
  225. self.clear = self._cache.clear
  226. self._threshold = threshold
  227. def _prune(self):
  228. if len(self._cache) > self._threshold:
  229. now = time()
  230. toremove = []
  231. for idx, (key, (expires, _)) in enumerate(self._cache.items()):
  232. if (expires != 0 and expires <= now) or idx % 3 == 0:
  233. toremove.append(key)
  234. for key in toremove:
  235. self._cache.pop(key, None)
  236. def _normalize_timeout(self, timeout):
  237. timeout = BaseCache._normalize_timeout(self, timeout)
  238. if timeout > 0:
  239. timeout = time() + timeout
  240. return timeout
  241. def get(self, key):
  242. try:
  243. expires, value = self._cache[key]
  244. if expires == 0 or expires > time():
  245. return pickle.loads(value)
  246. except (KeyError, pickle.PickleError):
  247. return None
  248. def set(self, key, value, timeout=None):
  249. expires = self._normalize_timeout(timeout)
  250. self._prune()
  251. self._cache[key] = (expires, pickle.dumps(value,
  252. pickle.HIGHEST_PROTOCOL))
  253. return True
  254. def add(self, key, value, timeout=None):
  255. expires = self._normalize_timeout(timeout)
  256. self._prune()
  257. item = (expires, pickle.dumps(value,
  258. pickle.HIGHEST_PROTOCOL))
  259. if key in self._cache:
  260. return False
  261. self._cache.setdefault(key, item)
  262. return True
  263. def delete(self, key):
  264. return self._cache.pop(key, None) is not None
  265. def has(self, key):
  266. try:
  267. expires, value = self._cache[key]
  268. return expires == 0 or expires > time()
  269. except KeyError:
  270. return False
  271. _test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match
  272. class MemcachedCache(BaseCache):
  273. """A cache that uses memcached as backend.
  274. The first argument can either be an object that resembles the API of a
  275. :class:`memcache.Client` or a tuple/list of server addresses. In the
  276. event that a tuple/list is passed, Werkzeug tries to import the best
  277. available memcache library.
  278. This cache looks into the following packages/modules to find bindings for
  279. memcached:
  280. - ``pylibmc``
  281. - ``google.appengine.api.memcached``
  282. - ``memcached``
  283. - ``libmc``
  284. Implementation notes: This cache backend works around some limitations in
  285. memcached to simplify the interface. For example unicode keys are encoded
  286. to utf-8 on the fly. Methods such as :meth:`~BaseCache.get_dict` return
  287. the keys in the same format as passed. Furthermore all get methods
  288. silently ignore key errors to not cause problems when untrusted user data
  289. is passed to the get methods which is often the case in web applications.
  290. :param servers: a list or tuple of server addresses or alternatively
  291. a :class:`memcache.Client` or a compatible client.
  292. :param default_timeout: the default timeout that is used if no timeout is
  293. specified on :meth:`~BaseCache.set`. A timeout of
  294. 0 indicates that the cache never expires.
  295. :param key_prefix: a prefix that is added before all keys. This makes it
  296. possible to use the same memcached server for different
  297. applications. Keep in mind that
  298. :meth:`~BaseCache.clear` will also clear keys with a
  299. different prefix.
  300. """
  301. def __init__(self, servers=None, default_timeout=300, key_prefix=None):
  302. BaseCache.__init__(self, default_timeout)
  303. if servers is None or isinstance(servers, (list, tuple)):
  304. if servers is None:
  305. servers = ['127.0.0.1:11211']
  306. self._client = self.import_preferred_memcache_lib(servers)
  307. if self._client is None:
  308. raise RuntimeError('no memcache module found')
  309. else:
  310. # NOTE: servers is actually an already initialized memcache
  311. # client.
  312. self._client = servers
  313. self.key_prefix = to_native(key_prefix)
  314. def _normalize_key(self, key):
  315. key = to_native(key, 'utf-8')
  316. if self.key_prefix:
  317. key = self.key_prefix + key
  318. return key
  319. def _normalize_timeout(self, timeout):
  320. timeout = BaseCache._normalize_timeout(self, timeout)
  321. if timeout > 0:
  322. timeout = int(time()) + timeout
  323. return timeout
  324. def get(self, key):
  325. key = self._normalize_key(key)
  326. # memcached doesn't support keys longer than that. Because often
  327. # checks for so long keys can occur because it's tested from user
  328. # submitted data etc we fail silently for getting.
  329. if _test_memcached_key(key):
  330. return self._client.get(key)
  331. def get_dict(self, *keys):
  332. key_mapping = {}
  333. have_encoded_keys = False
  334. for key in keys:
  335. encoded_key = self._normalize_key(key)
  336. if not isinstance(key, str):
  337. have_encoded_keys = True
  338. if _test_memcached_key(key):
  339. key_mapping[encoded_key] = key
  340. _keys = list(key_mapping)
  341. d = rv = self._client.get_multi(_keys)
  342. if have_encoded_keys or self.key_prefix:
  343. rv = {}
  344. for key, value in iteritems(d):
  345. rv[key_mapping[key]] = value
  346. if len(rv) < len(keys):
  347. for key in keys:
  348. if key not in rv:
  349. rv[key] = None
  350. return rv
  351. def add(self, key, value, timeout=None):
  352. key = self._normalize_key(key)
  353. timeout = self._normalize_timeout(timeout)
  354. return self._client.add(key, value, timeout)
  355. def set(self, key, value, timeout=None):
  356. key = self._normalize_key(key)
  357. timeout = self._normalize_timeout(timeout)
  358. return self._client.set(key, value, timeout)
  359. def get_many(self, *keys):
  360. d = self.get_dict(*keys)
  361. return [d[key] for key in keys]
  362. def set_many(self, mapping, timeout=None):
  363. new_mapping = {}
  364. for key, value in _items(mapping):
  365. key = self._normalize_key(key)
  366. new_mapping[key] = value
  367. timeout = self._normalize_timeout(timeout)
  368. failed_keys = self._client.set_multi(new_mapping, timeout)
  369. return not failed_keys
  370. def delete(self, key):
  371. key = self._normalize_key(key)
  372. if _test_memcached_key(key):
  373. return self._client.delete(key)
  374. def delete_many(self, *keys):
  375. new_keys = []
  376. for key in keys:
  377. key = self._normalize_key(key)
  378. if _test_memcached_key(key):
  379. new_keys.append(key)
  380. return self._client.delete_multi(new_keys)
  381. def has(self, key):
  382. key = self._normalize_key(key)
  383. if _test_memcached_key(key):
  384. return self._client.append(key, '')
  385. return False
  386. def clear(self):
  387. return self._client.flush_all()
  388. def inc(self, key, delta=1):
  389. key = self._normalize_key(key)
  390. return self._client.incr(key, delta)
  391. def dec(self, key, delta=1):
  392. key = self._normalize_key(key)
  393. return self._client.decr(key, delta)
  394. def import_preferred_memcache_lib(self, servers):
  395. """Returns an initialized memcache client. Used by the constructor."""
  396. try:
  397. import pylibmc
  398. except ImportError:
  399. pass
  400. else:
  401. return pylibmc.Client(servers)
  402. try:
  403. from google.appengine.api import memcache
  404. except ImportError:
  405. pass
  406. else:
  407. return memcache.Client()
  408. try:
  409. import memcache
  410. except ImportError:
  411. pass
  412. else:
  413. return memcache.Client(servers)
  414. try:
  415. import libmc
  416. except ImportError:
  417. pass
  418. else:
  419. return libmc.Client(servers)
  420. # backwards compatibility
  421. GAEMemcachedCache = MemcachedCache
  422. class RedisCache(BaseCache):
  423. """Uses the Redis key-value store as a cache backend.
  424. The first argument can be either a string denoting address of the Redis
  425. server or an object resembling an instance of a redis.Redis class.
  426. Note: Python Redis API already takes care of encoding unicode strings on
  427. the fly.
  428. .. versionadded:: 0.7
  429. .. versionadded:: 0.8
  430. `key_prefix` was added.
  431. .. versionchanged:: 0.8
  432. This cache backend now properly serializes objects.
  433. .. versionchanged:: 0.8.3
  434. This cache backend now supports password authentication.
  435. .. versionchanged:: 0.10
  436. ``**kwargs`` is now passed to the redis object.
  437. :param host: address of the Redis server or an object which API is
  438. compatible with the official Python Redis client (redis-py).
  439. :param port: port number on which Redis server listens for connections.
  440. :param password: password authentication for the Redis server.
  441. :param db: db (zero-based numeric index) on Redis Server to connect.
  442. :param default_timeout: the default timeout that is used if no timeout is
  443. specified on :meth:`~BaseCache.set`. A timeout of
  444. 0 indicates that the cache never expires.
  445. :param key_prefix: A prefix that should be added to all keys.
  446. Any additional keyword arguments will be passed to ``redis.Redis``.
  447. """
  448. def __init__(self, host='localhost', port=6379, password=None,
  449. db=0, default_timeout=300, key_prefix=None, **kwargs):
  450. BaseCache.__init__(self, default_timeout)
  451. if host is None:
  452. raise ValueError('RedisCache host parameter may not be None')
  453. if isinstance(host, string_types):
  454. try:
  455. import redis
  456. except ImportError:
  457. raise RuntimeError('no redis module found')
  458. if kwargs.get('decode_responses', None):
  459. raise ValueError('decode_responses is not supported by '
  460. 'RedisCache.')
  461. self._client = redis.Redis(host=host, port=port, password=password,
  462. db=db, **kwargs)
  463. else:
  464. self._client = host
  465. self.key_prefix = key_prefix or ''
  466. def _normalize_timeout(self, timeout):
  467. timeout = BaseCache._normalize_timeout(self, timeout)
  468. if timeout == 0:
  469. timeout = -1
  470. return timeout
  471. def dump_object(self, value):
  472. """Dumps an object into a string for redis. By default it serializes
  473. integers as regular string and pickle dumps everything else.
  474. """
  475. t = type(value)
  476. if t in integer_types:
  477. return str(value).encode('ascii')
  478. return b'!' + pickle.dumps(value)
  479. def load_object(self, value):
  480. """The reversal of :meth:`dump_object`. This might be called with
  481. None.
  482. """
  483. if value is None:
  484. return None
  485. if value.startswith(b'!'):
  486. try:
  487. return pickle.loads(value[1:])
  488. except pickle.PickleError:
  489. return None
  490. try:
  491. return int(value)
  492. except ValueError:
  493. # before 0.8 we did not have serialization. Still support that.
  494. return value
  495. def get(self, key):
  496. return self.load_object(self._client.get(self.key_prefix + key))
  497. def get_many(self, *keys):
  498. if self.key_prefix:
  499. keys = [self.key_prefix + key for key in keys]
  500. return [self.load_object(x) for x in self._client.mget(keys)]
  501. def set(self, key, value, timeout=None):
  502. timeout = self._normalize_timeout(timeout)
  503. dump = self.dump_object(value)
  504. if timeout == -1:
  505. result = self._client.set(name=self.key_prefix + key,
  506. value=dump)
  507. else:
  508. result = self._client.setex(name=self.key_prefix + key,
  509. value=dump, time=timeout)
  510. return result
  511. def add(self, key, value, timeout=None):
  512. timeout = self._normalize_timeout(timeout)
  513. dump = self.dump_object(value)
  514. return (
  515. self._client.setnx(name=self.key_prefix + key, value=dump) and
  516. self._client.expire(name=self.key_prefix + key, time=timeout)
  517. )
  518. def set_many(self, mapping, timeout=None):
  519. timeout = self._normalize_timeout(timeout)
  520. # Use transaction=False to batch without calling redis MULTI
  521. # which is not supported by twemproxy
  522. pipe = self._client.pipeline(transaction=False)
  523. for key, value in _items(mapping):
  524. dump = self.dump_object(value)
  525. if timeout == -1:
  526. pipe.set(name=self.key_prefix + key, value=dump)
  527. else:
  528. pipe.setex(name=self.key_prefix + key, value=dump,
  529. time=timeout)
  530. return pipe.execute()
  531. def delete(self, key):
  532. return self._client.delete(self.key_prefix + key)
  533. def delete_many(self, *keys):
  534. if not keys:
  535. return
  536. if self.key_prefix:
  537. keys = [self.key_prefix + key for key in keys]
  538. return self._client.delete(*keys)
  539. def has(self, key):
  540. return self._client.exists(self.key_prefix + key)
  541. def clear(self):
  542. status = False
  543. if self.key_prefix:
  544. keys = self._client.keys(self.key_prefix + '*')
  545. if keys:
  546. status = self._client.delete(*keys)
  547. else:
  548. status = self._client.flushdb()
  549. return status
  550. def inc(self, key, delta=1):
  551. return self._client.incr(name=self.key_prefix + key, amount=delta)
  552. def dec(self, key, delta=1):
  553. return self._client.decr(name=self.key_prefix + key, amount=delta)
  554. class FileSystemCache(BaseCache):
  555. """A cache that stores the items on the file system. This cache depends
  556. on being the only user of the `cache_dir`. Make absolutely sure that
  557. nobody but this cache stores files there or otherwise the cache will
  558. randomly delete files therein.
  559. :param cache_dir: the directory where cache files are stored.
  560. :param threshold: the maximum number of items the cache stores before
  561. it starts deleting some. A threshold value of 0
  562. indicates no threshold.
  563. :param default_timeout: the default timeout that is used if no timeout is
  564. specified on :meth:`~BaseCache.set`. A timeout of
  565. 0 indicates that the cache never expires.
  566. :param mode: the file mode wanted for the cache files, default 0600
  567. """
  568. #: used for temporary files by the FileSystemCache
  569. _fs_transaction_suffix = '.__wz_cache'
  570. #: keep amount of files in a cache element
  571. _fs_count_file = '__wz_cache_count'
  572. def __init__(self, cache_dir, threshold=500, default_timeout=300,
  573. mode=0o600):
  574. BaseCache.__init__(self, default_timeout)
  575. self._path = cache_dir
  576. self._threshold = threshold
  577. self._mode = mode
  578. try:
  579. os.makedirs(self._path)
  580. except OSError as ex:
  581. if ex.errno != errno.EEXIST:
  582. raise
  583. self._update_count(value=len(self._list_dir()))
  584. @property
  585. def _file_count(self):
  586. return self.get(self._fs_count_file) or 0
  587. def _update_count(self, delta=None, value=None):
  588. # If we have no threshold, don't count files
  589. if self._threshold == 0:
  590. return
  591. if delta:
  592. new_count = self._file_count + delta
  593. else:
  594. new_count = value or 0
  595. self.set(self._fs_count_file, new_count, mgmt_element=True)
  596. def _normalize_timeout(self, timeout):
  597. timeout = BaseCache._normalize_timeout(self, timeout)
  598. if timeout != 0:
  599. timeout = time() + timeout
  600. return int(timeout)
  601. def _list_dir(self):
  602. """return a list of (fully qualified) cache filenames
  603. """
  604. mgmt_files = [self._get_filename(name).split('/')[-1]
  605. for name in (self._fs_count_file,)]
  606. return [os.path.join(self._path, fn) for fn in os.listdir(self._path)
  607. if not fn.endswith(self._fs_transaction_suffix)
  608. and fn not in mgmt_files]
  609. def _prune(self):
  610. if self._threshold == 0 or not self._file_count > self._threshold:
  611. return
  612. entries = self._list_dir()
  613. now = time()
  614. for idx, fname in enumerate(entries):
  615. try:
  616. remove = False
  617. with open(fname, 'rb') as f:
  618. expires = pickle.load(f)
  619. remove = (expires != 0 and expires <= now) or idx % 3 == 0
  620. if remove:
  621. os.remove(fname)
  622. except (IOError, OSError):
  623. pass
  624. self._update_count(value=len(self._list_dir()))
  625. def clear(self):
  626. for fname in self._list_dir():
  627. try:
  628. os.remove(fname)
  629. except (IOError, OSError):
  630. self._update_count(value=len(self._list_dir()))
  631. return False
  632. self._update_count(value=0)
  633. return True
  634. def _get_filename(self, key):
  635. if isinstance(key, text_type):
  636. key = key.encode('utf-8') # XXX unicode review
  637. hash = md5(key).hexdigest()
  638. return os.path.join(self._path, hash)
  639. def get(self, key):
  640. filename = self._get_filename(key)
  641. try:
  642. with open(filename, 'rb') as f:
  643. pickle_time = pickle.load(f)
  644. if pickle_time == 0 or pickle_time >= time():
  645. return pickle.load(f)
  646. else:
  647. os.remove(filename)
  648. return None
  649. except (IOError, OSError, pickle.PickleError):
  650. return None
  651. def add(self, key, value, timeout=None):
  652. filename = self._get_filename(key)
  653. if not os.path.exists(filename):
  654. return self.set(key, value, timeout)
  655. return False
  656. def set(self, key, value, timeout=None, mgmt_element=False):
  657. # Management elements have no timeout
  658. if mgmt_element:
  659. timeout = 0
  660. # Don't prune on management element update, to avoid loop
  661. else:
  662. self._prune()
  663. timeout = self._normalize_timeout(timeout)
  664. filename = self._get_filename(key)
  665. try:
  666. fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
  667. dir=self._path)
  668. with os.fdopen(fd, 'wb') as f:
  669. pickle.dump(timeout, f, 1)
  670. pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
  671. rename(tmp, filename)
  672. os.chmod(filename, self._mode)
  673. except (IOError, OSError):
  674. return False
  675. else:
  676. # Management elements should not count towards threshold
  677. if not mgmt_element:
  678. self._update_count(delta=1)
  679. return True
  680. def delete(self, key, mgmt_element=False):
  681. try:
  682. os.remove(self._get_filename(key))
  683. except (IOError, OSError):
  684. return False
  685. else:
  686. # Management elements should not count towards threshold
  687. if not mgmt_element:
  688. self._update_count(delta=-1)
  689. return True
  690. def has(self, key):
  691. filename = self._get_filename(key)
  692. try:
  693. with open(filename, 'rb') as f:
  694. pickle_time = pickle.load(f)
  695. if pickle_time == 0 or pickle_time >= time():
  696. return True
  697. else:
  698. os.remove(filename)
  699. return False
  700. except (IOError, OSError, pickle.PickleError):
  701. return False
  702. class UWSGICache(BaseCache):
  703. """ Implements the cache using uWSGI's caching framework.
  704. .. note::
  705. This class cannot be used when running under PyPy, because the uWSGI
  706. API implementation for PyPy is lacking the needed functionality.
  707. :param default_timeout: The default timeout in seconds.
  708. :param cache: The name of the caching instance to connect to, for
  709. example: mycache@localhost:3031, defaults to an empty string, which
  710. means uWSGI will cache in the local instance. If the cache is in the
  711. same instance as the werkzeug app, you only have to provide the name of
  712. the cache.
  713. """
  714. def __init__(self, default_timeout=300, cache=''):
  715. BaseCache.__init__(self, default_timeout)
  716. if platform.python_implementation() == 'PyPy':
  717. raise RuntimeError("uWSGI caching does not work under PyPy, see "
  718. "the docs for more details.")
  719. try:
  720. import uwsgi
  721. self._uwsgi = uwsgi
  722. except ImportError:
  723. raise RuntimeError("uWSGI could not be imported, are you "
  724. "running under uWSGI?")
  725. self.cache = cache
  726. def get(self, key):
  727. rv = self._uwsgi.cache_get(key, self.cache)
  728. if rv is None:
  729. return
  730. return pickle.loads(rv)
  731. def delete(self, key):
  732. return self._uwsgi.cache_del(key, self.cache)
  733. def set(self, key, value, timeout=None):
  734. return self._uwsgi.cache_update(key, pickle.dumps(value),
  735. self._normalize_timeout(timeout),
  736. self.cache)
  737. def add(self, key, value, timeout=None):
  738. return self._uwsgi.cache_set(key, pickle.dumps(value),
  739. self._normalize_timeout(timeout),
  740. self.cache)
  741. def clear(self):
  742. return self._uwsgi.cache_clear(self.cache)
  743. def has(self, key):
  744. return self._uwsgi.cache_exists(key, self.cache) is not None