import logging
import os
import sys
from functools import wraps
import pickle
from dynaconf import settings
from common.redis_client import redis_client as redis
from common.utils import get_object_size
from termcolor import cprint
from cProfile import Profile
from pstats import Stats
from line_profiler import LineProfiler
REDIS_PREFIX = os.environ.get('REDIS_PREFIX', 'xdata_offline')
USE_REDIS_CACHE = os.environ.get("USE_REDIS_CACHE", 'true')
DEFAULT_EXPIRE_SECOND = int(os.environ.get("DEFAULT_EXPIRE_SECOND", 600))
logging.debug(f'--decorator--REDIS_PREFIX:{REDIS_PREFIX}, USE_REDIS_CACHE:{USE_REDIS_CACHE}')
def cache(ex: int = DEFAULT_EXPIRE_SECOND):
"""
Args:
ex: Defaults to 300 seconds; Never expire when less than or equal to 0
"""
def redis_cache(func):
tmp_redis_key = f"{REDIS_PREFIX}@webserver@{func.__module__}_{func.__name__}"
def generate_redis_key(*args, **kwargs):
# not contains defaults kwargs
return f"{tmp_redis_key}_{str(args)}_{str(kwargs)}"
key_set = f"{tmp_redis_key}_set"
@wraps(func)
def wrap(*args, **kwargs):
if USE_REDIS_CACHE != 'true':
return func(*args, **kwargs)
key = generate_redis_key(*args, **kwargs)
res = redis.get(key)
if res:
logging.debug(f'success get value from redis key:{key} ttl:{redis.ttl(key)}s')
try:
return pickle.loads(res)
except Exception as e:
logging.exception('pickle load exceptin')
res = func(*args, **kwargs)
if settings.ENV_FOR_DYNACONF != "DEVELOPMENT":
pickle_res = pickle.dumps(res)
# TODO: add other type: list/hash/sordset
if ex <= 0:
redis.set(key, pickle_res)
else:
redis.set(key, pickle_res, ex=ex)
logging.debug(f'success set redis key:{key}, ex:{ex}, size:{get_object_size(pickle_res)}')
redis.sadd(key_set, key)
return res
def clear_cache():
redis_keys = redis.smembers(key_set)
if redis_keys:
logging.debug(f'need clear redis_keys:{redis_keys}')
redis.delete(*redis_keys)
redis.delete(key_set)
wrap.clear_cache = clear_cache
# wrap.generate_redis_key = generate_redis_key
# wrap.key_set = key_set
return wrap
return redis_cache
def code_stack_used_time_parse(func):
@wraps(func)
def wrap(*args, **kwargs):
pro = Profile()
cprint(f"start run:{func.__name__}".center(30, "="), "green")
res = pro.runcall(func, *args, **kwargs)
stats = Stats(pro)
stats.strip_dirs()
stats.sort_stats('cumulative')
stats.print_stats(10)
return res
return wrap
def line_used_time_parse(func):
@wraps(func)
def wrap(*args, **kwargs):
profile = LineProfiler(func)
profile.enable()
cprint(f"start run:{func.__name__}".center(30, "="), "green")
res = func(*args, **kwargs)
profile.disable()
profile.print_stats(sys.stdout, output_unit=1e-3)
return res
return wrap