Redash 是一款开源的 BI 工具,提供了基于 web 的数据库查询和数据可视化功能。
Mac安装redash依赖
Redash依赖redis和postgres作为缓存和元数据存储,所以在部署之前先要安装Redis和PostgreSQL
mac安装redis
brew install redis
brew services start redis //启动redis
brew services stop redis
mac安装postgreSQL
brew install postgresql
brew services start postgresq
initdb /usr/local/var/postgre //初始化数据库
brew services restart postgresql //重启数据库
安装
安装python依赖包
pip install -r requirements.txt -r requirements_dev.txt
安装nodejs的包
npm install //安装node的包
npm run build //进行一次完整build
配置
在开发环境中,使用默认的配置能满足绝大多数的需求,如果要覆盖默认配置.需要在redash的根目录下设置一个.env文件,在该文件中设置环境变量,覆盖默认的.
查看所有变量
bin/run ./manage.py check_settings
JSON_AS_ASCII = True
AUTH_TYPE = api_key
MAIL_DEFAULT_SENDER = None
RATELIMIT_STRATEGY = fixed-window
FEATURE_AUTO_PUBLISH_NAMED_QUERIES = True
HSTS_MAX_AGE = 31556926
ENFORCE_HTTPS_PERMANENT = False
SESSION_REFRESH_EACH_REQUEST = True
CELERY_BROKER = redis://localhost:6379/0
REDIS_URL = redis://localhost:6379/0
TIME_FORMAT = HH:mm
SQLALCHEMY_COMMIT_ON_TEARDOWN = False
ALLOW_PARAMETERS_IN_EMBEDS = False
JOB_EXPIRY_TIME = 43200
SQLALCHEMY_TRACK_MODIFICATIONS = False
KYLIN_OFFSET = 0
LDAP_SEARCH_TEMPLATE = (cn=%(username)s)
QLOUD_AUTHORIZE_URL = https://yourauthn.com/oauth2.0/authorize
COOKIES_SECURE = False
MAIL_MAX_EMAILS = None
REMEMBER_COOKIE_SECURE = False
SENTRY_DSN =
VERSION_CHECK = True
STATSD_HOST = 127.0.0.1
QUERY_RESULTS_CLEANUP_ENABLED = True
REMEMBER_COOKIE_HTTPONLY = True
MULTI_ORG = False
LOG_FORMAT = [%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s
RATELIMIT_SWALLOW_ERRORS = False
QLOUD_ACCESS_TOKEN_URL = https://yourauthn.com/oauth2.0/accessToken
FRAME_OPTIONS_ALLOW_FROM =
RATELIMIT_HEADER_LIMIT = X-RateLimit-Limit
JSONIFY_MIMETYPE = application/json
PROXIES_COUNT = 1
QLOUD_OAUTH_ENABLED = True
ACCESS_CONTROL_REQUEST_METHOD = GET, POST, PUT
ACCESS_CONTROL_ALLOW_ORIGIN = set([])
EVENT_REPORTING_WEBHOOKS = []
RATELIMIT_HEADERS_ENABLED = False
RATELIMIT_HEADER_RESET = X-RateLimit-Reset
ENFORCE_FILE_SAVE = True
LDAP_LOGIN_ENABLED = False
HSTS_ENABLED = False
LIMITER_STORAGE = redis://localhost:6379/0
STATSD_USE_TAGS = False
DATE_FORMAT = DD/MM/YY
DASHBOARD_REFRESH_INTERVALS = [60, 300, 600, 1800, 3600, 43200, 86400]
FEATURE_POLICY =
CELERY_RESULT_EXPIRES = 14400
DATASOURCE_SECRET_KEY = c292a0a3aa32397cdb050e233733900f
FEATURE_SHOW_QUERY_RESULTS_COUNT = True
SESSION_COOKIE_DOMAIN = None
SESSION_COOKIE_NAME = session
REMOTE_USER_LOGIN_ENABLED = False
CONTENT_SECURITY_POLICY_REPORT_ONLY = False
HOST =
LOGGER_HANDLER_POLICY = always
SQLALCHEMY_RECORD_QUERIES = None
QLOUD_CLIENT_ID = qloud10070901oauth21
ALERTS_DEFAULT_MAIL_SUBJECT_TEMPLATE = ({state}) {alert_name}
MAIL_USE_SSL = False
QLOUD_AUTH_STOMP_PORT = 61613
GOOGLE_CLIENT_ID =
SQLALCHEMY_NATIVE_UNICODE = None
MAX_CONTENT_LENGTH = None
LDAP_HOST_URL = None
LDAP_SEARCH_DN = None
QUERY_RESULTS_CLEANUP_MAX_AGE = 7
PERMANENT_SESSION_LIFETIME = 31 days, 0:00:00
PAGE_SIZE = 20
QLOUD_CLIENT_SECRET = OUO10070901JN32rewffFE3
SQLALCHEMY_MAX_OVERFLOW = None
QLOUD_BASE_URL = https://yourauthn.com
LDAP_SSL = False
TRAP_HTTP_EXCEPTIONS = False
FEATURE_ALLOW_CUSTOM_JS_VISUALIZATIONS = False
REMOTE_USER_HEADER = X-Forwarded-Remote-User
LDAP_BIND_DN_PASSWORD =
PRESERVE_CONTEXT_ON_EXCEPTION = None
BIGQUERY_HTTP_TIMEOUT = 600
ENFORCE_HTTPS = False
SQLALCHEMY_ECHO = False
STATIC_ASSETS_PATH = /Users/xixuebin/PycharmProjects/huaruan/qloud-redash/redash/settings/../../client/dist/
CONTENT_SECURITY_POLICY_REPORT_URI =
SESSION_COOKIE_PATH = None
QLOUD_AUTH_STOMP_ADDRESS = qloudstomp.service.sd
ACCESS_CONTROL_ALLOW_CREDENTIALS = False
SQLPARSE_FORMAT_OPTIONS = {'keyword_case': 'upper', 'reindent': True}
GOOGLE_CLIENT_SECRET =
ACCESS_CONTROL_ALLOW_HEADERS = Content-Type
LDAP_AUTH_METHOD = SIMPLE
LDAP_DISPLAY_NAME_KEY = displayName
HSTS_PRELOAD = False
LOGGER_NAME = redash.app
KYLIN_LIMIT = 50000
LDAP_BIND_DN = None
QUERY_RESULTS_CLEANUP_COUNT = 100
SECRET_KEY = c292a0a3aa32397cdb050e233733900f
CONTENT_SECURITY_POLICY = default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; font-src 'self' data:; img-src 'self' http: https: data:; object-src 'none'; frame-ancestors 'none'; frame-src redash.io;
QLOUD_AUTH_DOMAIN_CHECK = True
STATSD_PREFIX = redash
LOG_LEVEL = INFO
APPLICATION_ROOT = None
SERVER_NAME = None
LDAP_CUSTOM_USERNAME_PROMPT = LDAP/AD/SSO username:
PREFERRED_URL_SCHEME = http
TESTING = False
TEMPLATES_AUTO_RELOAD = None
MAIL_USERNAME = None
SCHEMAS_REFRESH_SCHEDULE = 30
CELERYD_WORKER_LOG_FORMAT = [%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s
ALLOW_SCRIPTS_IN_USER_INPUT = False
FRAME_OPTIONS = deny
SQLALCHEMY_POOL_TIMEOUT = None
MAIL_ASCII_ATTACHMENTS = False
MAIL_SERVER = localhost
QLOUD_AUTH_DOMAIN = ['redash.com']
QLOUD_PROFILE_URL = https://yourauthn.com/oauth2.0/profile?access_token=
RATELIMIT_HEADER_REMAINING = X-RateLimit-Remaining
RATELIMIT_HEADER_RETRY_AFTER = Retry-After
LDAP_EMAIL_KEY = mail
USE_X_SENDFILE = False
GOOGLE_OAUTH_ENABLED = False
SQLALCHEMY_DATABASE_URI = postgresql:///postgres
SQLALCHEMY_DISABLE_POOL = False
MAIL_USE_TLS = False
SESSION_COOKIE_SECURE = False
SCHEMA_RUN_TABLE_SIZE_CALCULATIONS = False
SQLALCHEMY_BINDS = None
PAGE_SIZE_OPTIONS = [5, 10, 20, 50, 100]
LOG_STDOUT = False
TABLE_CELL_MAX_JSON_SIZE = 50000
QLOUD_USERS_URL = http://qmsauthz.pditdop.service.sd/users
KYLIN_ACCEPT_PARTIAL = False
DEBUG = False
FEATURE_DISABLE_REFRESH_QUERIES = False
QUERY_REFRESH_INTERVALS = [60, 300, 600, 900, 1800, 3600, 7200, 10800, 14400, 18000, 21600, 25200, 28800, 32400, 36000, 39600, 43200, 86400, 604800, 1209600, 2592000]
EXPLAIN_TEMPLATE_LOADING = False
THROTTLE_LOGIN_PATTERN = 50/hour
CELERY_RESULT_BACKEND = redis://localhost:6379/0
DESTINATIONS = ['redash.destinations.email', 'redash.destinations.slack', 'redash.destinations.webhook', 'redash.destinations.hipchat', 'redash.destinations.mattermost', 'redash.destinations.chatwork', 'redash.destinations.pagerduty', 'redash.destinations.hangoutschat']
SCHEMAS_REFRESH_QUEUE = celery
SQLALCHEMY_POOL_RECYCLE = None
CELERYD_WORKER_TASK_LOG_FORMAT = [%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s task_id=%(task_id)s %(message)s
LOG_PREFIX =
RATELIMIT_ENABLED = True
MAIL_PASSWORD = None
QLOUD_LOGOUT_URL = https://yourauthn.com/logout?service=
JSONIFY_PRETTYPRINT_REGULAR = True
SQLALCHEMY_POOL_SIZE = None
PROPAGATE_EXCEPTIONS = None
TRAP_BAD_REQUEST_ERRORS = False
STATSD_PORT = 8125
JSON_SORT_KEYS = True
SESSION_COOKIE_HTTPONLY = True
SEND_FILE_MAX_AGE_DEFAULT = 12:00:00
MAIL_PORT = 25
QUERY_RUNNERS = ['redash.query_runner.athena', 'redash.query_runner.big_query', 'redash.query_runner.google_spreadsheets', 'redash.query_runner.graphite', 'redash.query_runner.mongodb', 'redash.query_runner.couchbase', 'redash.query_runner.mysql', 'redash.query_runner.pg', 'redash.query_runner.url', 'redash.query_runner.influx_db', 'redash.query_runner.elasticsearch', 'redash.query_runner.amazon_elasticsearch', 'redash.query_runner.presto', 'redash.query_runner.databricks', 'redash.query_runner.hive_ds', 'redash.query_runner.impala_ds', 'redash.query_runner.vertica', 'redash.query_runner.clickhouse', 'redash.query_runner.yandex_metrica', 'redash.query_runner.rockset', 'redash.query_runner.treasuredata', 'redash.query_runner.sqlite', 'redash.query_runner.dynamodb_sql', 'redash.query_runner.mssql', 'redash.query_runner.memsql_ds', 'redash.query_runner.mapd', 'redash.query_runner.jql', 'redash.query_runner.google_analytics', 'redash.query_runner.axibase_tsd', 'redash.query_runner.salesforce', 'redash.query_runner.query_results', 'redash.query_runner.prometheus', 'redash.query_runner.qubole', 'redash.query_runner.db2', 'redash.query_runner.druid', 'redash.query_runner.kylin', 'redash.query_runner.drill', 'redash.query_runner.uptycs', 'redash.query_runner.snowflake', 'redash.query_runner.phoenix']
REFERRER_POLICY = strict-origin-when-cross-origin
INVITATION_TOKEN_MAX_AGE = 604800
CONTENT_SECURITY_POLICY_NONCE_IN = []
HSTS_INCLUDE_SUBDOMAINS = False
创建数据库
bin/run ./manage.py database create_tables
在创建数据库的时候,如果有异常,请检查数据库的配置.配置类似:export SQLALCHEMY_DATABASE_URI=postgresql://postgres@XX.XX.XX.XX:15432/postgre
服务启动
Redash主要的进程有2个
- web server
- Celery worker(s) & Scheduler
在开发环境中,我们也可以运行Webpack's dev server or watch utiliy, 命令如下:
- Web server:
bin/run ./manage.py runserver --debugger --reload
- Celery:
./bin/run celery worker --app=redash.worker --beat --Qscheduled_queries,queries,celery -c2
- Webpack dev server:
npm run start
如果一切服务运行正常,可访问页面http://127.0.0.1:5000
与Auth集成
Redash本身支持smal协议,也可以使用GOOGLE账号进行登录,使用google账号登录的时候,后跳使用的就是Oauth协议.我们这里介绍使用OAuth2与自己的认证集成的过程.
创建.env文件
创建.env文件,填写需要使用的参数,例如:
export REDASH_YOUR_CLIENT_ID=qloud10070901oauth21
export REDASH_YOUR_CLIENT_SECRET=OUO10070901JN32rewffFE3
export REDASH_YOUR_ACCESS_TOKEN_URL=https://yourauthn.com/oauth2.0/accessToken
export REDASH_YOUR_AUTHORIZE_URL=https://yourauthn.com/oauth2.0/authorize
export REDASH_YOUR_BASE_URL=https://yourauthn.com
export REDASH_YOUR_PROFILE_URL=https://yourauthn.com/oauth2.0/profile?access_token=
export REDASH_YOUR_LOGOUT_URL=https://yourauthn.com/logout?service=
export REDASH_YOUR_USERS_URL=http://qmsauthz.pditdop.service.sd/users
export REDASH_YOUR_AUTH_DOMAIN=redash.com
export REDASH_YOUR_AUTH_DOMAIN_CHECK=True
export REDASH_YOUR_AUTH_STOMP_ADDRESS=qloudstomp.service.sd
export REDASH_YOUR_AUTH_STOMP_PORT=61613
获取设置的环境变量值
QLOUD_CLIENT_ID = os.environ.get("REDASH_YOUR_CLIENT_ID", "")
QLOUD_CLIENT_SECRET = os.environ.get("REDASH_YOUR_CLIENT_SECRET", "")
QLOUD_ACCESS_TOKEN_URL = os.environ.get("REDASH_YOUR_ACCESS_TOKEN_URL", "")
QLOUD_AUTHORIZE_URL = os.environ.get("REDASH_YOUR_AUTHORIZE_URL", "")
QLOUD_BASE_URL = os.environ.get("REDASH_YOUR_BASE_URL", "")
QLOUD_PROFILE_URL = os.environ.get("REDASH_YOUR_PROFILE_URL", "")
QLOUD_LOGOUT_URL = os.environ.get("REDASH_YOUR_LOGOUT_URL", "")
QLOUD_USERS_URL = os.environ.get("REDASH_YOUR_USERS_URL", "")
QLOUD_AUTH_DOMAIN = os.environ.get("REDASH_YOUR_AUTH_DOMAIN", "").split(",")
QLOUD_AUTH_DOMAIN_CHECK = ("TRUE" == str(os.environ.get("REDASH_YOUR_AUTH_DOMAIN_CHECK")))
QLOUD_AUTH_STOMP_ADDRESS = os.environ.get("REDASH_YOUR_AUTH_STOMP_ADDRESS", "")
QLOUD_AUTH_STOMP_PORT = int(os.environ.get("REDASH_YOUR_AUTH_STOMP_PORT", 61613))
QLOUD_OAUTH_ENABLED = bool(QLOUD_CLIENT_ID and QLOUD_CLIENT_SECRET)
添加认证代码
redash的认证相关的代码都是在redash.authentication目录下,我们创建一个自己的认证文件your_oauth.py
import logging
import ssl
import requests
from flask import redirect, url_for, Blueprint, flash, request, session
from flask_oauthlib.client import OAuth
from redash import models, settings
from redash.authentication import create_and_login_user, logout_and_redirect_to_index, get_next_path
from redash.authentication.org_resolving import current_org
logger = logging.getLogger('qloud_oauth')
oauth = OAuth()
blueprint = Blueprint('qloud_oauth', __name__)
ssl._create_default_https_context = ssl._create_unverified_context
def qloud_remote_app():
if 'qloud' not in oauth.remote_apps:
oauth.remote_app('qloud',
base_url=settings.QLOUD_BASE_URL,
authorize_url=settings.QLOUD_AUTHORIZE_URL,
request_token_url=None,
request_token_params={
'scope': '%s %s %s' %
(settings.QLOUD_AUTHORIZE_URL, settings.QLOUD_ACCESS_TOKEN_URL,
settings.QLOUD_PROFILE_URL),
},
access_token_url=settings.QLOUD_ACCESS_TOKEN_URL,
access_token_params={
'client_id': settings.QLOUD_CLIENT_ID
},
access_token_method='GET',
consumer_key=settings.QLOUD_CLIENT_ID,
consumer_secret=settings.QLOUD_CLIENT_SECRET)
return oauth.qloud
def get_user_profile(access_token):
# headers = {'Authorization': 'OAuth {}'.format(access_token)}
logger.info('access token is %s', access_token)
# logger.info('header is %s', headers)
logger.info("requst url is %s " % (settings.QLOUD_PROFILE_URL + access_token))
response = requests.get(settings.QLOUD_PROFILE_URL + access_token, headers={}, verify=False)
if response.status_code == 401:
logger.warning("Failed getting user profile (response code 401).")
return None
return response.json()['attributes']
def verify_profile(org, profile):
if org.is_public:
return True
if not settings.QLOUD_AUTH_DOMAIN_CHECK:
logger.info("all user will sync")
return True
email = profile['email']
domain = email.split('@')[-1]
logger.info("domain is %s" % domain)
if settings.QLOUD_AUTH_DOMAIN_CHECK:
if domain in settings.QLOUD_AUTH_DOMAIN:
return True
return False
@blueprint.route('/<org_slug>/oauth/qloud', endpoint="authorize_org")
def org_login(org_slug):
session['org_slug'] = current_org.slug
return redirect(url_for(".authorize", next=request.args.get('next', None)))
@blueprint.route('/oauth/qloud', endpoint="authorize")
def login():
callback = url_for('.callback', _external=True)
next_path = request.args.get('next', url_for("redash.index", org_slug=session.get('org_slug')))
logger.info("Callback url: %s", callback)
logger.info("Next is: %s", next_path)
return qloud_remote_app().authorize(callback=callback, state=next_path)
@blueprint.route('/oauth/qloud_callback', endpoint="callback")
def authorized():
resp = qloud_remote_app().authorized_response()
access_token = resp['access_token']
if access_token is None:
logger.warning("Access token missing in call back request.")
flash("Validation error. Please retry.")
return redirect(url_for('redash.login'))
profile = get_user_profile(access_token)
if profile is None or profile['id'] is None:
flash("Validation error. Please retry.")
return redirect(url_for('redash.login'))
logger.info("user profile is %s" % profile)
# id mapping to name
user_name = profile['id']
if 'org_slug' in session:
org = models.Organization.get_by_slug(session.pop('org_slug'))
else:
org = current_org
if not verify_profile(org, profile):
logger.warning("User tried to login with unauthorized domain name: %s (org: %s)", profile['email'], org)
flash("Your Qloud Apps account ({}) isn't allowed.".format(profile['email']))
# check failed, please logout
return redirect(url_for('redash.logout', org_slug=org.slug))
if profile['photo'] is not None:
picture_url = "%s?sz=40" % profile['photo']
else:
picture_url = None
user = create_and_login_user(org, user_name, profile['email'], picture_url)
if user is None:
return logout_and_redirect_to_index()
unsafe_next_path = request.args.get('state') or url_for("redash.index", org_slug=org.slug)
next_path = get_next_path(unsafe_next_path)
return redirect(next_path)
修改login,登录跳转到自己的登录页面
def get_qloud_auth_url(next_path):
if settings.MULTI_ORG:
qloud_auth_url = url_for('qloud_oauth.authorize_org', next=next_path, org_slug=current_org.slug)
else:
qloud_auth_url = url_for('qloud_oauth.authorize', next=next_path)
return qloud_auth_url
# 修改方法def login(org_slug=None),添加如下代码
qloud_auth_url = get_qloud_auth_url(next_path)
if settings.QLOUD_OAUTH_ENABLED:
logger.info("redirect to qloud auth %s" % qloud_auth_url)
return redirect(qloud_auth_url)
修改登录函数,实现sso登出
@routes.route(org_scoped_rule('/logout'))
def logout(org_slug=None):
if settings.QLOUD_OAUTH_ENABLED:
logger.info("redirect to qloud auth")
logger.info("logout url %s" % (settings.QLOUD_LOGOUT_URL+get_login_url(
next=(request.host_url + '/qloud-auth/logout'))))
logout_user()
logger.info(settings.QLOUD_LOGOUT_URL + request.host_url + 'qloud-auth/logout')
return redirect(settings.QLOUD_LOGOUT_URL + request.host_url + 'qloud-auth/logout')
logout_user()
return redirect(get_login_url(next=None))
redash默认的用户登录信息是在客户端的cookie中存储的,无法实现从sso登出之后,实现redash登录,如果需要实现的话,应该修改redsh的session存储 在SSO登出的时候,通知redash在Server端删除session.等下次客户端访问的时候,进行cookie的删除.具体做法如下:
#在app.py中,导入flask_session
from flask_session import Session
#在create_app方法中添加如下代码
# set session in redis
app.config['SECRET_KEY'] = settings.SECRET_KEY
app.config['SESSION_TYPE'] = 'redis'
app.config['SESSION_KEY_PREFIX'] = 'redash-session:'
app.config['SESSION_REDIS'] = redis_connection
Session(app)
在SSO回调的方法中,删除用户session
#该方法为sso的回调方法
@routes.route(org_scoped_rule('/qloud-auth/logout'), methods=['GET', 'POST'])
def qloud_auth_logout(org_slug=None):
if request.method == 'POST':
logger.info("begin to qloud-auth logout with post")
request_data = request.form['logoutRequest']
logger.info("request data is %s" % request_data)
try:
tree = et.fromstring(request_data)
user_name = tree[0].text
logger.info("user_name %s" % user_name)
user = models.User.get_by_name_and_org(user_name, current_org)
logger.info("user_id %s" % user.id)
__delete_server_session(user.id)
except models.NoResultFound:
logger.error("do not found user with request body %s " % request_data)
return None
else:
logger.info("begin to qloud-auth logout with get")
logout_user()
return redirect(get_login_url(next=None))
def __delete_server_session(user_id):
keys = redis_connection.keys('redash-session:*')
for key in keys:
value = redis_connection.get(key)
logger.info("session value is %s" % value)
if ("V%s-" % user_id) in value:
redis_connection.delete(key)
return True
return False
在loader_user的时候,进行session查询和判断,没有找到session,通知客户端,删除cookie
def __check_server_session(user_id_with_identity):
keys = redis_connection.keys('redash-session:*')
for key in keys:
value = redis_connection.get(key)
logger.info("session value is %s" % value)
if user_id_with_identity in value:
return True
return False
@login_manager.user_loader
def load_user(user_id_with_identity):
org = current_org._get_current_object()
'''
Users who logged in prior to https://github.com/getredash/redash/pull/3174 going live are going
to have their (integer) user_id as their session user identifier.
These session user identifiers will be updated the first time they visit any page so we add special
logic to allow a frictionless transition.
This logic will be removed 2-4 weeks after going live, and users who haven't
visited any page during that time will simply have to log in again.
'''
logger.debug("session is %s " % session)
logger.debug("is_legacy_session_identifier is %s" % user_id_with_identity)
is_legacy_session_identifier = str(user_id_with_identity).find('-') < 0
if is_legacy_session_identifier:
user_id = user_id_with_identity
else:
user_id, _ = user_id_with_identity.split("-")
try:
user = models.User.get_by_id_and_org(user_id, org)
if user.is_disabled:
return None
if is_legacy_session_identifier:
login_user(user, remember=True)
elif user.get_id() != user_id_with_identity:
return None
elif not __check_server_session(user_id_with_identity):
# if server session dose not exist,note client to delete cookie
if 'user_id' in session:
session.pop('user_id')
if '_fresh' in session:
session.pop('_fresh')
cookie_name = current_app.config.get('REMEMBER_COOKIE_NAME', "remember_token")
if cookie_name in request.cookies:
session['remember'] = 'clear'
user_logged_out.send(current_app._get_current_object(), user=user)
return None
return user
except models.NoResultFound:
return None