nprasad2077/doris
1
0 Fork 0
mirror of https://github.com/apache/doris.git synced 2026-11-04 07:31:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

[chore](fe) Remove the standalone auditloader plugin from fe_plugins (#68563)

Browse Source 
### What problem does this PR solve?

Issue Number: None

Related PR: None

Problem Summary:

`fe_plugins/auditloader` is the old standalone audit loader plugin,
which had to be built with `build-plugin.sh` and installed with `INSTALL
PLUGIN`. FE now ships a built-in audit loader
(`org.apache.doris.plugin.audit.AuditLoader`), registered by `PluginMgr`
at startup, which writes audit events into `__internal_schema.audit_log`
and is switched by the global variable `enable_audit_plugin`. Nothing in
FE or the build depends on the standalone copy any more, and it has
drifted from the built-in one. This PR removes the module and its entry
in `fe_plugins/pom.xml`.

The legacy `pytest/deploy` scripts had an optional path that installed
this plugin. When `output/audit_loader/auditloader.zip` existed, they
rendered `plugin_auditload.conf`, unpacked the zip into the FE
directory, created `doris_audit_db__.doris_audit_tbl__` and ran `INSTALL
PLUGIN`. No build produces that zip, so the path never ran. This PR
removes that path, `plugin_auditload.conf`, and the `deploy_audit`
parameter it threaded through `prepare_palo_package()` and
`start_palo()`.
This commit is contained in:
Calvin Kirs authored and GitHub committed 2026-09-29 09:58:12 +08:00
1 parent 225bcd3018
commit 83d300d279
13 files changed
+4 -935
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pytest/deploy/deploy.py
+2 -7
View File
@@ -21,7 +21,6 @@
This module provide Palo deployer.
Date: 2015/10/07 17:23:06
"""
import os
import init_env
import backup
import config_be
@@ -36,10 +35,6 @@ import time
def deploy_palo():
"""deploy palo
"""
if os.path.exists('./output/audit_loader/auditloader.zip'):
deploy_audit = True
else:
deploy_audit = False
# 集群停止，备份
backup.backup_palo()
# 初始化fe/be目录等
@@ -48,13 +43,13 @@ def deploy_palo():
# 更新conf文件
process_config_file.process_palo_conf()
# 准备部署表
prepare_package.prepare_palo_package(deploy_audit=deploy_audit)
prepare_package.prepare_palo_package()
# hadoop_mkdir.create_hdfs_dir()
# 分发部署包到fe/be机器
distribute.distribute_package()
time.sleep(100)
# config_be.config_be()
start.start_palo(init_state=True, deploy_audit=deploy_audit)
start.start_palo(init_state=True)
if __name__ == '__main__':
pytest/deploy/plugin_auditload.conf
-47
View File
@@ -1,47 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
### plugin configuration
# The max size of a batch, default is 50MB
max_batch_size=52428800
# The max interval of batch loaded, default is 60 seconds
max_batch_interval_sec=60
# the max stmt length to be loaded in audit table, default is 4096
max_stmt_length=4096
# the capacity of audit queue, default is 1000
max_queue_size=1000
# Doris FE host for loading the audit, default is 127.0.0.1:8030.
# this should be the host port for stream load
frontend_host_port=127.0.0.1:8030
# Database of the audit table
database=doris_audit_db__
# Audit table name, to save the audit data.
table=doris_audit_tbl__
# Doris user. This user must have LOAD_PRIV to the audit table.
user=root
# Doris user's password
password=
pytest/deploy/prepare_package.py
+1 -4
View File
@@ -25,7 +25,7 @@ import os
import env_config
def prepare_palo_package(deploy_audit=False):
def prepare_palo_package():
"""prepare palo package
"""
os.system('mv fe.conf.out output/fe/conf/fe.conf')
@@ -36,9 +36,6 @@ def prepare_palo_package(deploy_audit=False):
'ulimit -c unlimited\\n nohup/g" output/be/bin/start_be.sh')
os.system('sed -i "s/limit3 -c 0 -n/limit3 -n/g" output/be/bin/start_be.sh')
os.system('mkdir -p output/be/var/pull_load > /dev/null 2>&1')
if deploy_audit:
os.system('unzip -q -u -d output/fe/plugin_auditloader output/audit_loader/auditloader.zip')
os.system('mv plugin_auditload.conf.out output/fe/plugin_auditloader/plugin.conf')
os.system('cd output;tar -cf - fe | gzip --fast > fe.tar.gz; cd -')
os.system('cd output;tar -cf - be | gzip --fast > be.tar.gz; cd -')
os.system('mv output/fe.tar.gz .')
pytest/deploy/process_config_file.py
-9
View File
@@ -81,20 +81,11 @@ def process_be_conf():
pass
def process_auditload_conf():
""""""
os.system('cp plugin_auditload.conf plugin_auditload.conf.out')
os.system('sed -i "s/password=.*/password=%s/g" plugin_auditload.conf.out' % env_config.fe_password)
os.system('sed -i "s/frontend_host_port=.*/frontend_host_port=%s:%s/g" plugin_auditload.conf.out' \
% (env_config.master, env_config.fe_query_port - 1000))
def process_palo_conf():
"""process palo conf
"""
process_fe_conf()
process_be_conf()
process_auditload_conf()
if __name__ == '__main__':
pytest/deploy/start.py
+1 -52
View File
@@ -172,56 +172,7 @@ def add_brokers():
os.system(cmd)
def add_auditload_plugin():
"""add audit load plugin"""
sql = """
create table doris_audit_db__.doris_audit_tbl__ \
( \
query_id varchar(48) comment 'Unique query id', \
\`time\` datetime not null comment 'Query start time', \
client_ip varchar(32) comment 'Client IP', \
user varchar(64) comment 'User name', \
db varchar(96) comment 'Database of this query', \
state varchar(8) comment 'Query result state. EOF, ERR, OK', \
query_time bigint comment 'Query execution time in millisecond', \
scan_bytes bigint comment 'Total scan bytes of this query', \
scan_rows bigint comment 'Total scan rows of this query', \
return_rows bigint comment 'Returned rows of this query', \
stmt_id int comment 'An incremental id of statement', \
is_query tinyint comment 'Is this statemt a query. 1 or 0', \
frontend_ip varchar(32) comment 'Frontend ip of executing this statement', \
cpu_time_ms bigint comment 'Total scan cpu time in millisecond of this query', \
sql_hash varchar(48) comment 'Hash value for this query', \
sql_digest varchar(48) comment 'Sql digest of this query, will be empty if not a slow query', \
peak_memory_bytes bigint comment 'Peak memory bytes used on all backends of this query', \
stmt string comment 'The original statement, trimed if longer than 2G ' \
) engine=OLAP \
duplicate key(query_id, \`time\`, client_ip) \
partition by range(\`time\`) () \
distributed by hash(query_id) buckets 1 \
properties( \
'dynamic_partition.time_unit' = 'HOUR', \
'dynamic_partition.start' = '-48', \
'dynamic_partition.end' = '3', \
'dynamic_partition.prefix' = 'p', \
'dynamic_partition.buckets' = '1', \
'dynamic_partition.enable' = 'true', \
'replication_num' = '3' \
);
"""
cmd = 'mysql -h %s -P%s -uroot -p%s -e "%s"' % (env_config.master, env_config.fe_query_port,
env_config.fe_password, "create database doris_audit_db__")
os.system(cmd)
cmd = 'mysql -h %s -P%s -uroot -p%s -e "%s"' % (env_config.master, env_config.fe_query_port,
env_config.fe_password, sql)
os.system(cmd)
sql = "INSTALL PLUGIN FROM '%s/fe/plugin_auditloader'" % env_config.fe_path
cmd = 'mysql -h %s -P%s -uroot -p%s -e "%s"' % (env_config.master, env_config.fe_query_port,
env_config.fe_password, sql)
os.system(cmd)
def start_palo(init_state=False, deploy_audit=False):
def start_palo(init_state=False):
"""start palo
"""
start_master()
@@ -235,8 +186,6 @@ def start_palo(init_state=False, deploy_audit=False):
start_other_fe()
start_be()
time.sleep(5)
if deploy_audit:
add_auditload_plugin()
if __name__ == '__main__':