diff --git a/bin/init-config.sh b/bin/init-config.sh index 26352936105..075ba003a79 100644 --- a/bin/init-config.sh +++ b/bin/init-config.sh @@ -21,87 +21,40 @@ INLONG_HOME=$( cd .. pwd ) -echo $INLONG_HOME source $INLONG_HOME/conf/inlong.conf init_inlong_agent() { echo "Init agent configuration parameters" cd $INLONG_HOME/inlong-agent/conf - sed -i 's/agent.local.ip=.*/'''agent.local.ip=${agent_local_ip}'''/g' agent.properties + sed -i 's/agent.local.ip=.*/'''agent.local.ip=${local_ip}'''/g' agent.properties + sed -i 's/agent.http.port=.*/'''agent.http.port=${agent_port}'''/g' agent.properties sed -i 's/agent.manager.vip.http.host=.*/'''agent.manager.vip.http.host=${manager_server_hostname}'''/g' agent.properties sed -i 's/agent.manager.vip.http.port=.*/'''agent.manager.vip.http.port=${manager_server_port}'''/g' agent.properties - sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:10081'''/g' agent.properties - sed -i 's/agent.prometheus.enable=.*/'''agent.prometheus.enable=${agent_prometheus_enable}'''/g' agent.properties - sed -i 's/agent.audit.enable=.*/'''agent.audit.enable=${agent_audit_enable}'''/g' agent.properties + sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:${audit_proxys_port}'''/g' agent.properties } init_inlong_audit() { - if [ $source_type == "pulsar" ]; then - echo "Init audit configuration parameters" - cd $INLONG_HOME/inlong-audit/conf + echo "Init audit configuration parameters" + cd $INLONG_HOME/inlong-audit/conf + sed -i 's#jdbc:mysql://.*apache_inlong_audit#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_audit'''#g' application.properties + sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application.properties + sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application.properties + if [ $mq_type == "pulsar" ]; then sed -i 's#pulsar://.*#'''${pulsar_service_url}'''#g' audit-proxy-pulsar.conf sed -i 's#pulsar://.*#'''${pulsar_service_url}'''#g' application.properties - sed -i 's#jdbc:mysql://.*apache_inlong_audit#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_audit'''#g' application.properties - sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application.properties - sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application.properties + fi + if [ $mq_type == "tubemq" ]; then + sed -i 's/agent1.sinks.tube-sink-msg1.master-host-port-list=.*/'''agent1.sinks.tube-sink-msg1.master-host-port-list=${tube_master_url}'''/g' audit-proxy-tube.conf + sed -i 's/agent1.sinks.tube-sink-msg2.master-host-port-list=.*/'''agent1.sinks.tube-sink-msg2.master-host-port-list=${tube_master_url}'''/g' audit-proxy-tube.conf + sed -i 's/audit.tube.masterlist=.*/'''audit.tube.masterlist=${tube_master_url}'''/g' application.properties fi } init_inlong_dataproxy() { echo "Init dataproxy configuration parameters" cd $INLONG_HOME/inlong-dataproxy/conf - if [ $source_type == "pulsar" ]; then - if [ -f "flume-mulit-pulsar-tcp-example.conf" ]; then - mv flume.conf flume-tubemq.conf - mv flume-mulit-pulsar-tcp-example.conf flume.conf - fi - sed -i 's#pulsar://.*#'''${pulsar_service_url}'''#g' flume.conf - else - if [ -f "flume-tubemq.conf" ]; then - mv flume.conf flume-mulit-pulsar-tcp-example.conf - mv flume-tubemq.conf flume.conf - fi - sed -i 's/master-host-port-list.*/'''master-host-port-list=${tubemaster_hostname}:${tubemaster_port}'''/g' flume.conf - fi - sed -i 's/manager_hosts=.*/'''manager_hosts=${manager_server_hostname}:${manager_server_port}'''/g' common.properties - sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:10081'''/g' common.properties -} - -init_inlong_tubemaster() { - echo "Init tubemq server configuration parameters" - cd $INLONG_HOME/inlong-tubemq-server/conf - echo "Init tubemq master configuration" - sed -i 's/hostName=.*/'''hostName=${tubemaster_hostname}'''/g' master.ini - sed -i 's/port=.*/'''port=${tubemaster_port}'''/g' master.ini - sed -i 's/webPort=.*/'''webPort=${tubemaster_webport}'''/g' master.ini - sed -i 's#metaDataPath=.*#'''metaDataPath=${metadata_path}'''#g' master.ini - sed -i 's/;metaDataPath/metaDataPath/g' master.ini - sed -i 's/confModAuthToken=.*/'''confModAuthToken=${confmod_authtoken}'''/g' master.ini - sed -i 's/zkServerAddr=.*/'''zkServerAddr=${zkserver_addr}'''/g' master.ini - sed -i 's/repHelperHost=.*/'''repHelperHost=${tubemaster_hostname}:9001'''/g' master.ini - sed -i 's/;repHelperHost/'''repHelperHost'''/g' master.ini -} - -init_inlong_tubebroker() { - echo "Init tubemq broker configuration" - cd $INLONG_HOME/inlong-tubemq-server/conf - sed -i 's/brokerId=.*/'''brokerId=${broker_id}'''/g' broker.ini - sed -i 's/hostName=.*/'''hostName=${tubebroker_hostname}'''/g' broker.ini - sed -i 's/port=.*/'''port=${tubebroker_port}'''/g' broker.ini - sed -i 's/webPort=.*/'''webPort=${tubebroker_webport}'''/g' broker.ini - sed -i 's/masterAddressList=.*/'''masterAddressList=${tubemaster_hostname}:${tubemaster_port}'''/g' broker.ini - sed -i 's#primaryPath=.*#'''primaryPath=${primary_path}'''#g' broker.ini - sed -i 's/zkServerAddr=.*/'''zkServerAddr=${zkserver_addr}'''/g' broker.ini -} - -init_inlong_tubemanager() { - echo "Init tubemq manager configuration" - cd $INLONG_HOME/inlong-tubemq-manager/conf - sed -i 's#jdbc:mysql://.*tubemanager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/tubemanager'''#g' application.properties - sed -i 's/spring.datasource.username=.*/'''spring.datasource.username=${spring_datasource_username}'''/g' application.properties - sed -i 's/spring.datasource.password=.*/'''spring.datasource.password=${spring_datasource_password}'''/g' application.properties - sed -i 's/#.*spring.datasource/spring.datasource/g' application.properties - sed -i 's/server.port=.*/'''server.port=${tube_manager_port}'''/g' application.properties + sed -i 's/manager.hosts=.*/'''manager.hosts=${manager_server_hostname}:${manager_server_port}'''/g' common.properties + sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:${audit_proxys_port}'''/g' common.properties } init_inlong_manager() { @@ -109,29 +62,22 @@ init_inlong_manager() { cd $INLONG_HOME/inlong-manager/conf sed -i 's/spring.profiles.active=.*/'''spring.profiles.active=${spring_profiles_active}'''/g' application.properties sed -i 's/server.port=.*/'''server.port=${manager_server_port}'''/g' application.properties - sed -i 's#jdbc:mysql://.*apache_inlong_manager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_manager'''#g' application-${spring_profiles_active}.properties - sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application-${spring_profiles_active}.properties - sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application-${spring_profiles_active}.properties - if [ $source_type == "tubemq" ]; then - sed -i 's#cluster.tube.manager=.*#'''cluster.tube.manager=http://${tube_manager_ip}:${tube_manager_port}'''#g' application-${spring_profiles_active}.properties - sed -i 's#cluster.tube.master=.*#'''cluster.tube.master=${tubemaster_hostname}:${tubemaster_port}'''#g' application-${spring_profiles_active}.properties - else - sed -i 's#pulsar.adminUrl=.*#'''pulsar.adminUrl=${pulsar_admin_url}'''#g' application-${spring_profiles_active}.properties - sed -i 's#pulsar.serviceUrl=.*#'''pulsar.serviceUrl=${pulsar_service_url}'''#g' application-${spring_profiles_active}.properties - sed -i 's/pulsar.defaultTenant=.*/'''pulsar.defaultTenant=${pulsar_default_tenant}'''/g' application-${spring_profiles_active}.properties + if [ $spring_profiles_active == "dev" ]; then + sed -i 's#jdbc:mysql://.*apache_inlong_manager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_manager'''#g' application-dev.properties + sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application-dev.properties + sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application-dev.properties + fi + if [ $spring_profiles_active == "prod" ]; then + sed -i 's#jdbc:mysql://.*apache_inlong_manager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_manager'''#g' application-prod.properties + sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application-prod.properties + sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application-prod.properties fi - sed -i 's/cluster.zk.url=.*/'''cluster.zk.url=${zkserver_addr}'''/g' application-${spring_profiles_active}.properties - sed -i 's/cluster.zk.root=.*/'''cluster.zk.root=${cluster_zk_root}'''/g' application-${spring_profiles_active}.properties - sed -i 's/sort.appName=.*/'''sort.appName=${sort_app_name}'''/g' application-${spring_profiles_active}.properties } if [ $# -eq 0 ]; then init_inlong_agent init_inlong_audit init_inlong_dataproxy - init_inlong_tubemaster - init_inlong_tubebroker - init_inlong_tubemanager init_inlong_manager else init_inlong_$1 diff --git a/bin/inlong-daemon b/bin/inlong-daemon index 92d7088b760..a67b0161ed0 100644 --- a/bin/inlong-daemon +++ b/bin/inlong-daemon @@ -22,25 +22,31 @@ INLONG_HOME=$( cd .. pwd ) -TUBEMQDBNAME="tubemanager" # tubemq manager database -INLONGDBNAME="apache_inlong_manager" # inlong manager Metabase name source $INLONG_HOME/conf/inlong.conf command_help() { echo "Usage: ./inlong-daemon (start|stop) where command is one of: audit Run a Audit Service - tubemaster Run a TubeMQ Master Service - tubebroker Run a TubeMQ Broker Service - tubemanager Run a TubeMQ Manager Service manager Run a Manager Service dashboard Run a Dashboard Service dataproxy Run a Dataproxy Service agent Run a Agent Service - sort Run a Sort Service standalone Run a Standalone mode(All Service)" } +# wait the service to start +wait_port_to_listen() { + service_name=$1 + service_port=$2 + result=$(lsof -Pi :${service_port} -sTCP:LISTEN) + while [[ -z "$result" ]]; do + echo "waiting $service_name to start, sleep 3s ..." + sleep 3 + result=$(lsof -Pi :${service_port} -sTCP:LISTEN) + done +} + # if less than two arguments supplied if [ $# -lt 2 ]; then command_help @@ -57,67 +63,6 @@ init_compo() { fi } -start_inlong_tubemaster() { - init_compo - echo "start-up tubemq master ..." - cd $INLONG_HOME/inlong-tubemq-server/bin - chmod 755 tubemq.sh - ./tubemq.sh master start -} - -start_inlong_tubebroker() { - init_compo - # Judge whether the TubeMqMaster is started in the current system - tubemq_master_thread=$($JAVA_HOME/bin/jps | grep MasterStartup) - if [[ ! -n "$tubemq_master_thread" ]]; then - echo "The system does not start tubemqMaster or starts abnormally. Please start tubemqMaster normally first" - exit 1 - fi - # add broker - curl -d 'type=op_modify&method=admin_add_broker_configure&brokerId='"$broker_id"'&brokerIp='"$tubebroker_hostname"'&createUser=docker&brokerPort='"$tubebroker_port"'&deletePolicy=delete,168h&numPartitions=3&unflushThreshold=1000&acceptPublish=true&acceptSubscribe=true&unflushInterval=10000&confModAuthToken='"$confmod_authtoken"'' http://$tubemaster_hostname:$tubemaster_webport/webapi.htm - # online - curl -d 'type=op_modify&method=admin_online_broker_configure&brokerId='"$broker_id"'&modifyUser=docker&confModAuthToken='"$confmod_authtoken"'' http://$tubemaster_hostname:$tubemaster_webport/webapi.htm - # broker start - echo "start-up tubemq broker ..." - cd $INLONG_HOME/inlong-tubemq-server/bin - chmod 755 tubemq.sh - ./tubemq.sh broker start -} - -start_inlong_tubemanager() { - init_compo - # Judge whether the TubeMqServer is started in the current system - tubemq_broker_thread=$($JAVA_HOME/bin/jps | grep BrokerStartup) - tubemq_master_thread=$($JAVA_HOME/bin/jps | grep MasterStartup) - if [[ ! -n "$tubemq_broker_thread" && ! -n "$tubemq_master_thread" ]]; then - echo "The system does not start tubemqServer or starts abnormally. Please start tubemqServer(master and broker) normally first" - exit 1 - fi - echo "start-up tubemq manager ..." - cd $INLONG_HOME/inlong-tubemq-manager/bin - # create tubemanager database - create_db_sql="create database IF NOT EXISTS ${TUBEMQDBNAME}" - mysql -h${spring_datasource_hostname} -P${spring_datasource_port} -u${spring_datasource_username} -p${spring_datasource_password} -e "${create_db_sql}" - ./start-manager.sh - sleep 10 - # first init - flag=$(cat init-tube-cluster.sh | grep "flag=1") - if [ ! -n "$flag" ]; then - echo "init shell config" - sed -i 's/TUBE_MANAGER_IP=.*/'''TUBE_MANAGER_IP=${tube_manager_ip}'''/g' init-tube-cluster.sh - sed -i 's/TUBE_MANAGER_PORT=.*/'''TUBE_MANAGER_PORT=${tube_manager_port}'''/g' init-tube-cluster.sh - sed -i 's/TUBE_MASTER_IP=.*/'''TUBE_MASTER_IP=${tubemaster_hostname}'''/g' init-tube-cluster.sh - sed -i 's/TUBE_MASTER_PORT=.*/'''TUBE_MASTER_PORT=${tubemaster_port}'''/g' init-tube-cluster.sh - sed -i 's/TUBE_MASTER_WEB_PORT=.*/'''TUBE_MASTER_WEB_PORT=${tubemaster_webport}'''/g' init-tube-cluster.sh - sed -i 's/TUBE_MASTER_TOKEN=.*/'''TUBE_MASTER_TOKEN=${confmod_authtoken}'''/g' init-tube-cluster.sh - echo "init tubemq cluster" - ./init-tube-cluster.sh - echo -e "\nflag=1" >>init-tube-cluster.sh - else - echo "tubemq cluster initialized,skip" - fi -} - start_inlong_audit() { init_compo echo "init apache_inlong_audit" @@ -127,138 +72,103 @@ start_inlong_audit() { inlong_num=$(echo $inlong_audit_count | tr -cd "[0-9]") if [ $inlong_num -eq 0 ]; then mysql -h${spring_datasource_hostname} -P${spring_datasource_port} -u${spring_datasource_username} -p${spring_datasource_password} ${LOG_FILE} 2>${LOG_DIR}/error.log & - -# Process ID -PID="$!" -STARTUP_LOG="${STARTUP_LOG}application pid: ${PID}\n" - -# The startup log is appended to the startup log file -echo -e ${STARTUP_LOG} >>${LOG_STARTUP_PATH} -# Print startup log -echo -e ${STARTUP_LOG} +nohup java ${JAVA_OPT} -Dlog4j2.formatMsgNoLookups=true -Dlog4j.formatMsgNoLookups=true -cp ${CLASSPATH} ${MAIN_CLASS} 1>${LOG_FILE} 2>${LOG_DIR}/error.log & \ No newline at end of file diff --git a/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh b/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh index c9ab444c3f7..b79593012cb 100644 --- a/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh +++ b/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh @@ -25,7 +25,7 @@ spring.mvc.pathmatch.matching-strategy = ANT_PATH_MATCHER topic.config.schedule=0/5 * * * * ? broker.reload.schedule=0/5 * * * * ? # mysql configuration for manager -spring.datasource.url=jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/tubemanager?useSSL=false +spring.datasource.url=jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/apache_inlong_tubemq?useSSL=false spring.datasource.username=$MYSQL_USER spring.datasource.password=$MYSQL_PASSWD # server port diff --git a/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql b/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql index 7a7cc2acb89..a7d4e47a15b 100644 --- a/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql +++ b/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql @@ -21,7 +21,7 @@ SET FOREIGN_KEY_CHECKS = 0; -- ---------------------------- -- database for TubeMQ Manager -- ---------------------------- -CREATE DATABASE IF NOT EXISTS tubemanager; +CREATE DATABASE IF NOT EXISTS apache_inlong_tubemq; -- ---------------------------- -- Table structure for broker diff --git a/inlong-tubemq/tubemq-manager/src/main/resources/application.properties b/inlong-tubemq/tubemq-manager/src/main/resources/application.properties index 841c97a09b7..f042c6ee186 100644 --- a/inlong-tubemq/tubemq-manager/src/main/resources/application.properties +++ b/inlong-tubemq/tubemq-manager/src/main/resources/application.properties @@ -21,7 +21,7 @@ spring.mvc.pathmatch.matching-strategy = ANT_PATH_MATCHER topic.config.schedule=0/5 * * * * ? broker.reload.schedule=0/5 * * * * ? # mysql configuration for manager -# spring.datasource.url=jdbc:mysql://mysql_ip:mysql_port/tubemanager?useSSL=false&allowPublicKeyRetrieval=true +# spring.datasource.url=jdbc:mysql://mysql_ip:mysql_port/apache_inlong_tubemq?useSSL=false&allowPublicKeyRetrieval=true # spring.datasource.username=mysql_username # spring.datasource.password=mysql_password # server port