-
Notifications
You must be signed in to change notification settings - Fork 4.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[DSIP-25][Remote Logging] Split remote logging configuration (#15826)
Co-authored-by: Rick Cheng <[email protected]>
- Loading branch information
1 parent
fa6ea8b
commit d218b02
Showing
12 changed files
with
249 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
82 changes: 82 additions & 0 deletions
82
...common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.dolphinscheduler.common.config; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.util.Properties; | ||
| import java.util.Set; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; | ||
| import org.springframework.core.io.InputStreamResource; | ||
|
|
||
| @Slf4j | ||
| public class ImmutableYamlDelegate implements IPropertyDelegate { | ||
|
|
||
| private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml"; | ||
|
|
||
| private final Properties properties; | ||
|
|
||
| public ImmutableYamlDelegate() { | ||
| this(REMOTE_LOGGING_YAML_NAME); | ||
| } | ||
|
|
||
| public ImmutableYamlDelegate(String... yamlAbsolutePath) { | ||
| properties = new Properties(); | ||
| // read from classpath | ||
| for (String fileName : yamlAbsolutePath) { | ||
| try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) { | ||
| YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); | ||
| factory.setResources(new InputStreamResource(fis)); | ||
| factory.afterPropertiesSet(); | ||
| Properties subProperties = factory.getObject(); | ||
| properties.putAll(subProperties); | ||
| } catch (IOException e) { | ||
| log.error("Load property: {} error, please check if the file exist under classpath", | ||
| yamlAbsolutePath, e); | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
| printProperties(); | ||
| } | ||
|
|
||
| public ImmutableYamlDelegate(Properties properties) { | ||
| this.properties = properties; | ||
| } | ||
|
|
||
| @Override | ||
| public String get(String key) { | ||
| return properties.getProperty(key); | ||
| } | ||
|
|
||
| @Override | ||
| public String get(String key, String defaultValue) { | ||
| return properties.getProperty(key, defaultValue); | ||
| } | ||
|
|
||
| @Override | ||
| public Set<String> getPropertyKeys() { | ||
| return properties.stringPropertyNames(); | ||
| } | ||
|
|
||
| private void printProperties() { | ||
| properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v)); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
61 changes: 61 additions & 0 deletions
61
dolphinscheduler-common/src/main/resources/remote-logging.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| remote-logging: | ||
| # Whether to enable remote logging | ||
| enable: false | ||
| # if remote-logging.enable = true, set the target of remote logging | ||
| target: OSS | ||
| # if remote-logging.enable = true, set the log base directory | ||
| base.dir: logs | ||
| # if remote-logging.enable = true, set the number of threads to send logs to remote storage | ||
| thread.pool.size: 10 | ||
| # required if you set remote-logging.target=OSS | ||
| oss: | ||
| # oss access key id, required if you set remote-logging.target=OSS | ||
| access.key.id: <access.key.id> | ||
| # oss access key secret, required if you set remote-logging.target=OSS | ||
| access.key.secret: <access.key.secret> | ||
| # oss bucket name, required if you set remote-logging.target=OSS | ||
| bucket.name: <bucket.name> | ||
| # oss endpoint, required if you set remote-logging.target=OSS | ||
| endpoint: <endpoint> | ||
| # required if you set remote-logging.target=S3 | ||
| s3: | ||
| # s3 access key id, required if you set remote-logging.target=S3 | ||
| access.key.id: <access.key.id> | ||
| # s3 access key secret, required if you set remote-logging.target=S3 | ||
| access.key.secret: <access.key.secret> | ||
| # s3 bucket name, required if you set remote-logging.target=S3 | ||
| bucket.name: <bucket.name> | ||
| # s3 endpoint, required if you set remote-logging.target=S3 | ||
| endpoint: <endpoint> | ||
| # s3 region, required if you set remote-logging.target=S3 | ||
| region: <region> | ||
| google.cloud.storage: | ||
| # the location of the google cloud credential, required if you set remote-logging.target=GCS | ||
| credential: /path/to/credential | ||
| # gcs bucket name, required if you set remote-logging.target=GCS | ||
| bucket.name: <your-bucket> | ||
| abs: | ||
| # abs account name, required if you set resource.storage.type=ABS | ||
| account.name: <your-account-name> | ||
| # abs account key, required if you set resource.storage.type=ABS | ||
| account.key: <your-account-key> | ||
| # abs container name, required if you set resource.storage.type=ABS | ||
| container.name: <your-container-name> | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
61 changes: 61 additions & 0 deletions
61
dolphinscheduler-common/src/test/resources/remote-logging.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| remote-logging: | ||
| # Whether to enable remote logging | ||
| enable: false | ||
| # if remote-logging.enable = true, set the target of remote logging | ||
| target: OSS | ||
| # if remote-logging.enable = true, set the log base directory | ||
| base.dir: logs | ||
| # if remote-logging.enable = true, set the number of threads to send logs to remote storage | ||
| thread.pool.size: 10 | ||
| # required if you set remote-logging.target=OSS | ||
| oss: | ||
| # oss access key id, required if you set remote-logging.target=OSS | ||
| access.key.id: <access.key.id> | ||
| # oss access key secret, required if you set remote-logging.target=OSS | ||
| access.key.secret: <access.key.secret> | ||
| # oss bucket name, required if you set remote-logging.target=OSS | ||
| bucket.name: <bucket.name> | ||
| # oss endpoint, required if you set remote-logging.target=OSS | ||
| endpoint: <endpoint> | ||
| # required if you set remote-logging.target=S3 | ||
| s3: | ||
| # s3 access key id, required if you set remote-logging.target=S3 | ||
| access.key.id: <access.key.id> | ||
| # s3 access key secret, required if you set remote-logging.target=S3 | ||
| access.key.secret: <access.key.secret> | ||
| # s3 bucket name, required if you set remote-logging.target=S3 | ||
| bucket.name: <bucket.name> | ||
| # s3 endpoint, required if you set remote-logging.target=S3 | ||
| endpoint: <endpoint> | ||
| # s3 region, required if you set remote-logging.target=S3 | ||
| region: <region> | ||
| google.cloud.storage: | ||
| # the location of the google cloud credential, required if you set remote-logging.target=GCS | ||
| credential: /path/to/credential | ||
| # gcs bucket name, required if you set remote-logging.target=GCS | ||
| bucket.name: <your-bucket> | ||
| abs: | ||
| # abs account name, required if you set resource.storage.type=ABS | ||
| account.name: <your-account-name> | ||
| # abs account key, required if you set resource.storage.type=ABS | ||
| account.key: <your-account-key> | ||
| # abs container name, required if you set resource.storage.type=ABS | ||
| container.name: <your-container-name> | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters