adds multi-datasource support based on configuration which includes OCI Vault support and various fixes.

README.md

@@ -98,6 +98,30 @@ OAuth2 https://spring.io/guides/tutorials/spring-boot-oauth2/
 
 The reader is referred to this material to configure security and other aspects as appropriate.
 
+## Support for Multiple Databases
+
+This feature can be used in two ways:
+
+1. By accessing the `/scrape` endpoint and passing a `target` parameter that is the full dsn value. 
+   For example: `http://localhost:9161/scrape?target`
+2. By accessing the `/scrape` endpoint and passing a `name` parameter that is the name of the datasource 
+   as defined in the yaml file located at the environment variable `MULTI_DATASOURCE_CONFIG`
+    The yaml file takes the following form and an example can be found under examples/multidatasource_config.yaml :
+```    
+    dataSourceName :
+       serviceName :
+       userName :
+       password :
+       TNS_ADMIN :
+       # if present, OCI Vault is used for password rather than "password" attribute
+       passwordOCID :
+       # the following is applicable only if OCI config file is used rather than instance principals authentication (generally only the case in development)
+       ociConfigFile :
+       ociRegion :
+       ociProfile :
+```   
+The feature is currently only applicable to metrics, not tracing or logging.
+The feature currently uses the same global `DEFAULT_METRICS` config for every datasource.
 
 [Metrics Exporter]: Metrics.md
 [Log Exporter]: Logs.md

examples/metrics/default-metrics.toml

@@ -3,4 +3,10 @@ context = "sessions"
 labels = ["inst_id", "status", "type"]
 metricsdesc = { value = "Gauge metric with count of sessions by status and type." }
 request = "select inst_id, status, type, count(*) as value from gv$session group by status, type, inst_id"
-ignorezeroresult = true
+ignorezeroresult = true
+
+[[metric]]
+context = "context_with_labels"
+labels = [ "label_1", "label_2" ]
+request = "SELECT 1 as value_1, 2 as value_2, 'First label' as label_1, 'Second label' as label_2 FROM DUAL"
+metricsdesc = { value_1 = "Simple example returning always 1.", value_2 = "Same but returning always 2." }

examples/multidatasource_config.yaml

@@ -0,0 +1,22 @@
+dataSourceMyFirstDB :
+  serviceName : myservicename1
+  userName : myuser1
+  password : mypassword1
+  TNS_ADMIN : /somefolder/Wallet_somewallet1
+  # if present, OCI Vault is used for password rather than "password" attribute
+#  passwordOCID :
+  # the following is applicable only if OCI config file is used rather than instance principals authentication (generally only the case in development)
+#  ociConfigFile :
+#  ociRegion :
+#  ociProfile :
+dataSourceMySecondDB :
+  serviceName : myservicename1
+  userName : myuser2
+  password : mypassword2
+  TNS_ADMIN : /somefolder/Wallet_somewallet2
+#  # if present, OCI Vault is used for password rather than "password" attribute
+#  passwordOCID :
+  # the following is applicable only if OCI config file is used rather than instance principals authentication (generally only the case in development)
+#  ociConfigFile :
+#  ociRegion :
+#  ociProfile :

pom.xml

@@ -127,6 +127,10 @@
 			<artifactId>oci-java-sdk-secrets</artifactId>
 			<version>1.32.2</version>
 		</dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
 	</dependencies>
 
 	<build>

src/main/java/oracle/observability/DataSourceConfig.java

@@ -0,0 +1,16 @@
+package oracle.observability;
+
+import lombok.Data;
+
+@Data
+public class DataSourceConfig {
+    private String dataSourceName;
+    private String serviceName;
+    private String userName;
+    private String password;
+    private String TNS_ADMIN;
+    private String passwordOCID;
+    private String ociConfigFile;
+    private String ociRegion;
+    private String ociProfile;
+}

src/main/java/oracle/observability/ObservabilityExporter.java

@@ -6,15 +6,13 @@
 import com.oracle.bmc.secrets.model.Base64SecretBundleContentDetails;
 import com.oracle.bmc.secrets.requests.GetSecretBundleRequest;
 import com.oracle.bmc.secrets.responses.GetSecretBundleResponse;
-import oracle.observability.metrics.MetricsExporter;
 import oracle.ucp.jdbc.PoolDataSource;
 import oracle.ucp.jdbc.PoolDataSourceFactory;
 import org.apache.commons.codec.binary.Base64;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
 import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
@@ -26,61 +24,110 @@ public class ObservabilityExporter {
     public File DEFAULT_METRICS_FILE;
     public String CUSTOM_METRICS      = System.getenv("CUSTOM_METRICS"); //
     public String QUERY_TIMEOUT       = System.getenv("QUERY_TIMEOUT"); // "5"
+    public static final String CONTEXT = "context";
+    public static final String REQUEST = "request";
+
+    //Single/global datasource config related....
     public String DATABASE_MAXIDLECONNS       = System.getenv("DATABASE_MAXIDLECONNS"); // "0"
     public String DATABASE_MAXOPENCONNS       = System.getenv("DATABASE_MAXOPENCONNS"); // "10"
     public static String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
-    public static String DATA_SOURCE_USER = System.getenv("DATA_SOURCE_USER"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
-    public static String DATA_SOURCE_PASSWORD = System.getenv("DATA_SOURCE_PASSWORD"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
-    public static String DATA_SOURCE_SERVICENAME = System.getenv("DATA_SOURCE_SERVICENAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
-    public String TNS_ADMIN = System.getenv("TNS_ADMIN");  //eg /msdataworkshop/creds
-    public String OCI_REGION = System.getenv("OCI_REGION");  //eg us-ashburn-1
-    public String VAULT_SECRET_OCID = System.getenv("VAULT_SECRET_OCID");  //eg ocid....
-    public String OCI_CONFIG_FILE = System.getenv("OCI_CONFIG_FILE");  //eg "~/.oci/config"
-    public String OCI_PROFILE = System.getenv("OCI_PROFILE");  //eg "DEFAULT"
-    public static final String CONTEXT = "context";
-    public static final String REQUEST = "request";
+    //if all three of the following exist, they are internally concatenated and override/used as DATA_SOURCE_NAME
+    public static String DATA_SOURCE_USER = System.getenv("DATA_SOURCE_USER"); //eg %USER%
+    public static String DATA_SOURCE_PASSWORD = System.getenv("DATA_SOURCE_PASSWORD"); //eg $(dbpassword)
+    public static String DATA_SOURCE_SERVICENAME = System.getenv("DATA_SOURCE_SERVICENAME"); //eg %PDB_NAME%_tp
+    public static String TNS_ADMIN = System.getenv("TNS_ADMIN");  //eg /msdataworkshop/creds
 
-    static {
+    public static String OCI_REGION = System.getenv("OCI_REGION");  //eg us-ashburn-1
+    public static String VAULT_SECRET_OCID = System.getenv("VAULT_SECRET_OCID");  //eg ocid....
+    public static String OCI_CONFIG_FILE = System.getenv("OCI_CONFIG_FILE");  //eg "~/.oci/config"
+    public static String OCI_PROFILE = System.getenv("OCI_PROFILE");  //eg "DEFAULT"
 
+    //MULTI_DATASOURCE_CONFIG related....
+    public static String MULTI_DATASOURCE_CONFIG = System.getenv("MULTI_DATASOURCE_CONFIG");
+    public static final String SERVICE_NAME_STRING = "serviceName";
+    public static final String USER_NAME_STRING = "userName";
+    public static final String PASSWORD_STRING = "password";
+    public static final String TNS_ADMIN_STRING = "TNS_ADMIN";
+    public static final String PASSWORD_OCID_STRING = "passwordOCID";
+    public static final String OCI_CONFIG_FILE_STRING = "ociConfigFile";
+    public static final String OCI_REGION_STRING = "ociRegion";
+    public static final String OCI_PROFILE_STRING = "ociProfile";
+
+    static { // not really necessary but gives information that a global datasource is in use
         if (DATA_SOURCE_USER != null && DATA_SOURCE_PASSWORD != null && DATA_SOURCE_SERVICENAME != null) {
             DATA_SOURCE_NAME = DATA_SOURCE_USER + "/" + DATA_SOURCE_PASSWORD + "@" + DATA_SOURCE_SERVICENAME;
             LOGGER.info("DATA_SOURCE_NAME = DATA_SOURCE_USER + \"/\" + DATA_SOURCE_PASSWORD + \"@\" + DATA_SOURCE_SERVICENAME");
             //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
         }
     }
-    PoolDataSource observabilityDB;
+    PoolDataSource globalObservabilityDB;
+
+    //This map is used for multi-datasource scraping, both when using dns target string and config
     Map<String, PoolDataSource> dataSourceNameToDataSourceMap = new HashMap<>();
 
+    //This map is used for multi-datasource scraping when using config only
+    public static Map<String, DataSourceConfig> dataSourceNameToDataSourceConfigMap = new HashMap<>();
+
+    //used by logs and tracing exporters as they do not currently support multi-datasource config
     public PoolDataSource getPoolDataSource() throws SQLException {
-        return getPoolDataSource(DATA_SOURCE_NAME);
+        return getPoolDataSource(DATA_SOURCE_NAME, false);
     }
-    public PoolDataSource getPoolDataSource(String dataSourceName) throws SQLException {
-        if (dataSourceName.equals(DATA_SOURCE_NAME)) {
-            if (observabilityDB != null) return observabilityDB;
-            return observabilityDB = getDataSource(DATA_SOURCE_NAME);
+
+    public PoolDataSource getPoolDataSource(String dataSourceName, boolean isScrapeByName) throws SQLException {
+        if (DATA_SOURCE_NAME != null && dataSourceName.equals(DATA_SOURCE_NAME)) {
+            if (globalObservabilityDB != null) return globalObservabilityDB;
+            return globalObservabilityDB = getDataSource(DATA_SOURCE_NAME);
         } else {
             if(dataSourceNameToDataSourceMap.containsKey(dataSourceName) && dataSourceNameToDataSourceMap.get(dataSourceName) != null)
                 return dataSourceNameToDataSourceMap.get(dataSourceName);
-            PoolDataSource poolDataSource = getDataSource(dataSourceName);
+
+            System.out.println("putting dataSourceName:" + dataSourceName + " isScrapeByName:" + isScrapeByName +
+                    " ObservabilityExporter.dataSourceNameToDataSourceConfigMap.get(dataSourceName):"+
+                    ObservabilityExporter.dataSourceNameToDataSourceConfigMap.get(dataSourceName));
+            PoolDataSource poolDataSource = isScrapeByName?
+                getDataSource(ObservabilityExporter.dataSourceNameToDataSourceConfigMap.get(dataSourceName))
+                :getDataSource(dataSourceName);
             dataSourceNameToDataSourceMap.put(dataSourceName, poolDataSource);
             return poolDataSource;
         }
     }
 
     private PoolDataSource getDataSource(String dataSourceName) throws SQLException {
-        PoolDataSource poolDataSource = PoolDataSourceFactory.getPoolDataSource();
-        poolDataSource.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource");
         String user = dataSourceName.substring(0, dataSourceName.indexOf("/"));
         String pw = dataSourceName.substring(dataSourceName.indexOf("/") + 1, dataSourceName.indexOf("@"));
         String serviceName = dataSourceName.substring(dataSourceName.indexOf("@") + 1);
-        String url = "jdbc:oracle:thin:@" + serviceName + "?TNS_ADMIN=" + TNS_ADMIN;
+        return getPoolDataSource(dataSourceName, user, pw, serviceName, TNS_ADMIN,
+                VAULT_SECRET_OCID, OCI_CONFIG_FILE, OCI_PROFILE, OCI_REGION, false);
+    }
+    private PoolDataSource getDataSource(DataSourceConfig dataSourceConfig) throws SQLException {
+        return getPoolDataSource(dataSourceConfig.getDataSourceName(),
+                dataSourceConfig.getUserName(),
+                dataSourceConfig.getPassword(),
+                dataSourceConfig.getServiceName(),
+                dataSourceConfig.getTNS_ADMIN(),
+                dataSourceConfig.getPasswordOCID(),
+                dataSourceConfig.getOciConfigFile(),
+                dataSourceConfig.getOciProfile(),
+                dataSourceConfig.getOciRegion(),
+                true);
+    }
+
+    private PoolDataSource getPoolDataSource(
+            String dataSourceName, String user, String pw, String serviceName, String tnsAdmin,
+            String vaultSecretOcid, String ociConfigFile, String ociProfile, String ociRegion, boolean isScrapeByName) throws SQLException {
+        System.out.println("getPoolDataSource dataSourceName = " + dataSourceName + ", user = " + user + ", pw = " + pw + ", serviceName = " + serviceName + ", vaultSecretOcid = " + vaultSecretOcid + ", ociConfigFile = " + ociConfigFile + ", ociProfile = " + ociProfile + ", ociRegion = " + ociRegion + ", isScrapeByName = " + isScrapeByName);
+        PoolDataSource poolDataSource = PoolDataSourceFactory.getPoolDataSource();
+        poolDataSource.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource");
+        String url = "jdbc:oracle:thin:@" + serviceName + "?TNS_ADMIN=" + tnsAdmin;
         poolDataSource.setURL(url);
         poolDataSource.setUser(user);
-        if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("") || !dataSourceName.equals(DATA_SOURCE_NAME)) {
+        if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("") ||
+                //vault is not supported with scrape by dns currently, only with scrape by datasource name and global datasource
+                (!isScrapeByName && !dataSourceName.equals(DATA_SOURCE_NAME)) ) {
             poolDataSource.setPassword(pw);
         } else {
             try {
-                poolDataSource.setPassword(getPasswordFromVault());
+                poolDataSource.setPassword(getPasswordFromVault(vaultSecretOcid, ociConfigFile, ociProfile, ociRegion));
             } catch (IOException e) {
                 throw new SQLException(e);
             }
@@ -89,18 +136,18 @@ private PoolDataSource getDataSource(String dataSourceName) throws SQLException
     }
 
 
-    public String getPasswordFromVault() throws IOException {
+    public String getPasswordFromVault(String vaultSecretOcid, String ociConfigFile, String ociProfile, String ociRegion) throws IOException {
         SecretsClient secretsClient;
-        if (OCI_CONFIG_FILE == null || OCI_CONFIG_FILE.trim().equals("")) {
+        if (ociConfigFile == null || ociConfigFile.trim().equals("")) {
             secretsClient = new SecretsClient(InstancePrincipalsAuthenticationDetailsProvider.builder().build());
         } else {
-            String profile = OCI_PROFILE==null || OCI_PROFILE.trim().equals("") ? "DEFAULT": OCI_PROFILE;
-            secretsClient = new SecretsClient(new ConfigFileAuthenticationDetailsProvider(OCI_CONFIG_FILE, profile));
+            String profile = ociProfile ==null || ociProfile.trim().equals("") ? "DEFAULT": ociProfile;
+            secretsClient = new SecretsClient(new ConfigFileAuthenticationDetailsProvider(ociConfigFile, profile));
         }
-        secretsClient.setRegion(OCI_REGION);
+        secretsClient.setRegion(ociRegion);
         GetSecretBundleRequest getSecretBundleRequest = GetSecretBundleRequest
                 .builder()
-                .secretId(VAULT_SECRET_OCID)
+                .secretId(vaultSecretOcid )
                 .stage(GetSecretBundleRequest.Stage.Current)
                 .build();
         GetSecretBundleResponse getSecretBundleResponse = secretsClient.getSecretBundle(getSecretBundleRequest);

src/main/java/oracle/observability/logs/LogsExporter.java

@@ -43,6 +43,7 @@ public void run() {
 				LOGGER.debug("LogsExporter default metrics from:" + DEFAULT_METRICS);
 				if(LOG_INTERVAL!=null && !LOG_INTERVAL.trim().equals("")) logInterval = Integer.getInteger(LOG_INTERVAL);
 				LOGGER.debug("LogsExporter logInterval:" + logInterval);
+				//todo move to common/ObservabilityExporter location and log something friendly if it does not exist and exit, ie fast fail startup
 				File tomlfile = new File(DEFAULT_METRICS);
 				TomlMapper mapper = new TomlMapper();
 				JsonNode jsonNode = mapper.readerFor(LogsExporterConfigEntry.class).readTree(new FileInputStream(tomlfile));

src/main/java/oracle/observability/metrics/CollectorRegistryWithGaugeMap.java

@@ -0,0 +1,12 @@
+package oracle.observability.metrics;
+
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.Gauge;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class CollectorRegistryWithGaugeMap extends CollectorRegistry {
+    Map<String, Gauge> gaugeMap = new HashMap<>();
+
+}