refactor: redefine configuration structure (database-lab#278):

- change configs structure
- adjust Docker image
- clean up README

Author: agneum

.gitignore

@@ -8,3 +8,6 @@
 /deploy/
 
 config/config.yml
+configs/config.yml
+configs/joe.yml
+meta/sessions.json

Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache bash postgresql-client
 WORKDIR /home/
 
 COPY ./bin/joe ./bin/joe
-COPY ./config ./config
+COPY ./configs/explain ./explain
 
 EXPOSE $SERVER_PORT
 

README.md

@@ -18,100 +18,8 @@ at the upper right corner):
 
 ![Add a star](./assets/star.gif)
 
-To discuss Joe, [join our community Slack](https://database-lab-team-slack-invite.herokuapp.com/).
-
-## Install Software
-
-### 1. Database Lab
-Install and setup [Database Lab](https://gitlab.com/postgres-ai/database-lab)
- 
-Prepare one or more Database Lab instances before configuring Joe bot. 
-
-> ⚠ Make sure that `accessHost` is specified in the Database Lab configuration file.
-
-Then, configure ways of communication with Joe.
-
-### 2. Configure communication channels
-
-There are two available types of communication:
-- Web UI powered by [Postgres.ai Console](https://postgres.ai/console/)
-- Slack
-
-You can use both of them in parallel. Feel free to implement more types of communication: see [communication channels issues](https://gitlab.com/postgres-ai/joe/-/issues?label_name%5B%5D=Communication+channel).
-
-### 2a. Set up Joe in Postgres.ai Console ("Web UI")
-If you don't need Web UI and prefer working with Joe only in messengers (such as Slack), proceed to the next step.
-
-To configure Web UI:
-
-1. First, get your `PLATFORM_TOKEN`. In [Postgres.ai Console](https://postgres.ai/console/), switch to proper organization and open the `Access Tokens` page.
-1. Then, go to the `Joe instances` page from the `SQL Optimization` sidebar section, choose a project from the dropdown menu and `Add instance`.
-1. Generate `Signing secret`. Use the secret as `WEBUI_SIGNING_SECRET` at the configuration file. We will add and verify the URL on the last step.
-
-
-### 2b. Slack App
-Configure a new Slack App in order to use Joe in Slack and add the app to your
-team Workspace. Joe Bot should be available with public URL calls from Slack.
-1. Create "#db-lab" channel in your Slack Workspace (You can use another channel name).
-1. [Create a new Slack App](https://api.slack.com/apps?new_app=1).
-    * Use "Joe Bot" as App Name and select a proper team Workspace.
-1. Add Bot User.
-    * Use "Joe Bot" as Display Name and "joe-bot" as the default username.
-1. Grant permissions on the "OAuth & Permissions" page for the following "Bot Token Scopes" :
-    * `channels:history`
-    * `chat:write`
-    * `files:read`
-    * `files:write`
-    * `incoming-webhook`
-    * `reactions:write`
-    * `users.profile:read`
-    * `users:read`
-1. Run Joe Bot with `Bot User OAuth Access Token ("xoxb-TOKEN")` from "OAuth & Permissions" Feature and `Signing Secret` from "Basic Information" page (See **Run** below).
-1. Enable Incoming Webhooks Feature.
-    * Press "Add New Webhook to Workspace" and select a previously created channel to post token.
-1. Enable Event Subscriptions Feature.
-    * Specify Request URL (URL will be verified by Slack API) (e.g. http://35.200.200.200:2400, https://joe.dev.domain.com). You would need to run Joe with proper settings before you could verify Request URL.
-
-### 3. Run
-Deploy Joe instance in your infrastructure. You would need to:
-
-1. Configure communication channels. You can copy the example `config/config.example.yml` to `~/.dblab/joe.yml`, inspect all configuration options, and adjust if needed.
-   
-1. Run the Joe Docker image to connect with the Database Lab server according to the previous configurations. 
-    Example:
-
-    ```bash
-    docker run \
-      --name joe_bot \
-      --publish 2400:2400 \
-      --volume ~/.dblab/joe.yml:/home/config/config.yml \
-      --restart=on-failure \
-      --detach \
-      postgresai/joe:latest
-    ``` 
-    The Joe instance will be running by port 2400 of the current machine.
-    
-1. Make a publicly accessible HTTP(S) server port specified in the configuration for Web UI/Slack Events Request URL.
-
-### 4. Verify the configuration
-
-### 4a. Finish the WebUI configuration
-
-1. Return to the page of Joe configuration in the Console, enter the URL with the specific path `/webui/`. For example, `https://joe.dev.domain.com/webui/`.
-1. Press the `Verify` button to check connection and `Add` the instance after the verification is passed.
-1. Choose the created instance and send a command.
-
-
-### 4b. Finish the Slack App configuration
-1. Enable Event Subscriptions Feature.
-    * Go to the "Event Subscriptions" page.
-    * Specify Request URL adding the specific for connection path: `/slack/` (URL will be verified by Slack API). You would need to run Joe with proper settings before you could verify Request URL. For example, `https://joe.dev.domain.com/slack/`
-    * In the "Subscribe to Bot Events" dropdown-tab add `message.channels`.
-    * Press "Save Changes".
-
-1. Invite "Joe Bot" to "#db-lab" channel.
-1. Send a command to the #db-lab channel. For example, `help`.
-
+## Installation
+Follow the [tutorial](https://postgres.ai/docs/tutorials/joe-setup) to install Joe Bot and start using it for PostgreSQL query optimization
 
 ## Development
 See our [GitLab Container Registry](https://gitlab.com/postgres-ai/joe/container_registry) for develop builds. 
@@ -120,4 +28,4 @@ See our [GitLab Container Registry](https://gitlab.com/postgres-ai/joe/container
 
 Bug reports, ideas, and merge requests are welcome: https://gitlab.com/postgres-ai/joe
 
-To discuss Joe, join our Slack: https://database-lab-team-slack-invite.herokuapp.com/
+To discuss Joe, [join our community Slack](https://slack.postgres.ai/)

cmd/joe/main.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path"
 	"syscall"
 	"time"
 
@@ -30,9 +31,6 @@ import (
 
 const (
 	shutdownTimeout = 60 * time.Second
-
-	configFilePath   = "config/config.yml"
-	sessionsFilePath = "config/sessions.json"
 )
 
 // ldflag variables.
@@ -41,7 +39,7 @@ var buildTime, version string
 func main() {
 	version := formatBotVersion()
 
-	botCfg, err := loadConfig(configFilePath)
+	botCfg, err := loadAppConfig()
 	if err != nil {
 		log.Fatal("failed to load config: ", err)
 	}
@@ -60,7 +58,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	shutdownCh := setShutdownListener()
 
-	sessionsStorage := storage.NewJSONSessionData(sessionsFilePath)
+	sessionsStorage := storage.NewJSONSessionData(path.Join(config.MetadataPath, config.SessionsFilename))
 	if err := sessionsStorage.Load(); err != nil {
 		log.Fatal("unable to load sessions data: ", err)
 	}
@@ -87,8 +85,11 @@ func main() {
 	}
 }
 
-func loadConfig(configPath string) (*config.Config, error) {
-	var botCfg config.Config
+func loadAppConfig() (*config.Config, error) {
+	var (
+		botCfg     config.Config
+		configPath = path.Join(config.ConfigsPath, config.AppFilename)
+	)
 
 	if err := cleanenv.ReadConfig(configPath, &botCfg); err != nil {
 		return nil, errors.Wrap(err, "failed to read a config file")

config/envs/default.sh

@@ -1,5 +1,5 @@
 # Joe Bot configuration.
-# Copy this file to: ./config/config.yml
+# Copy this file to: ~/.dblab/joe/configs/joe.yml
 
 app:
   # HTTP server IP address or host.

config/config.example.yml renamed to configs/config.example.yml

@@ -10,16 +10,6 @@ run() {
 
   set +x
 
-  # Instead of editing default values here create a separate config file.
-  # e.g. for staging environment create a file /deploy/configs/staging.sh
-  # use `export` to define variables and run with `ENV=staging makerun.sh`.
-  if [ -z ${ENV+x} ]; then
-    echo "Using default variables"
-    source ./config/envs/default.sh
-  else
-    source ./config/envs/${ENV}.sh
-  fi
-
   # Read and set git status info if it wasn't done before.
   if [ "$INCLUDE_GIT_STATUS" == "true" ] && [ -z "$GIT_COMMIT_HASH" ]; then
     echo "Fetching git status..."

config/explain.yaml renamed to configs/explain/explain.yaml

@@ -12,6 +12,20 @@ import (
 	"gitlab.com/postgres-ai/joe/pkg/pgexplain"
 )
 
+const (
+	// ConfigsPath declares the directory where configuration files are stored.
+	ConfigsPath = "configs"
+
+	// AppFilename declares name of the application configuration file.
+	AppFilename = "joe.yml"
+
+	// MetadataPath declares path where metadata files are stored.
+	MetadataPath = "meta"
+
+	// SessionsFilename declares name of the file that stores current Joe Bot sessions.
+	SessionsFilename = "sessions.json"
+)
+
 // Config defines an App configuration.
 type Config struct {
 	App            App                          `yaml:"app"`

do.sh

@@ -7,6 +7,7 @@ package config
 import (
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 
 	"github.com/pkg/errors"
@@ -15,19 +16,32 @@ import (
 	"gitlab.com/postgres-ai/joe/pkg/pgexplain"
 )
 
+const (
+	// explainPath declares the directory where explain config files are stored.
+	explainPath = "explain"
+
+	// explainFilename declares name of explain configuration file.
+	explainFilename = "explain.yaml"
+)
+
 // LoadExplainConfig loads and parses an explain configuration.
 func LoadExplainConfig() (pgexplain.ExplainConfig, error) {
 	var explainConfig pgexplain.ExplainConfig
 
-	if err := loadConfig(&explainConfig, "explain.yaml"); err != nil {
+	if err := readConfig(&explainConfig, path.Join(explainPath, explainFilename)); err != nil {
 		return explainConfig, err
 	}
 
 	return explainConfig, nil
 }
 
-func loadConfig(config interface{}, name string) error {
-	b, err := ioutil.ReadFile(getConfigPath(name))
+func readConfig(config interface{}, name string) error {
+	cfgPath, err := getConfigPath(name)
+	if err != nil {
+		return errors.Wrap(err, "failed to build config path")
+	}
+
+	b, err := ioutil.ReadFile(cfgPath)
 	if err != nil {
 		return errors.Errorf("Error loading %s config file: %v", name, err)
 	}
@@ -39,10 +53,16 @@ func loadConfig(config interface{}, name string) error {
 	return nil
 }
 
-func getConfigPath(name string) string {
-	bindir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
-	dir, _ := filepath.Abs(filepath.Dir(bindir))
-	path := dir + string(os.PathSeparator) + "config" + string(os.PathSeparator) + name
+func getConfigPath(name string) (string, error) {
+	binDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	if err != nil {
+		return "", err
+	}
+
+	dir, err := filepath.Abs(filepath.Dir(binDir))
+	if err != nil {
+		return "", err
+	}
 
-	return path
+	return path.Join(dir, name), nil
 }