feat: change config management from env to yaml (#9)

DipandaAser · web-flow · commit 62e0c920633e · 2022-08-27T14:32:43.000+01:00
* disable test caching

* switch config from env to yaml

* update readme to match the new changes about config file

* add production ready docker compose file

* force recreate of container with docker compose

* update CI workflow

* update CI workflow

* update CI workflow

* update CI workflow
diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1,6 @@
 .env.test
-*.env
+*.env
+config.yaml
+config
+/config.example.yaml
+/config.test.example.yaml
diff --git a/.env.example b/.env.example
diff --git a/.env.test.example b/.env.test.example
diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml
@@ -22,14 +22,14 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
       - name: Run Tests
         run: |
-          (
-            echo BOT_TOKEN=${{ secrets.CI_BOT_TOKEN_1 }}
-            echo BOT_TOKENS=${{ secrets.CI_BOT_TOKEN_1 }},${{ secrets.CI_BOT_TOKEN_2 }},${{ secrets.CI_BOT_TOKEN_3 }}
-            echo TOKENS=${{ secrets.CI_BOT_TOKEN_1 }},${{ secrets.CI_BOT_TOKEN_2 }},${{ secrets.CI_BOT_TOKEN_3 }}
-            echo DRAFT_CHAT_ID=${{ secrets.CI_DRAFT_CHAT_ID }}
-            echo CHAT_ID=${{ secrets.CI_CHAT_ID }}
-            ) > .env
-          cp .env .env.test
+          mkdir config
+          echo "tokens:" >./config-test/config.yaml
+          echo "  - "${{ secrets.CI_BOT_TOKEN_1 }}"" >> ./config-test/config.yaml
+          echo "  - "${{ secrets.CI_BOT_TOKEN_2 }}"" >> ./config-test/config.yaml
+          echo "  - "${{ secrets.CI_BOT_TOKEN_3 }}"" >> ./config-test/config.yaml
+          echo "api_key: "" " >> ./config-test/config.yaml
+          echo "chat_id: ${{ secrets.CI_CHAT_ID }}" >> ./config-test/config.yaml
+          echo "draft_chat_id: ${{ secrets.CI_DRAFT_CHAT_ID }}" >> ./config-test/config.yaml
           make test
           chmod 777 ./scripts/wait-for-it/wait-for-it.sh
           make docker-e2etest
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
 .idea
 .env.test
-*.env
+*.env
+config.yaml
+config.test.yaml
+config
diff --git a/Dockerfile b/Dockerfile
@@ -14,17 +14,23 @@ COPY . .
 RUN go build -o bin/tg-bot-storage ./cmd/main.go
 
 
-RUN apk add --no-cache ca-certificates
+#RUN apk add --no-cache ca-certificates
 
 # build image with the binary
-FROM scratch
+FROM alpine
 
+RUN apk add --no-cache ca-certificates
+RUN apk add bash
 # copy certificate to be able to make https request to telegram
-COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+#COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+
+WORKDIR /app
 
 # copy the binary
-COPY --from=build /build/bin/tg-bot-storage /
+COPY --from=build /build/bin/tg-bot-storage /app/
+
+VOLUME /app/config
 
 EXPOSE 7000
 
-ENTRYPOINT ["/tg-bot-storage"]
+ENTRYPOINT ["/app/tg-bot-storage"]
diff --git a/Makefile b/Makefile
@@ -4,7 +4,7 @@ APPNAME=tg-bot-storage
 ## test: run tests on cmd and pkg files.
 .PHONY: test
 test: vet fmt
-	CI="false" go test ./...
+	CI="false" go test -v -count=1 ./...
 
 ## build: build application binary.
 .PHONY: build
@@ -28,7 +28,7 @@ e2etest:
 
 ## docker-e2etest: run e2etests in a docker compose
 docker-e2etest:
-	docker-compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from e2etests
+	docker-compose -f docker-compose.test.yml up --force-recreate --abort-on-container-exit --exit-code-from e2etests
 
 ## docker-build: build the api docker image
 .PHONY: docker-build
diff --git a/README.md b/README.md
@@ -1,17 +1,30 @@
 # Telegram Bot Storage(in development)
+
 ![Build Status](https://github.com/dipandaaser/tg-bot-storage/workflows/CI/badge.svg)
 [![License](https://img.shields.io/github/license/dipandaaser/tg-bot-storage)](LICENSE)
 [![Release](https://img.shields.io/github/release/dipandaaser/tg-bot-storage.svg)](https://github.com/dipandaaser/tg-bot-storage/releases/latest)
 [![GitHub Releases Stats of tg-bot-storage](https://img.shields.io/github/downloads/dipandaaser/tg-bot-storage/total.svg?logo=github)](https://somsubhra.github.io/github-release-stats/?username=dipandaaser&repository=tg-bot-storage)
 
-Telegram Bot Storage is a simple library for storing files in Telegram using bot and by passing limits listed on [telegram bot limits website](https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this)
+Telegram Bot Storage is a simple library for storing files in Telegram using bot and by passing limits listed
+on [telegram bot limits website](https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this)
 
 ### Development
 
 #### Dependencies
+
 - Golang 1.16+
 - GNU Make
 
+#### How to run
+
+Create a `config.yaml` file under the [config](config) folder.Use [config.example.yaml](config.example.yaml) as a
+template to fill the file.
+
+Run the following command: `make run` or `go run ./cmd/main.go`
+
+
 #### How to test
-Create a `.env.test` file with [.env.test.example](.env.test.example) as a template OR just set env var with the same name.
-Run the following command: `make test`
+Create a `config.yaml` file under the [config-test](config-test)
+folder. Use [config.test.example.yaml](config.example.yaml) as a template to fill the file
+
+Run the following commands: `make test` and `make docker-e2etest`
diff --git a/config-test/config.go b/config-test/config.go
@@ -0,0 +1,36 @@
+//
+package config_test
+
+import (
+	"github.com/DipandaAser/tg-bot-storage/internal/config"
+	"gopkg.in/yaml.v3"
+	"os"
+)
+
+type Config struct {
+	config.Config `yaml:",inline"`
+	//ChatID is the ID of the chat to store message, this is use in some test
+	ChatID int64 `yaml:"chat_id"`
+	//DraftChatID is the ID of the chat used as a draft chat when downloading file
+	DraftChatID int64 `yaml:"draft_chat_id"`
+}
+
+var defaultConfig *Config
+
+//GetDefaultConfig returns a config with default values from the yaml configFilePath
+func GetConfig(configFilePath string) Config {
+	if defaultConfig == nil {
+		defaultConfig = &Config{Config: config.Config{Tokens: make([]string, 0)}}
+		file, err := os.Open(configFilePath)
+		if err != nil {
+			return Config{}
+		}
+		defer file.Close()
+
+		err = yaml.NewDecoder(file).Decode(defaultConfig)
+		if err != nil {
+			return Config{}
+		}
+	}
+	return *defaultConfig
+}
diff --git a/config.example.yaml b/config.example.yaml
@@ -0,0 +1,5 @@
+tokens:
+  - "bot token 1"
+  - "bot token 2"
+  - "bot token 3"
+api_key: ""
diff --git a/config.test.example.yaml b/config.test.example.yaml
@@ -0,0 +1,9 @@
+tokens:
+  - "bot token 1"
+  - "bot token 2"
+  - "bot token 3"
+api_key: ""
+#chat_id(int) is the ID of the chat to store message, this is use in some test
+chat_id:
+#draft_chat_id(int) is the ID of the chat used as a draft chat when downloading file
+draft_chat_id:
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -2,29 +2,24 @@ version: '3'
 
 services:
   api:
-    restart: always
+    restart: on-failure
     build:
       context: .
       dockerfile: Dockerfile
-    env_file:
-      - ./.env
-    environment:
-      api_key: "12345"
     ports:
-      - 7000:7000
+      - "7000:7000"
     extra_hosts:
       - host.docker.internal:host-gateway
+    volumes:
+      - "./config-test:/app/config"
 
   e2etests:
     depends_on:
       - api
     image: golang:buster
     command: /app/scripts/wait-for-it/wait-for-it.sh api:7000 -t 300 -- make -C /app e2etest-compose
-    env_file:
-      - ./.env
     environment:
       CI: "true"
       api_host: "http://api:7000/"
-      api_key: "12345"
     volumes:
       - .:/app
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,10 @@
+version: '3'
+
+services:
+  bot-storage:
+    image: ghcr.io/dipandaaser/bot-storage:latest
+    restart: on-failure
+    ports:
+      - "7000:7000"
+    volumes:
+      - "./config:/app/config"
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -1,28 +1,36 @@
 package config
 
 import (
-	"github.com/joho/godotenv"
+	"gopkg.in/yaml.v3"
 	"os"
-	"strings"
+)
+
+const (
+	configFilePath = "./config/config.yaml"
 )
 
 var defaultConfig *Config
 
 //Config describes the server configuration
 type Config struct {
-	Tokens []string
-	ApiKey string
+	Tokens []string `yaml:"tokens"`
+	ApiKey string   `yaml:"api_key"`
 }
 
-//GetDefaultConfig returns a config with default values  and env variables
+//GetDefaultConfig returns a config with default values from the yaml configFilePath
 func GetDefaultConfig() Config {
 	if defaultConfig == nil {
-		//load postgres env variables
-		_ = godotenv.Load()
+		defaultConfig = &Config{Tokens: make([]string, 0)}
+		// read the config file
+		file, err := os.Open(configFilePath)
+		if err != nil {
+			return Config{}
+		}
+		defer file.Close()
 
-		defaultConfig = &Config{
-			Tokens: strings.Split(os.Getenv("TOKENS"), ","),
-			ApiKey: os.Getenv("API_KEY"),
+		err = yaml.NewDecoder(file).Decode(defaultConfig)
+		if err != nil {
+			return Config{}
 		}
 	}
 	return *defaultConfig
diff --git a/pkg/bot/client_test.go b/pkg/bot/client_test.go
@@ -2,35 +2,27 @@ package bot
 
 import (
 	"bytes"
-	"github.com/joho/godotenv"
+	config_test "github.com/DipandaAser/tg-bot-storage/config-test"
 	"io/ioutil"
 	"log"
-	"os"
-	"strconv"
 	"strings"
 	"sync"
 	"testing"
 )
 
 const (
-	ENVBOTTOKEN    = "BOT_TOKEN"
-	ENVCHATID      = "CHAT_ID"
-	ENVDRAFTCHATID = "DRAFT_CHAT_ID"
+	configFilePath = "../../config-test/config.yaml"
 )
 
-func init() {
-	_ = godotenv.Load("../../.env.test")
-}
-
 func Test_UploadFileReader(t *testing.T) {
-	client, err := NewClient(os.Getenv(ENVBOTTOKEN))
+	client, err := NewClient(config_test.GetConfig(configFilePath).Tokens[0])
 	if err != nil {
 		t.Fatal(err)
 		return
 	}
 
 	t.Run("Upload one file", func(t *testing.T) {
-		chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64)
+		chatId := config_test.GetConfig(configFilePath).ChatID
 		data := bytes.NewReader([]byte("data"))
 		_, err = client.UploadFileReader(chatId, t.Name(), data)
 		if err != nil {
@@ -41,14 +33,14 @@ func Test_UploadFileReader(t *testing.T) {
 }
 
 func Test_UploadFileBuffer(t *testing.T) {
-	client, err := NewClient(os.Getenv(ENVBOTTOKEN))
+	client, err := NewClient(config_test.GetConfig(configFilePath).Tokens[0])
 	if err != nil {
 		t.Fatal(err)
 		return
 	}
 
 	t.Run("Send one file", func(t *testing.T) {
-		chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64)
+		chatId := config_test.GetConfig(configFilePath).ChatID
 		data := []byte("data")
 		_, err := client.UploadFileBuffer(chatId, t.Name(), data)
 		if err != nil {
@@ -59,22 +51,22 @@ func Test_UploadFileBuffer(t *testing.T) {
 }
 
 func Test_DownloadFileReader(t *testing.T) {
-	client, err := NewClient(os.Getenv(ENVBOTTOKEN))
+	client, err := NewClient(config_test.GetConfig(configFilePath).Tokens[0])
 	if err != nil {
 		t.Fatal(err)
 		return
 	}
 
 	t.Run("Download one file", func(t *testing.T) {
-		chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64)
+		chatId := config_test.GetConfig(configFilePath).ChatID
 		fileContent := "data"
 		msgIdentifier, err := client.UploadFileReader(chatId, t.Name(), strings.NewReader(fileContent))
 		if err != nil {
 			t.Error(err)
 			return
 		}
 
-		draftChatId, _ := strconv.ParseInt(os.Getenv(ENVDRAFTCHATID), 10, 64)
+		draftChatId := config_test.GetConfig(configFilePath).DraftChatID
 		result, err := client.DownloadFileReader(msgIdentifier, draftChatId)
 		if err != nil {
 			t.Error(err)
@@ -94,7 +86,7 @@ func Test_DownloadFileReader(t *testing.T) {
 	})
 
 	t.Run("Stress Multiple Download", func(t *testing.T) {
-		chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64)
+		chatId := config_test.GetConfig(configFilePath).ChatID
 		fileContent := "data"
 		msgIdentifier, err := client.UploadFileReader(chatId, t.Name(), strings.NewReader(fileContent))
 		if err != nil {
@@ -103,7 +95,7 @@ func Test_DownloadFileReader(t *testing.T) {
 		}
 
 		wg := sync.WaitGroup{}
-		draftChatId, _ := strconv.ParseInt(os.Getenv(ENVDRAFTCHATID), 10, 64)
+		draftChatId := config_test.GetConfig(configFilePath).DraftChatID
 		lock := sync.Mutex{}
 		count := 0
 		total := 5
diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go
diff --git a/pkg/rest-client/client_test.go b/pkg/rest-client/client_test.go
