Merge pull request #1 from PSNAppz/develop

Release v1.0

Author: PSNAppz

.env.sample

@@ -1,7 +1,8 @@
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=123456
-POSTGRES_HOST=localhost
+POSTGRES_USER=fold-elk
+POSTGRES_PASSWORD=password
+POSTGRES_HOST=postgres
 POSTGRES_PORT=5432
-POSTGRES_DB=fold-etl-pipeline
+POSTGRES_DB=fold_elk
 
-ELASTICSEARCH_URL="http://elasticsearch:9200"
+ELASTICSEARCH_URL="http://elasticsearch:9200"
+ELASTIC_VERSION=7.17.9

.gitignore

@@ -14,3 +14,39 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 .env
+
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

.terraform.lock.hcl

@@ -0,0 +1,12 @@
+FROM golang:1.19.5
+
+COPY go.mod go.sum /go/src/github.com/PSNAppz/Fold-ELK/
+WORKDIR /go/src/github.com/PSNAppz/Fold-ELK
+RUN go mod download
+
+COPY . /go/src/github.com/PSNAppz/Fold-ELK
+RUN go build -o /usr/bin/fold-elk github.com/PSNAppz/Fold-ELK/cmd/api
+
+EXPOSE 8080 8080
+ENTRYPOINT ["/usr/bin/fold-elk"]
+

Dockerfile

@@ -1,2 +1,41 @@
 # Fold-ELK
 
+[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/12298080-b2a522f0-5323-4aa1-b803-c73160cde976?action=collection%2Ffork&collection-url=entityId%3D12298080-b2a522f0-5323-4aa1-b803-c73160cde976%26entityType%3Dcollection%26workspaceId%3D42654ab9-e148-4e67-b4b3-90fd805dfb7f)
+
+The postman collection contains all the API endpoints along with examples.
+### The objectives of this repo are as follows:
+
+- To build a data pipeline that syncs and transforms the data stored in a PostgresSQL Database to Elasticsearch, making the data searchable in a fuzzy and relational manner.
+- To build a REST API service that will query Elasticsearch.
+
+### Project brief
+
+Fold-ELK has a set of configuration files for the ELK stack (Elasticsearch, Logstash, Kibana). You can follow the below steps to get the ELK stack up and running.
+CRUD APIs are built using go-gin and the data is stored in PostgresSQL. The data is then synced to Elasticsearch using Logstash. The APIs are then used to query Elasticsearch.
+Data is seeded using faker, which is used to populate the PostgresSQL database.
+
+## Getting Started
+
+### Prerequisites
+
+- Docker
+- Docker Compose
+- Go
+
+### Setting up the docker environment
+
+- Clone the repo
+- Run `mv .env.example .env` to create the environment file
+- Run `docker-compose up --build` to build and bootup the ELK stack
+- Once the ELK stack is up and running, you can use the below 
+- Kibana: http://localhost:5601
+- Elasticsearch: http://localhost:9200
+- API: http://localhost:8080
+
+### Running the API locally
+
+The CRUD API can be run locally using the below steps:
+
+- Make sure the ELK stack is up and running using the above steps
+- Run `go build -o bin/application cmd/api/main.go` to build the API binary
+- Run `./bin/application` to run the API and start listening on port 8080

README.md

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"os"
+	"strconv"
+
+	"github.com/elastic/go-elasticsearch/v8"
+
+	"github.com/PSNAppz/Fold-ELK/db"
+	"github.com/PSNAppz/Fold-ELK/handler"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog"
+)
+
+func main() {
+	var dbPort int
+	var err error
+	logger := zerolog.New(os.Stderr).With().Timestamp().Logger()
+
+	port := os.Getenv("POSTGRES_PORT")
+	if dbPort, err = strconv.Atoi(port); err != nil {
+		logger.Err(err).Msg("failed to parse database port")
+		os.Exit(1)
+	}
+	dbConfig := db.Config{
+		Host:     os.Getenv("POSTGRES_HOST"),
+		Port:     dbPort,
+		Username: os.Getenv("POSTGRES_USER"),
+		Password: os.Getenv("POSTGRES_PASSWORD"),
+		DbName:   os.Getenv("POSTGRES_DB"),
+		Logger:   logger,
+	}
+	logger.Info().Interface("config", &dbConfig).Msg("config:")
+	dbInstance, err := db.Init(dbConfig)
+	if err != nil {
+		logger.Err(err).Msg("Connection failed")
+		os.Exit(1)
+	}
+	logger.Info().Msg("Database connection established")
+
+	esClient, err := elasticsearch.NewDefaultClient()
+	if err != nil {
+		logger.Err(err).Msg("Connection failed")
+		os.Exit(1)
+	}
+
+	h := handler.New(dbInstance, esClient, logger)
+	router := gin.Default()
+	rg := router.Group("/v1")
+	h.Register(rg)
+	router.Run(":8080")
+}

bin/application

@@ -0,0 +1,48 @@
+package db
+
+import (
+	"database/sql"
+	"fmt"
+
+	_ "github.com/lib/pq"
+	"github.com/rs/zerolog"
+)
+
+var (
+	ErrNoRecord = fmt.Errorf("no matching record found")
+	insertOp    = "insert"
+	deleteOp    = "delete"
+	updateOp    = "update"
+)
+
+type Database struct {
+	Conn   *sql.DB
+	Logger zerolog.Logger
+}
+
+type Config struct {
+	Host     string
+	Port     int
+	Username string
+	Password string
+	DbName   string
+	Logger   zerolog.Logger
+}
+
+func Init(cfg Config) (Database, error) {
+	db := Database{}
+	dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
+		cfg.Host, cfg.Port, cfg.Username, cfg.Password, cfg.DbName)
+	conn, err := sql.Open("postgres", dsn)
+	if err != nil {
+		return db, err
+	}
+
+	db.Conn = conn
+	db.Logger = cfg.Logger
+	err = db.Conn.Ping()
+	if err != nil {
+		return db, err
+	}
+	return db, nil
+}