Add SSL support for S3 compatible Minio storage server (#27)

Author: Md. Emruz Hossain

README.md

@@ -6,7 +6,7 @@
 [![Twitter](https://img.shields.io/twitter/follow/appscodehq.svg?style=social&logo=twitter&label=Follow)](https://twitter.com/intent/follow?screen_name=AppsCodeHQ)
 
 # osm
-Object Store Manipulator (osm: pronounced like `awesome`) - `curl` for cloud storage services. 🙌 `osm` can create & delete buckets and upload, download & delete files from buckets for AWS S3, DigitalOcean Spaces, Google Cloud Storage, Microsoft Azure storage and OpenStack Swift. Its single binary can be easily packaged instead of official python based clis inside Docker images.
+Object Store Manipulator (osm: pronounced like `awesome`) - `curl` for cloud storage services. 🙌 `osm` can create & delete buckets and upload, download & delete files from buckets for AWS S3, AWS S3 compatible other storage services(i.e. Minio), DigitalOcean Spaces, Google Cloud Storage, Microsoft Azure storage and OpenStack Swift. Its single binary can be easily packaged instead of official python based clis inside Docker images.
 
 ## Install OSM
 You can download and install a pre-built binary:
@@ -43,10 +43,10 @@ Available Commands:
   config      OSM configuration
   help        Help about any command
   lc          List containers
-  ls          List container
+  ls          List items in a container
   mc          Make container
   pull        Pull item from container
-  push        Push item from container
+  push        Push item to container
   rc          Remove container
   rm          Remove item from container
   stat        Stat item from container
@@ -59,10 +59,13 @@ Flags:
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
-      --osmconfig string                 Path to osm config (default "/home/tamal/.osm/config")
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
+
+Use "osm [command] --help" for more information about a command.
+
 ```
 
 ### OSM Configuration
@@ -72,6 +75,9 @@ This allows providing commands one time for multiple subsequent operations with
 # AWS S3:
 osm config set-context osm-s3 --provider=s3 --s3.access_key_id=<key_id> --s3.secret_key=<secret_key> --s3.region=us-east-1
 
+# TLS secure Minio server
+osm config set-context osm-minio --provider=s3 --s3.access_key_id=<minio_access_key> --s3.secret_key=<minio_secret_key> --s3.endpoint=<minio_server_address> --s3.cacert_file=<root_ca_file_path>
+
 # DigitalOcean Spaces:
 osm config set-context osm-do --provider=s3 --s3.access_key_id=<key_id> --s3.secret_key=<secret_key> --s3.endpoint=nyc3.digitaloceanspaces.com
 

cmds/config/set.go

@@ -25,6 +25,7 @@ type setContextRequest struct {
 	s3ConfigRegion           string
 	s3ConfigSecretKey        string
 	s3ConfigDisableSSL       bool
+	s3CACertFile             string
 	gcsConfigJSONKeyPath     string
 	gcsConfigProjectId       string
 	gcsConfigScopes          string
@@ -73,6 +74,7 @@ func newCmdSet() *cobra.Command {
 	setCmd.Flags().StringVar(&req.s3ConfigSecretKey, s3.Kind+"."+s3.ConfigSecretKey, "", "S3 config secret key")
 	setCmd.Flags().StringVar(&req.s3ConfigAuthType, s3.Kind+"."+s3.ConfigAuthType, "accesskey", "S3 config auth type (accesskey, iam)")
 	setCmd.Flags().BoolVar(&req.s3ConfigDisableSSL, s3.Kind+"."+s3.ConfigDisableSSL, false, "S3 config disable SSL")
+	setCmd.Flags().StringVar(&req.s3CACertFile, s3.Kind+"."+s3.ConfigCACertFile, "", "S3 config cacert_file for custom endpoint(i.e minio)")
 
 	setCmd.Flags().StringVar(&req.gcsConfigJSONKeyPath, gcs.Kind+".json_key_path", "", "GCS config json key path")
 	setCmd.Flags().StringVar(&req.gcsConfigProjectId, gcs.Kind+"."+gcs.ConfigProjectId, "", "GCS config project id")
@@ -122,6 +124,9 @@ func setContext(req *setContextRequest, configPath string) {
 		if req.s3ConfigAuthType != "" {
 			nc.Config[s3.ConfigAuthType] = req.s3ConfigAuthType
 		}
+		if req.s3CACertFile != "" {
+			nc.Config[s3.ConfigCACertFile] = req.s3CACertFile
+		}
 		nc.Config[s3.ConfigDisableSSL] = strconv.FormatBool(req.s3ConfigDisableSSL)
 	case gcs.Kind:
 		nc.Provider = gcs.Kind

cmds/ls.go

@@ -21,7 +21,7 @@ func NewCmdListIetms() *cobra.Command {
 	req := &itemListRequest{}
 	cmd := &cobra.Command{
 		Use:               "ls <name>",
-		Short:             "List container",
+		Short:             "List items in a container",
 		Example:           "osm ls mybucket",
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {

cmds/push.go

@@ -21,7 +21,7 @@ func NewCmdPush() *cobra.Command {
 	req := &itemPushRequest{}
 	cmd := &cobra.Command{
 		Use:               "push <src> <dest>",
-		Short:             "Push item from container",
+		Short:             "Push item to container",
 		Example:           "osm push -c mybucket f1.txt /tmp/f1.txt",
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {

docs/README.md

@@ -1,28 +1,28 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/appscode/osm)](https://goreportcard.com/report/github.com/appscode/osm)
 
 # osm
-Object Store Manipulator (osm: pronounced like `awesome`) - `curl` for cloud storage services. 🙌 `osm` can create & delete buckets and upload, download & delete files from buckets for AWS S3, Google Cloud Storage, Microsoft Azure storage and OpenStack Swift. Its single binary can be easily packaged instead of official python based clis inside Docker images.
+Object Store Manipulator (osm: pronounced like `awesome`) - `curl` for cloud storage services. 🙌 `osm` can create & delete buckets and upload, download & delete files from buckets for AWS S3, AWS S3 compatible other storage services(i.e. Minio), DigitalOcean Spaces, Google Cloud Storage, Microsoft Azure storage and OpenStack Swift. Its single binary can be easily packaged instead of official python based clis inside Docker images.
 
 ## Install OSM
 You can download and install a pre-built binary:
 ```console
 # Linux amd 64-bit:
-wget -O osm https://cdn.appscode.com/binaries/osm/0.6.1/osm-linux-amd64 \
+wget -O osm https://cdn.appscode.com/binaries/osm/0.6.3/osm-linux-amd64 \
   && chmod +x osm \
   && sudo mv osm /usr/local/bin/
 
 # Linux 386 32-bit:
-wget -O osm https://cdn.appscode.com/binaries/osm/0.6.1/osm-linux-386 \
+wget -O osm https://cdn.appscode.com/binaries/osm/0.6.3/osm-linux-386 \
   && chmod +x osm \
   && sudo mv osm /usr/local/bin/
 
 # Mac 64-bit
-wget -O osm https://cdn.appscode.com/binaries/osm/0.6.1/osm-darwin-amd64 \
+wget -O osm https://cdn.appscode.com/binaries/osm/0.6.3/osm-darwin-amd64 \
   && chmod +x osm \
   && sudo mv osm /usr/local/bin/
 
 # Mac 32-bit
-wget -O osm https://cdn.appscode.com/binaries/osm/0.6.1/osm-darwin-386 \
+wget -O osm https://cdn.appscode.com/binaries/osm/0.6.3/osm-darwin-386 \
   && chmod +x osm \
   && sudo mv osm /usr/local/bin/
 ```
@@ -37,20 +37,24 @@ osm [command]
 Available Commands:
   config      OSM configuration
   help        Help about any command
-  ls          List container
+  lc          List containers
+  ls          List items in a container
   mc          Make container
   pull        Pull item from container
-  push        Push item from container
+  push        Push item to container
   rc          Remove container
   rm          Remove item from container
   stat        Stat item from container
   version     Prints binary version number.
 
 Flags:
       --alsologtostderr                  log to standard error as well as files
+      --enable-analytics                 Send usage events to Google Analytics (default true)
+  -h, --help                             help for osm
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
@@ -66,6 +70,12 @@ This allows providing commands one time for multiple subsequent operations with
 # AWS S3:
 osm config set-context osm-s3 --provider=s3 --s3.access_key_id=<key_id> --s3.secret_key=<secret_key> --s3.region=us-east-1
 
+# TLS secure Minio server
+osm config set-context osm-minio --provider=s3 --s3.access_key_id=<minio_access_key> --s3.secret_key=<minio_secret_key> --s3.endpoint=<minio_server_address> --s3.cacert_file=<root_ca_file_path>
+
+# DigitalOcean Spaces:
+osm config set-context osm-do --provider=s3 --s3.access_key_id=<key_id> --s3.secret_key=<secret_key> --s3.endpoint=nyc3.digitaloceanspaces.com
+
 # Google Cloud Storage:
 osm config set-context osm-gs --provider=google --google.json_key_path=<path_sa_file> --google.project_id=<my_project>
 

docs/reference/osm.md

@@ -35,7 +35,7 @@ osm [command] [flags]
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
-      --osmconfig string                 Path to osm config (default "/home/tamal/.osm/config")
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
@@ -45,10 +45,10 @@ osm [command] [flags]
 
 * [osm config](/docs/reference/osm_config.md)	 - OSM configuration
 * [osm lc](/docs/reference/osm_lc.md)	 - List containers
-* [osm ls](/docs/reference/osm_ls.md)	 - List container
+* [osm ls](/docs/reference/osm_ls.md)	 - List items in a container
 * [osm mc](/docs/reference/osm_mc.md)	 - Make container
 * [osm pull](/docs/reference/osm_pull.md)	 - Pull item from container
-* [osm push](/docs/reference/osm_push.md)	 - Push item from container
+* [osm push](/docs/reference/osm_push.md)	 - Push item to container
 * [osm rc](/docs/reference/osm_rc.md)	 - Remove container
 * [osm rm](/docs/reference/osm_rm.md)	 - Remove item from container
 * [osm stat](/docs/reference/osm_stat.md)	 - Stat item from container

docs/reference/osm_config.md

@@ -41,7 +41,7 @@ osm config view
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
-      --osmconfig string                 Path to osm config (default "/home/tamal/.osm/config")
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging

docs/reference/osm_config_current-context.md

@@ -41,7 +41,7 @@ osm config current-context
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
-      --osmconfig string                 Path to osm config (default "/home/tamal/.osm/config")
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging

docs/reference/osm_config_get-contexts.md

@@ -41,7 +41,7 @@ osm config get-contexts
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
-      --osmconfig string                 Path to osm config (default "/home/tamal/.osm/config")
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging

docs/reference/osm_config_set-context.md

@@ -40,6 +40,7 @@ osm config set-context <name>
       --provider string                Cloud storage provider
       --s3.access_key_id string        S3 config access key id
       --s3.auth_type string            S3 config auth type (accesskey, iam) (default "accesskey")
+      --s3.cacert_file string          S3 config cacert_file for custom endpoint(i.e minio)
       --s3.disable_ssl                 S3 config disable SSL
       --s3.endpoint string             S3 config endpoint
       --s3.region string               S3 config region
@@ -65,7 +66,7 @@ osm config set-context <name>
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
-      --osmconfig string                 Path to osm config (default "/home/tamal/.osm/config")
+      --osmconfig string                 Path to osm config (default "$HOME/.osm/config")
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
   -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging