Support latest Erlang/OTP and Refactor

.gitignore

@@ -1,24 +1,43 @@
+# rebar3 build directories
 _build/
+apps/*/_build/
+
+# Legacy rebar2 deps (no longer used)
 deps/
-rel/leo_manager/leo_manager/
-rel/leo_manager/service/
-rel/leo_gateway/leo_gateway/
-rel/leo_storage/leo_storage/
+
+# Release output
 package/
 
+# Generated config files
 rel/leo_manager/files/leo_manager.conf
 rel/leo_manager/files/leo_manager.schema
+apps/leo_manager/priv/leo_manager.conf
+apps/leo_manager/priv/leo_manager.schema
 
-# ignore this because it is env dependent
-rel/common/launch.environment
-
-# reltool.config files are generated from reltool.config.in
+# Legacy reltool output (no longer used with rebar3)
+rel/leo_manager/leo_manager/
+rel/leo_manager/service/
+rel/leo_gateway/leo_gateway/
+rel/leo_storage/leo_storage/
 rel/leo_gateway/reltool.config
 rel/leo_manager/reltool.config
 rel/leo_storage/reltool.config
 
+# Environment specific
+rel/common/launch.environment
+
+# App specific
 apps/leo_gateway/cache/
 
+# Documentation
 docs/_book/
 docs/node_modules/
 site/
+
+# Editor and OS files
+*.swp
+*.swo
+*~
+.DS_Store
+*.beam
+erl_crash.dump

.travis.yml

@@ -3,10 +3,7 @@ script: "make test"
 notifications:
   email: false
 otp_release:
-  - 19.3
-  - 20.3
-  - 21.3
-  - 22.2
+  - 28.0
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -y check

Makefile

@@ -3,6 +3,7 @@
 # LeoFS
 #
 # Copyright (c) 2012-2018 Rakuten, Inc.
+# Copyright (c) 2019-2025 Lions Data, Ltd.
 #
 # This file is provided to you under the Apache License,
 # Version 2.0 (the "License"); you may not use this file
@@ -19,79 +20,78 @@
 # under the License.
 #
 #======================================================================
-.PHONY: all compile deps clean distclean test generate release replace_launch_env release_for_test pkgsrc
+.PHONY: all compile clean distclean test release replace_launch_env release_for_test pkgsrc
 
-ifneq ($(with_sd_notify),yes)
-with_sd_notify := no
-endif
+all: compile
 
-all: deps compile
 compile:
-	@./rebar compile
-deps:
-	@./rebar get-deps
+	(cd apps/leo_manager && rebar3 compile)
+	(cd apps/leo_storage && rebar3 compile)
+	(cd apps/leo_gateway && rebar3 compile)
+
 clean:
-	@./rebar clean
+	(cd apps/leo_manager && rebar3 clean)
+	(cd apps/leo_storage && rebar3 clean)
+	(cd apps/leo_gateway && rebar3 clean)
 	make -C pkg clean
+
 distclean:
-	@./rebar delete-deps
-	@./rebar clean
-	rm -f rel/leo_*/reltool.config
+	(cd apps/leo_manager && rm -rf _build)
+	(cd apps/leo_storage && rm -rf _build)
+	(cd apps/leo_gateway && rm -rf _build)
+	rm -rf _build
+
 test:
-	(cd apps/leo_manager && make)
-	(cd apps/leo_gateway && make)
-	(cd apps/leo_storage && make)
-generate:
-	rm -rf rel/leo_manager/leo_manager/
-	rm -rf rel/leo_storage/leo_storage/
-	rm -rf rel/leo_gateway/leo_gateway/
-	@./rebar compile
-	(cd rel/leo_manager && ../../rebar generate)
-	(cd rel/leo_storage && ../../rebar generate)
-	(cd rel/leo_gateway && ../../rebar generate)
-sd_notify:
-	make -C deps/sd_notify
-reltool:
-	for reltool_config in rel/leo_*/reltool.config.in; do \
-		./make_reltool.sh $(with_sd_notify) $$reltool_config > $$(echo $$reltool_config | sed s/.in$$//); \
-	done
-release: reltool
-	@./rebar compile
+	(cd apps/leo_manager && rebar3 eunit)
+	(cd apps/leo_storage && rebar3 eunit)
+	(cd apps/leo_gateway && rebar3 eunit)
+
+dialyzer:
+	(cd apps/leo_manager && rebar3 dialyzer)
+	(cd apps/leo_storage && rebar3 dialyzer)
+	(cd apps/leo_gateway && rebar3 dialyzer)
+
+xref:
+	(cd apps/leo_manager && rebar3 xref)
+	(cd apps/leo_storage && rebar3 xref)
+	(cd apps/leo_gateway && rebar3 xref)
+
+release:
 	rm -rf package/leo_*
 	#
 	# manager-master
 	#
-	rm -rf rel/leo_manager/leo_manager/
+	(cd apps/leo_manager && \
+		cp priv/leo_manager_0.conf priv/leo_manager.conf && \
+		cp priv/leo_manager_0.schema priv/leo_manager.schema && \
+		rebar3 release -n leo_manager)
 	mkdir -p package/leo_manager_0
-	cp apps/leo_manager/priv/leo_manager_0.conf rel/leo_manager/files/leo_manager.conf
-	cp apps/leo_manager/priv/leo_manager_0.schema rel/leo_manager/files/leo_manager.schema
-	(cd rel/leo_manager && ../../rebar generate)
-	cp -r rel/leo_manager/leo_manager/* package/leo_manager_0/
+	cp -r apps/leo_manager/_build/default/rel/leo_manager/* package/leo_manager_0/
 	#
 	# manager-slave
 	#
-	rm -rf rel/leo_manager/leo_manager/
+	(cd apps/leo_manager && \
+		rm -rf _build/default/rel/leo_manager && \
+		cp priv/leo_manager_1.conf priv/leo_manager.conf && \
+		cp priv/leo_manager_1.schema priv/leo_manager.schema && \
+		rebar3 release -n leo_manager)
 	mkdir -p package/leo_manager_1
-	cp apps/leo_manager/priv/leo_manager_1.conf rel/leo_manager/files/leo_manager.conf
-	cp apps/leo_manager/priv/leo_manager_1.schema rel/leo_manager/files/leo_manager.schema
-	(cd rel/leo_manager && ../../rebar generate)
-	cp -r rel/leo_manager/leo_manager/* package/leo_manager_1/
+	cp -r apps/leo_manager/_build/default/rel/leo_manager/* package/leo_manager_1/
 	#
 	# storage
 	#
-	rm -rf rel/leo_storage/leo_storage/
+	(cd apps/leo_storage && rebar3 release -n leo_storage)
 	mkdir -p package/leo_storage
-	(cd rel/leo_storage && ../../rebar generate)
-	cp -r rel/leo_storage/leo_storage/* package/leo_storage/
+	cp -r apps/leo_storage/_build/default/rel/leo_storage/* package/leo_storage/
 	#
 	# gateway
 	#
-	rm -rf rel/leo_gateway/leo_gateway/
+	(cd apps/leo_gateway && rebar3 release -n leo_gateway)
 	mkdir -p package/leo_gateway
-	(cd rel/leo_gateway && ../../rebar generate)
-	cp -r rel/leo_gateway/leo_gateway/* package/leo_gateway/
+	cp -r apps/leo_gateway/_build/default/rel/leo_gateway/* package/leo_gateway/
 	cp README.md package/
 	cp leofs-adm package/
+
 replace_launch_env:
 	( echo '# Directories used by launch script can be re-defined here, if needed'; \
 	  echo '# Default values will be picked for commented or empty parameters'; \
@@ -109,11 +109,14 @@ replace_launch_env:
 	  echo 'RUNNER_USER=${USER}'; \
 	) > rel/common/launch.environment
 	sudo rm -rf /tmp/home
+
 release_for_test: replace_launch_env release
 
 pkgsrc: release
 	make -C pkg
+
 build_doc:
 	rm -rf site/ && mkdocs build
+
 serve_doc: build_doc
 	mkdocs serve --dev-addr 0.0.0.0:8000

apps/leo_gateway/.gitignore

@@ -14,3 +14,5 @@ erl_crash.dump
 leo_gateway.dot
 .rebar/
 rel/leo_gateway/
+rebar3.crashdump
+build/

apps/leo_gateway/Makefile

@@ -1,55 +1,54 @@
-.PHONY: all get_deps compile xref eunit release check_plt build_plt dialyzer doc callgraph graphviz clean distclean gen_rpc gen_nfs
-
-REBAR := ./rebar
-APPS = erts kernel stdlib sasl crypto compiler inets mnesia public_key runtime_tools snmp syntax_tools tools xmerl webtool
-LIBS = deps/leo_commons/ebin deps/leo_logger/ebin deps/leo_object_storage/ebin \
-       deps/leo_redundant_manager/ebin deps/leo_mq/ebin deps/leo_statistics/ebin deps/leo_rpc/ebin \
-       deps/leo_s3_libs/ebin deps/leo_cache/ebin deps/leo_dcerl/ebin deps/leo_mcerl/ebin \
-       deps/savanna_commons/ebin deps/savanna_agent/ebin deps/erpcgen/ebin
-PLT_FILE = .leo_gateway_dialyzer_plt
-COMMON_PLT_FILE = .common_dialyzer_plt
-DOT_FILE = leo_gateway.dot
-CALL_GRAPH_FILE = leo_gateway.png
-
-all: get_deps gen_nfs compile xref eunit
-get_deps:
+.PHONY: all compile xref eunit dialyzer clean distclean doc gen_rpc gen_nfs get-deps patch-deps
+
+REBAR := rebar3
+
+all: compile xref eunit
+
+get-deps:
 	@$(REBAR) get-deps
-compile:
+
+patch-deps:
+	@./scripts/patch_deps.sh
+
+compile: get-deps patch-deps
 	@$(REBAR) compile
+
 xref:
-	@$(REBAR) xref skip_deps=true
+	@$(REBAR) xref
+
 eunit:
-	@$(REBAR) eunit skip_deps=true
-release:
-	rm -rf rel/leo_gateway
-	@$(REBAR) compile
-	(cd rel/ && ../rebar generate)
-check_plt:
-	@$(REBAR) compile
-	dialyzer --check_plt --plts $(PLT_FILE) $(COMMON_PLT_FILE) --apps $(APPS)
-build_plt:
-	@$(REBAR) compile
-	dialyzer --build_plt --output_plt $(PLT_FILE) --apps $(LIBS)
-build_plt_common:
-	@$(REBAR) compile
-	dialyzer --build_plt --output_plt $(COMMON_PLT_FILE) --apps $(APPS)
+	@$(REBAR) eunit
+
 dialyzer:
-	@$(REBAR) compile
-	dialyzer --plts $(PLT_FILE) $(COMMON_PLT_FILE) -r ebin/ --dump_callgraph $(DOT_FILE) -Wrace_conditions | fgrep -v -f ./dialyzer.ignore-warnings
-doc: compile
-	@$(REBAR) doc
-callgraph: graphviz
-	dot -Tpng -o$(CALL_GRAPH_FILE) $(DOT_FILE)
-graphviz:
-	$(if $(shell which dot),,$(error "To make the depgraph, you need graphviz installed"))
+	@$(REBAR) dialyzer
+
+doc:
+	@$(REBAR) ex_doc
+
 clean:
 	@$(REBAR) clean
+
 distclean:
-	@$(REBAR) delete-deps
-	@$(REBAR) clean
-gen_rpc: deps
-	(cd deps/erpcgen/;make)
+	@$(REBAR) clean -a
+	@rm -rf _build
+
+gen_rpc:
+	@$(REBAR) compile
+	@cd _build/default/lib/erpcgen && $(MAKE)
+
 gen_nfs: gen_rpc
-	./deps/erpcgen/priv/erpcgen -a [svc_callback,xdr,hrl] src/leo_nfs_proto3.x
-	./deps/erpcgen/priv/erpcgen -a [svc_callback,xdr,hrl] src/leo_nfs_mount3.x
-	./deps/erpcgen/priv/erpcgen -a [svc_callback,xdr,hrl] src/leo_nlm_proto4.x
+	./_build/default/lib/erpcgen/priv/erpcgen -a [svc_callback,xdr,hrl] src/leo_nfs_proto3.x
+	./_build/default/lib/erpcgen/priv/erpcgen -a [svc_callback,xdr,hrl] src/leo_nfs_mount3.x
+	./_build/default/lib/erpcgen/priv/erpcgen -a [svc_callback,xdr,hrl] src/leo_nlm_proto4.x
+
+shell:
+	@$(REBAR) shell
+
+test:
+	@$(REBAR) ct
+
+fmt:
+	@$(REBAR) fmt
+
+check-fmt:
+	@$(REBAR) fmt --check

apps/leo_gateway/README.md

@@ -1,31 +1,43 @@
 # leo_gateway
-
-[![Build Status](https://secure.travis-ci.org/leo-project/leo_gateway.png?branch=develop)](http://travis-ci.org/leo-project/leo_gateway)
-
 ## Overview
 
 * **leo_gateway**, LeoGateway is one of the core components of [LeoFS](https://github.com/leo-project/leofs). Main roles are described below.
-  * LeoFS's Gateway use [Cowboy](https://github.com/extend/cowboy) as Erlang's HTTP server.
-  * Able to speak [Amazon S3 compatible REST API](http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/Welcome.html?r=5754).
+  * LeoFS's Gateway uses [Cowboy 2.x](https://github.com/ninenines/cowboy) as Erlang's HTTP server.
   * Implemented a subset of Caching in HTTP(RFC2616).
 
 ## Architecture
 
-LeoFS Gateway consists of the fast HTTP-Server - [Cowboy](https://github.com/ninenines/cowboy), the API handler and [the cache mecanism](https://github.com/leo-project/leo_cache). It provides the REST-API and Amazon S3-API. You’re able to easily access LeoFS with S3-Clients such as `s3cmd`, `DragonDisk`, program languages - `Erlang`, `Java`, `Ruby`, `Python`, `Go` and so on.
+LeoFS Gateway consists of the fast HTTP-Server - [Cowboy 2.x](https://github.com/ninenines/cowboy), the API handler and [the cache mechanism](https://github.com/leo-project/leo_cache). It provides the REST-API and Amazon S3-API. You're able to easily access LeoFS with S3-Clients such as `s3cmd`, `DragonDisk`, program languages - `Erlang`, `Java`, `Ruby`, `Python`, `Go` and so on.
 
 ![leo_gateway_architecture](https://raw.githubusercontent.com/leo-project/leofs/master/docs/assets/leofs-architecture.002.jpg)
 
 A client requests an object or a bucket operation to LeoFS Gateway then LeoFS Gateway requests the message of operation to a storage-node.
 
 A destination storage node is decided by the routing-table. It is called RING which is generated and provided at LeoFS Manager and which is based on consistent-hashing.
 
-Also, LeoFS Gateway provides built-in support for the object-cache mechanism in order to realize Keeping high performance and reduction of traffic between LeoFS Gateway and [LeoFS Storage](https://github.com/leo-project/leo_storage).
+Also, LeoFS Gateway provides built-in support for the object-cache mechanism in order to realize Keeping high performance and reduction of traffic between LeoFS Gateway and [LeoFS Storage](https://github.com/leo-project/apps/leo_storage).
+
+## Features
+
+### HTTP Range Requests (RFC 7233)
+
+LeoFS Gateway supports HTTP Range requests for partial content retrieval:
+
+* Normal ranges: `bytes=0-499` (first 500 bytes)
+* Open-ended ranges: `bytes=500-` (from byte 500 to end)
+* Suffix ranges: `bytes=-500` (last 500 bytes)
+
+### AWS Signature Version 4 Chunked Upload
+
+LeoFS Gateway supports AWS Signature Version 4 chunked upload with per-chunk signature verification for secure streaming uploads.
+
+## Build
 
 * The detail document is [here](http://leo-project.net/leofs/docs/).
-* `leo_gateway` uses [rebar](https://github.com/rebar/rebar) build system. Makefile so that simply running "make" at the top level should work.
-* `leo_gateway` requires [Erlang R16B03-1 or later](http://www.erlang.org/).
+* `leo_gateway` uses [rebar3](https://github.com/erlang/rebar3) build system. Makefile so that simply running "make" at the top level should work.
+* `leo_gateway` requires Erlang/OTP 24 or later.
 
 ## Sponsors
 
-* LeoProject/LeoFS was sponsored by [Rakuten, Inc.](https://global.rakuten.com/corp/) from 2012 to Dec of 2018.
 * LeoProject/LeoFS is sponsored by [Lions Data, Ltd.](https://lions-data.com/) from Jan of 2019.
+* LeoProject/LeoFS was sponsored by [Rakuten, Inc.](https://global.rakuten.com/corp/) from 2012 to Dec of 2018.

apps/leo_gateway/config/sys.config

@@ -0,0 +1,10 @@
+[
+ %% SASL config
+ {sasl, [
+         {sasl_error_logger, {file, "log/sasl-error.log"}},
+         {errlog_type, error},
+         {error_logger_mf_dir, "log/sasl"},      % Log directory
+         {error_logger_mf_maxbytes, 10485760},   % 10 MB max file size
+         {error_logger_mf_maxfiles, 5}           % 5 files max
+        ]}
+].