Modularization

.gitlab-ci.yml

@@ -0,0 +1,110 @@
+image: 192.168.178.73:6088/ceph-dovecot-combined:fe6a06c8907bd7635018205c7eae05dc4c363575
+
+services:
+  - docker:19.03.13-dind
+
+variables:
+  DOCKER_REGISTRY_URL: "192.168.178.73:6088"
+  LC_ALL: "en_US.UTF-8"
+  LANG: "en_US.UTF-8"
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: ""
+  PROJECT_NAME: dovecot-ceph-plugin
+  GIT_SUBMODULE_STRATEGY: recursive
+  ROOT_PATH: "apps/develop/apps/"
+
+stages:
+  - build
+  - docker
+  - publish
+
+build:
+  stage: build
+  tags: ["kubernetes"]
+  artifacts:
+   paths:
+    - imaptest.log
+    - dovecot.log
+  script:
+    - /opt/ceph-container/bin/entrypoint.sh demo &
+    - ./autogen.sh
+    - ./configure --with-dovecot=/usr/local/lib/dovecot --enable-maintainer-mode --enable-debug --with-integration-tests --enable-valgrind --enable-debug
+    - make clean install
+    - ps -aux
+    - ceph df
+    - ./setup.sh
+    - dovecot 
+    - ldconfig
+    - smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f [email protected] -t t1 inet:127.0.0.1:1024 
+    - imaptest user=t%d pass=t port=10143 users=10 clients=10 error_quit secs=30 output=./imaptest.log
+    - doveadm -Dv backup -u t1 -m INBOX mdbox:/usr/local/var/mail/mdbox/t1
+    - doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails
+    - doveadm -D -c /usr/local/etc/dovecot_mdbox/dovecot.conf fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/mdbox.t1.mails
+
+
+    - src/scripts/sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted
+    - src/scripts/sort.sh /root/mdbox.t1.mails /root/mdbox.t1.mails.sorted
+    - type diff
+    - diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted
+    - echo $?
+
+    - rm -rf /usr/local/var/mail/rbox/t1
+    - doveadm -Dv -c /usr/local/etc/dovecot_mdbox/dovecot.conf backup -u t1 -m INBOX rbox:/usr/local/var/mail/rbox/t1
+    - doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails
+    - src/scripts/sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted
+    - diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted
+    - echo $?
+    - rm -rf /usr/local/var/mail/rbox
+    - imaptest user=t%d pass=t port=10143 error_quit secs=15 copybox=INBOX.Drafts output=./imaptest.log
+    - doveadm -D force-resync -u t1 INBOX
+
+  #  - git grep -c "Error" -- ./dovecot.log
+  #  - git grep -c "failed" -- ./dovecot.log
+  #  - git grep -c "Internal error" -- ./dovecot.log
+  #  - git grep -c "killed" -- ./dovecot.log
+  #  - git grep -c "Panic" -- ./dovecot.log
+  #  - git grep -c "Fatal" -- ./dovecot.log
+  #  - git grep -c "BUG" -- ./dovecot.log
+  #  - ceph df
+  #  - make check  
+build_image:
+  tags:
+    - kubernetes
+  stage: docker
+  image:
+    name: gcr.io/kaniko-project/executor:v1.14.0-debug
+    entrypoint: [""]
+  before_script:
+    - |
+      echo "$NEXUS_IMAGE_REGISTRY_SSH_KEY" >> /kaniko/ssl/certs/ca-certificates.crt
+  script:
+    - /kaniko/executor
+      --context "$CI_PROJECT_DIR"
+      --dockerfile "${CI_PROJECT_DIR}/Dockerfile"
+      --destination "$DOCKER_REGISTRY_URL/$PROJECT_NAME:$CI_COMMIT_SHA"
+      --skip-tls-verify
+
+publish_to_master:
+  dependencies: 
+    - build_image
+  stage: publish
+  tags: 
+    - shell 
+  before_script:
+    - mkdir dev_temp
+    - cd dev_temp
+    - git clone http://${CI_USERNAME}:${CI_PUSH_TOKEN}@${CI_SERVER_HOST}/devops/dev.git
+    - git config --global user.email "[email protected]"
+    - git config --global user.name "GitLab CI/CD"
+  script:
+    - cd dev
+    - git checkout develop
+    - git stash
+    - cd $ROOT_PATH
+    - cd $PROJECT_NAME
+    - export image2="yq e -i '.spec.template.spec.containers[0].image = \""$DOCKER_REGISTRY_URL/$PROJECT_NAME:$CI_COMMIT_SHA\""' deployment.yaml"
+    - eval "$image2"
+    - git commit -am 'kubernetes-deployment.yaml has been updated with new image'
+    - git push origin develop
+    - cd ../../../../../..
+    - rm -r dev_temp

.travis.yml

@@ -23,43 +23,43 @@ script:
 - docker exec ceph_demo sh -c 'cp -r /etc/ceph/* /root/cfg'
 - docker exec build sh -c 'chmod 777 /etc/ceph/*'
 - docker exec ceph_demo sh -c 'ceph tell mon.\* injectargs "--mon-allow-pool-delete=true"'
-- travis_wait 30 docker exec build sh -c 'cd repo/src/tests; make check-valgrind'
-- docker exec build sh -c 'cd repo; cat src/tests/test-suite-memcheck.log'
+#- travis_wait 30 docker exec build sh -c 'cd repo/src/tests; make check-valgrind'
+#- docker exec build sh -c 'cd repo; cat src/tests/test-suite-memcheck.log'
 #- travis_wait 30 docker exec build sh -c 'cd repo/src/tests; make check'
 - docker exec build sh -c 'ldconfig'
 - docker exec build sh -c 'chmod -R 777 /usr/local/var/'
 - docker exec --detach build sh -c 'service dovecot.service start'
-- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 users=100 clients=25 error_quit secs=15 output=/var/log/imaptest.log'
-- docker exec build sh -c 'smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f [email protected] -t t1 inet:127.0.0.1:1024'
-- docker exec build sh -c 'doveadm -D altmove -u t2 ALL'
-- docker exec build sh -c 'doveadm -D altmove -r -u t2 ALL'
+#- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 users=100 clients=25 error_quit secs=15 output=/var/log/imaptest.log'
+#- docker exec build sh -c 'smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f [email protected] -t t1 inet:127.0.0.1:1024'
+#- docker exec build sh -c 'doveadm -D altmove -u t2 ALL'
+#- docker exec build sh -c 'doveadm -D altmove -r -u t2 ALL'
 
-- docker exec build sh -c 'doveadm -Dv backup -u t1 -m INBOX mdbox:/usr/local/var/mail/mdbox/t1'
-- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails'
-- docker exec build sh -c 'doveadm -D -c /usr/local/etc/dovecot_mdbox/dovecot.conf fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/mdbox.t1.mails'
-- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted'
-- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/mdbox.t1.mails /root/mdbox.t1.mails.sorted'
+#- docker exec build sh -c 'doveadm -Dv backup -u t1 -m INBOX mdbox:/usr/local/var/mail/mdbox/t1'
+#- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails'
+#- docker exec build sh -c 'doveadm -D -c /usr/local/etc/dovecot_mdbox/dovecot.conf fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/mdbox.t1.mails'
+#- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted'
+#- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/mdbox.t1.mails /root/mdbox.t1.mails.sorted'
 
-- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted'
-- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox/t1'
-- docker exec build sh -c 'doveadm -Dv -c /usr/local/etc/dovecot_mdbox/dovecot.conf backup -u t1 -m INBOX rbox:/usr/local/var/mail/rbox/t1'
-- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails'
-- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted'
-- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted'
+#- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted'
+#- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox/t1'
+#- docker exec build sh -c 'doveadm -Dv -c /usr/local/etc/dovecot_mdbox/dovecot.conf backup -u t1 -m INBOX rbox:/usr/local/var/mail/rbox/t1'
+#- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails'
+#- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted'
+#- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted'
 
-- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
-- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10110 profile=/root/pop3-profile.conf users=100 clients=10 error_quit secs=15 output=/var/log/imaptest.log'
+#- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
+#- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10110 profile=/root/pop3-profile.conf users=100 clients=10 error_quit secs=15 output=/var/log/imaptest.log'
 
-- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
-- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 error_quit secs=15 copybox=INBOX.Drafts output=/var/log/imaptest.log'
-- docker exec build sh -c 'doveadm -D force-resync -u t1 INBOX'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Error:\""'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"failed:\""'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Internal error\""'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"killed\""'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Panic:\""'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Fatal:\""'
-- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/imaptest.log | grep \"BUG:\""'
+#- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
+# #- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 error_quit secs=15 copybox=INBOX.Drafts output=/var/log/imaptest.log'
+# - docker exec build sh -c 'doveadm -D force-resync -u t1 INBOX'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Error:\""'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"failed:\""'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Internal error\""'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"killed\""'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Panic:\""'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Fatal:\""'
+# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/imaptest.log | grep \"BUG:\""'
 
 after_script:
 - docker stop build ceph_demo

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Change Log
 ## [0.0.47](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.47) (2022-12-05)
 - #355 fix gzip trailer when stream is empty
+       fix save_method 1+2 buffersize (1 byte short) 
+       bugfix-355-fix-buffersize-write-method
 
+
+## [0.0.46](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.45) (2022-11-22)
 - #349 bugfix doveadm rmb create ceph index validate object metadata
 
 ## [0.0.45](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.45) (2022-11-22)

CLAUDE.md

@@ -0,0 +1,141 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This is a Dovecot plugin that enables email storage in Ceph RADOS objects. It provides two main plugins:
+- **RADOS Storage Plugin (rbox)**: Stores emails as RADOS objects while keeping metadata in filesystem/CephFS
+- **RADOS Dictionary Plugin**: Stores Dovecot dictionaries in Ceph OMAP key/value store
+
+The project uses a hybrid storage model where emails are immutable RADOS objects and index/cache data remains in the filesystem for performance.
+
+## Build System
+
+The project uses GNU Autotools (autoconf/automake) for configuration and building:
+
+### Build Commands
+```bash
+# Generate build system
+./autogen.sh
+
+# Configure build (standard installation)
+./configure
+
+# Configure with custom dovecot location
+./configure --with-dovecot=/path/to/dovecot
+
+# Configure with custom prefix
+./configure --prefix=/usr/local
+
+# Build
+make
+
+# Install
+make install
+```
+
+### Build Options
+- `--with-dict=yes/no`: Build RADOS dictionary plugin (default: yes)
+- `--with-storage=yes/no`: Build RADOS storage plugin (default: yes)
+- `--with-tests=yes/no`: Build tests (default: yes)
+- `--with-integration-tests=yes/no`: Build integration tests (default: no)
+
+## Dependencies
+
+Required packages:
+- `libjansson-devel` (>= 2.9)
+- `librados2-devel` (>= 10.2.5)
+- `dovecot-devel`
+- `spdlog` (optional, for logging)
+
+## Testing
+
+### Unit Tests
+```bash
+# Run individual test executables from src/tests/
+./src/tests/storage-rbox/it_test_storage_rbox
+./src/tests/storage-rbox/it_test_read_mail_rbox
+./src/tests/librmb/it_test_librmb
+```
+
+### Integration Tests
+Enable with `--with-integration-tests=yes` during configure.
+
+### Docker Development Environment
+```bash
+cd docker
+docker-compose up
+# Access container: docker exec -it dovecot-dev bash
+```
+
+## Code Architecture
+
+### Core Components
+
+#### librmb (src/librmb/)
+Core library providing RADOS abstractions:
+- `rados-cluster-impl`: Ceph cluster connection management
+- `rados-storage-impl`: RADOS storage operations
+- `rados-mail-impl`: Mail object handling
+- `rados-dictionary-impl`: Dictionary operations
+- `rados-metadata-*`: Metadata storage implementations
+
+#### Storage Plugin (src/storage-rbox/)
+Dovecot storage plugin implementation:
+- `rbox-storage.cpp`: Main storage backend
+- `rbox-mail.cpp`: Mail object interface
+- `rbox-save.cpp`: Mail saving logic
+- `rbox-sync.cpp`: Mailbox synchronization
+- `rbox-copy.cpp`: Mail copying operations
+
+#### Dictionary Plugin (src/dict-rados/)
+Dovecot dictionary plugin for OMAP storage:
+- `dict-rados.cpp`: Dictionary implementation
+- `libdict-rados-plugin.c`: Plugin interface
+
+#### Tools (src/librmb/tools/rmb/)
+Command-line tools for managing RADOS mailboxes:
+- `rmb.cpp`: Main RMB tool
+- `rmb-commands-impl.cpp`: Command implementations
+
+### Key Interfaces
+
+The code implements Dovecot's plugin interfaces:
+- `mail_storage_vfuncs`: Storage operations
+- `mail_vfuncs`: Mail object operations  
+- `dict_vfuncs`: Dictionary operations
+
+### Configuration
+
+Configuration is handled through:
+- Dovecot configuration files
+- Ceph configuration files (`ceph.conf`)
+- JSON configuration for advanced settings
+
+## Development Guidelines
+
+### Code Style
+- C++11 standard
+- Line length: 120 characters (see CPPLINT.cfg)
+- Use existing code patterns and conventions
+- Headers are organized by component
+
+### Testing Strategy
+- Unit tests for individual components
+- Integration tests for end-to-end functionality
+- Manual testing with IMAP clients (Thunderbird, telnet)
+- Performance testing with imaptest
+
+### Memory Management
+- RAII patterns for resource management
+- Proper cleanup of Ceph contexts
+- Buffer management for large mail objects
+
+## Important Notes
+
+- This is experimental code, not production-ready
+- Mail objects are immutable once stored
+- Index data should be on fast storage (SSD/CephFS)
+- Mail objects can use erasure coding for efficiency
+- The plugin supports both standalone and clustered Dovecot deployments

Dockerfile

@@ -0,0 +1,8 @@
+FROM 192.168.178.73:6088/ceph-dovecot-runtime:f3031c31a2aae0bcfcb504d660a6d4dc555e804d
+WORKDIR /repo
+COPY ./ /repo
+RUN ./autogen.sh
+RUN ./configure --with-dovecot=/usr/local/lib/dovecot --enable-maintainer-mode --enable-debug --with-integration-tests --enable-valgrind --enable-debug
+RUN make install
+ENTRYPOINT ["/bin/bash", "-c", "./startup.sh" ]
+

Dockerfile.dev

@@ -0,0 +1,7 @@
+FROM 192.168.178.73:6088/ceph-dovecot-combined:f551f222f7890a89a2261dcc024e720ba9239bf9 
+WORKDIR /repo
+COPY ./ /repo
+RUN ./autogen.sh
+RUN ./configure --with-dovecot=/usr/local/lib/dovecot --enable-maintainer-mode --enable-debug --with-integration-tests --enable-valgrind --enable-debug
+RUN make clean install
+RUN ./setup.sh

README.md

@@ -79,6 +79,9 @@ If you are using CentOS make sure you also have the following package installed:
     ./configure --with-dovecot=/home/user/workspace/core
     make install
 
+### Docker compose run with dovecot and ceph cluster
+   cd docker
+   docker-compose up 
 ## Thanks
 
 <table border="0">

configure.ac

@@ -53,6 +53,22 @@ AC_CHECK_HEADER([rados/librados.h], [
   LIBS="$LIBS -lrados"
 ], [AC_MSG_ERROR([cannot build without RADOS support: librados.h not found])])
 
+AC_ARG_WITH([SPDLOG],
+  AS_HELP_STRING([--with-SPDLOG=warn|error], [set action for missing spdlog package]),
+  [SPDLOG_ACTION="$withval"],
+  [SPDLOG_ACTION="error"])
+
+# Check for the presence of spdlog library
+PKG_CHECK_MODULES([SPDLOG], [spdlog], [$SPDLOG_ACTION], [SPDLOG_ACTION="error"])
+
+# Check if spdlog is found and add it to LDFLAGS
+if test "x$SPDLOG_LIBS" != "x"; then
+  LDFLAGS="$LDFLAGS $SPDLOG_LIBS"
+  AC_DEFINE([SPD_LOGGING_FRAMEWORK], [1], [Define if spdlog is found])
+else
+  AC_DEFINE([NO_LOGGING_FRAMEWORK], [1], [Define if spdlog is not found])
+fi
+
 AC_CHECK_FUNC(rados_read_op_omap_get_vals2, AC_DEFINE(HAVE_OMAP_GET_VALS2, 1, [Define if you have the `rados_read_op_omap_get_vals2' function]))
 AC_CHECK_FUNC(rados_set_alloc_hint2, AC_DEFINE(HAVE_ALLOC_HINT_2, 1, [Define if you have the `set_alloc_hint2' function]))
 AC_CHECK_FUNC(rados_read_op_omap_get_keys2, AC_DEFINE(HAVE_OMAP_GET_KEYS_2, 1, [Define if you have the `omap_get_keys2' function]))
@@ -232,6 +248,9 @@ src/storage-rbox/Makefile
 src/librmb/tools/Makefile
 src/librmb/tools/rmb/Makefile
 src/tests/Makefile
+src/storage-interface/Makefile
+src/storage-interface/tools/Makefile
+src/storage-interface/tools/rmb/Makefile
 ])
 
 AC_OUTPUT