diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..55ade448 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,110 @@ +image: 192.168.178.73:6088/ceph-dovecot-combined:fe6a06c8907bd7635018205c7eae05dc4c363575 + +services: + - docker:19.03.13-dind + +variables: + DOCKER_REGISTRY_URL: "192.168.178.73:6088" + LC_ALL: "en_US.UTF-8" + LANG: "en_US.UTF-8" + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + PROJECT_NAME: dovecot-ceph-plugin + GIT_SUBMODULE_STRATEGY: recursive + ROOT_PATH: "apps/develop/apps/" + +stages: + - build + - docker + - publish + +build: + stage: build + tags: ["kubernetes"] + artifacts: + paths: + - imaptest.log + - dovecot.log + script: + - /opt/ceph-container/bin/entrypoint.sh demo & + - ./autogen.sh + - ./configure --with-dovecot=/usr/local/lib/dovecot --enable-maintainer-mode --enable-debug --with-integration-tests --enable-valgrind --enable-debug + - make clean install + - ps -aux + - ceph df + - ./setup.sh + - dovecot + - ldconfig + - smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f test@example.com -t t1 inet:127.0.0.1:1024 + - imaptest user=t%d pass=t port=10143 users=10 clients=10 error_quit secs=30 output=./imaptest.log + - doveadm -Dv backup -u t1 -m INBOX mdbox:/usr/local/var/mail/mdbox/t1 + - doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails + - doveadm -D -c /usr/local/etc/dovecot_mdbox/dovecot.conf fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/mdbox.t1.mails + + + - src/scripts/sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted + - src/scripts/sort.sh /root/mdbox.t1.mails /root/mdbox.t1.mails.sorted + - type diff + - diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted + - echo $? + + - rm -rf /usr/local/var/mail/rbox/t1 + - doveadm -Dv -c /usr/local/etc/dovecot_mdbox/dovecot.conf backup -u t1 -m INBOX rbox:/usr/local/var/mail/rbox/t1 + - doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails + - src/scripts/sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted + - diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted + - echo $? + - rm -rf /usr/local/var/mail/rbox + - imaptest user=t%d pass=t port=10143 error_quit secs=15 copybox=INBOX.Drafts output=./imaptest.log + - doveadm -D force-resync -u t1 INBOX + + # - git grep -c "Error" -- ./dovecot.log + # - git grep -c "failed" -- ./dovecot.log + # - git grep -c "Internal error" -- ./dovecot.log + # - git grep -c "killed" -- ./dovecot.log + # - git grep -c "Panic" -- ./dovecot.log + # - git grep -c "Fatal" -- ./dovecot.log + # - git grep -c "BUG" -- ./dovecot.log + # - ceph df + # - make check +build_image: + tags: + - kubernetes + stage: docker + image: + name: gcr.io/kaniko-project/executor:v1.14.0-debug + entrypoint: [""] + before_script: + - | + echo "$NEXUS_IMAGE_REGISTRY_SSH_KEY" >> /kaniko/ssl/certs/ca-certificates.crt + script: + - /kaniko/executor + --context "$CI_PROJECT_DIR" + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "$DOCKER_REGISTRY_URL/$PROJECT_NAME:$CI_COMMIT_SHA" + --skip-tls-verify + +publish_to_master: + dependencies: + - build_image + stage: publish + tags: + - shell + before_script: + - mkdir dev_temp + - cd dev_temp + - git clone http://${CI_USERNAME}:${CI_PUSH_TOKEN}@${CI_SERVER_HOST}/devops/dev.git + - git config --global user.email "gitlab@gitlab.com" + - git config --global user.name "GitLab CI/CD" + script: + - cd dev + - git checkout develop + - git stash + - cd $ROOT_PATH + - cd $PROJECT_NAME + - export image2="yq e -i '.spec.template.spec.containers[0].image = \""$DOCKER_REGISTRY_URL/$PROJECT_NAME:$CI_COMMIT_SHA\""' deployment.yaml" + - eval "$image2" + - git commit -am 'kubernetes-deployment.yaml has been updated with new image' + - git push origin develop + - cd ../../../../../.. + - rm -r dev_temp diff --git a/.travis.yml b/.travis.yml index 341995fa..5473b9fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,43 +23,43 @@ script: - docker exec ceph_demo sh -c 'cp -r /etc/ceph/* /root/cfg' - docker exec build sh -c 'chmod 777 /etc/ceph/*' - docker exec ceph_demo sh -c 'ceph tell mon.\* injectargs "--mon-allow-pool-delete=true"' -- travis_wait 30 docker exec build sh -c 'cd repo/src/tests; make check-valgrind' -- docker exec build sh -c 'cd repo; cat src/tests/test-suite-memcheck.log' +#- travis_wait 30 docker exec build sh -c 'cd repo/src/tests; make check-valgrind' +#- docker exec build sh -c 'cd repo; cat src/tests/test-suite-memcheck.log' #- travis_wait 30 docker exec build sh -c 'cd repo/src/tests; make check' - docker exec build sh -c 'ldconfig' - docker exec build sh -c 'chmod -R 777 /usr/local/var/' - docker exec --detach build sh -c 'service dovecot.service start' -- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 users=100 clients=25 error_quit secs=15 output=/var/log/imaptest.log' -- docker exec build sh -c 'smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f test@example.com -t t1 inet:127.0.0.1:1024' -- docker exec build sh -c 'doveadm -D altmove -u t2 ALL' -- docker exec build sh -c 'doveadm -D altmove -r -u t2 ALL' +#- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 users=100 clients=25 error_quit secs=15 output=/var/log/imaptest.log' +#- docker exec build sh -c 'smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f test@example.com -t t1 inet:127.0.0.1:1024' +#- docker exec build sh -c 'doveadm -D altmove -u t2 ALL' +#- docker exec build sh -c 'doveadm -D altmove -r -u t2 ALL' -- docker exec build sh -c 'doveadm -Dv backup -u t1 -m INBOX mdbox:/usr/local/var/mail/mdbox/t1' -- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails' -- docker exec build sh -c 'doveadm -D -c /usr/local/etc/dovecot_mdbox/dovecot.conf fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/mdbox.t1.mails' -- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted' -- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/mdbox.t1.mails /root/mdbox.t1.mails.sorted' +#- docker exec build sh -c 'doveadm -Dv backup -u t1 -m INBOX mdbox:/usr/local/var/mail/mdbox/t1' +#- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails' +#- docker exec build sh -c 'doveadm -D -c /usr/local/etc/dovecot_mdbox/dovecot.conf fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/mdbox.t1.mails' +#- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted' +#- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/mdbox.t1.mails /root/mdbox.t1.mails.sorted' -- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted' -- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox/t1' -- docker exec build sh -c 'doveadm -Dv -c /usr/local/etc/dovecot_mdbox/dovecot.conf backup -u t1 -m INBOX rbox:/usr/local/var/mail/rbox/t1' -- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails' -- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted' -- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted' +#- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted' +#- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox/t1' +#- docker exec build sh -c 'doveadm -Dv -c /usr/local/etc/dovecot_mdbox/dovecot.conf backup -u t1 -m INBOX rbox:/usr/local/var/mail/rbox/t1' +#- docker exec build sh -c 'doveadm -D fetch -u t1 "guid date.received date.sent flags pop3.uidl seq size.virtual uid user mailbox-guid mailbox" ALL > /root/rbox.t1.mails' +#- docker exec build sh -c 'cd repo/src/scripts; ./sort.sh /root/rbox.t1.mails /root/rbox.t1.mails.sorted' +#- docker exec build sh -c 'diff -y -W 1500 /root/rbox.t1.mails.sorted /root/mdbox.t1.mails.sorted' -- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox' -- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10110 profile=/root/pop3-profile.conf users=100 clients=10 error_quit secs=15 output=/var/log/imaptest.log' +#- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox' +#- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10110 profile=/root/pop3-profile.conf users=100 clients=10 error_quit secs=15 output=/var/log/imaptest.log' -- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox' -- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 error_quit secs=15 copybox=INBOX.Drafts output=/var/log/imaptest.log' -- docker exec build sh -c 'doveadm -D force-resync -u t1 INBOX' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Error:\""' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"failed:\""' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Internal error\""' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"killed\""' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Panic:\""' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Fatal:\""' -- docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/imaptest.log | grep \"BUG:\""' +#- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox' +# #- docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 error_quit secs=15 copybox=INBOX.Drafts output=/var/log/imaptest.log' +# - docker exec build sh -c 'doveadm -D force-resync -u t1 INBOX' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Error:\""' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"failed:\""' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Internal error\""' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"killed\""' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Panic:\""' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Fatal:\""' +# - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/imaptest.log | grep \"BUG:\""' after_script: - docker stop build ceph_demo diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5eb577..a8536b6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # Change Log ## [0.0.47](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.47) (2022-12-05) - #355 fix gzip trailer when stream is empty + fix save_method 1+2 buffersize (1 byte short) + bugfix-355-fix-buffersize-write-method + +## [0.0.46](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.45) (2022-11-22) - #349 bugfix doveadm rmb create ceph index validate object metadata ## [0.0.45](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.45) (2022-11-22) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..922e1063 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,141 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is a Dovecot plugin that enables email storage in Ceph RADOS objects. It provides two main plugins: +- **RADOS Storage Plugin (rbox)**: Stores emails as RADOS objects while keeping metadata in filesystem/CephFS +- **RADOS Dictionary Plugin**: Stores Dovecot dictionaries in Ceph OMAP key/value store + +The project uses a hybrid storage model where emails are immutable RADOS objects and index/cache data remains in the filesystem for performance. + +## Build System + +The project uses GNU Autotools (autoconf/automake) for configuration and building: + +### Build Commands +```bash +# Generate build system +./autogen.sh + +# Configure build (standard installation) +./configure + +# Configure with custom dovecot location +./configure --with-dovecot=/path/to/dovecot + +# Configure with custom prefix +./configure --prefix=/usr/local + +# Build +make + +# Install +make install +``` + +### Build Options +- `--with-dict=yes/no`: Build RADOS dictionary plugin (default: yes) +- `--with-storage=yes/no`: Build RADOS storage plugin (default: yes) +- `--with-tests=yes/no`: Build tests (default: yes) +- `--with-integration-tests=yes/no`: Build integration tests (default: no) + +## Dependencies + +Required packages: +- `libjansson-devel` (>= 2.9) +- `librados2-devel` (>= 10.2.5) +- `dovecot-devel` +- `spdlog` (optional, for logging) + +## Testing + +### Unit Tests +```bash +# Run individual test executables from src/tests/ +./src/tests/storage-rbox/it_test_storage_rbox +./src/tests/storage-rbox/it_test_read_mail_rbox +./src/tests/librmb/it_test_librmb +``` + +### Integration Tests +Enable with `--with-integration-tests=yes` during configure. + +### Docker Development Environment +```bash +cd docker +docker-compose up +# Access container: docker exec -it dovecot-dev bash +``` + +## Code Architecture + +### Core Components + +#### librmb (src/librmb/) +Core library providing RADOS abstractions: +- `rados-cluster-impl`: Ceph cluster connection management +- `rados-storage-impl`: RADOS storage operations +- `rados-mail-impl`: Mail object handling +- `rados-dictionary-impl`: Dictionary operations +- `rados-metadata-*`: Metadata storage implementations + +#### Storage Plugin (src/storage-rbox/) +Dovecot storage plugin implementation: +- `rbox-storage.cpp`: Main storage backend +- `rbox-mail.cpp`: Mail object interface +- `rbox-save.cpp`: Mail saving logic +- `rbox-sync.cpp`: Mailbox synchronization +- `rbox-copy.cpp`: Mail copying operations + +#### Dictionary Plugin (src/dict-rados/) +Dovecot dictionary plugin for OMAP storage: +- `dict-rados.cpp`: Dictionary implementation +- `libdict-rados-plugin.c`: Plugin interface + +#### Tools (src/librmb/tools/rmb/) +Command-line tools for managing RADOS mailboxes: +- `rmb.cpp`: Main RMB tool +- `rmb-commands-impl.cpp`: Command implementations + +### Key Interfaces + +The code implements Dovecot's plugin interfaces: +- `mail_storage_vfuncs`: Storage operations +- `mail_vfuncs`: Mail object operations +- `dict_vfuncs`: Dictionary operations + +### Configuration + +Configuration is handled through: +- Dovecot configuration files +- Ceph configuration files (`ceph.conf`) +- JSON configuration for advanced settings + +## Development Guidelines + +### Code Style +- C++11 standard +- Line length: 120 characters (see CPPLINT.cfg) +- Use existing code patterns and conventions +- Headers are organized by component + +### Testing Strategy +- Unit tests for individual components +- Integration tests for end-to-end functionality +- Manual testing with IMAP clients (Thunderbird, telnet) +- Performance testing with imaptest + +### Memory Management +- RAII patterns for resource management +- Proper cleanup of Ceph contexts +- Buffer management for large mail objects + +## Important Notes + +- This is experimental code, not production-ready +- Mail objects are immutable once stored +- Index data should be on fast storage (SSD/CephFS) +- Mail objects can use erasure coding for efficiency +- The plugin supports both standalone and clustered Dovecot deployments \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f67cc989 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM 192.168.178.73:6088/ceph-dovecot-runtime:f3031c31a2aae0bcfcb504d660a6d4dc555e804d +WORKDIR /repo +COPY ./ /repo +RUN ./autogen.sh +RUN ./configure --with-dovecot=/usr/local/lib/dovecot --enable-maintainer-mode --enable-debug --with-integration-tests --enable-valgrind --enable-debug +RUN make install +ENTRYPOINT ["/bin/bash", "-c", "./startup.sh" ] + diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..b6c974ab --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,7 @@ +FROM 192.168.178.73:6088/ceph-dovecot-combined:f551f222f7890a89a2261dcc024e720ba9239bf9 +WORKDIR /repo +COPY ./ /repo +RUN ./autogen.sh +RUN ./configure --with-dovecot=/usr/local/lib/dovecot --enable-maintainer-mode --enable-debug --with-integration-tests --enable-valgrind --enable-debug +RUN make clean install +RUN ./setup.sh diff --git a/README.md b/README.md index 7c95998e..c0a10439 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,9 @@ If you are using CentOS make sure you also have the following package installed: ./configure --with-dovecot=/home/user/workspace/core make install +### Docker compose run with dovecot and ceph cluster + cd docker + docker-compose up ## Thanks diff --git a/configure.ac b/configure.ac index bcb784e5..570a396b 100644 --- a/configure.ac +++ b/configure.ac @@ -53,6 +53,22 @@ AC_CHECK_HEADER([rados/librados.h], [ LIBS="$LIBS -lrados" ], [AC_MSG_ERROR([cannot build without RADOS support: librados.h not found])]) +AC_ARG_WITH([SPDLOG], + AS_HELP_STRING([--with-SPDLOG=warn|error], [set action for missing spdlog package]), + [SPDLOG_ACTION="$withval"], + [SPDLOG_ACTION="error"]) + +# Check for the presence of spdlog library +PKG_CHECK_MODULES([SPDLOG], [spdlog], [$SPDLOG_ACTION], [SPDLOG_ACTION="error"]) + +# Check if spdlog is found and add it to LDFLAGS +if test "x$SPDLOG_LIBS" != "x"; then + LDFLAGS="$LDFLAGS $SPDLOG_LIBS" + AC_DEFINE([SPD_LOGGING_FRAMEWORK], [1], [Define if spdlog is found]) +else + AC_DEFINE([NO_LOGGING_FRAMEWORK], [1], [Define if spdlog is not found]) +fi + AC_CHECK_FUNC(rados_read_op_omap_get_vals2, AC_DEFINE(HAVE_OMAP_GET_VALS2, 1, [Define if you have the `rados_read_op_omap_get_vals2' function])) AC_CHECK_FUNC(rados_set_alloc_hint2, AC_DEFINE(HAVE_ALLOC_HINT_2, 1, [Define if you have the `set_alloc_hint2' function])) AC_CHECK_FUNC(rados_read_op_omap_get_keys2, AC_DEFINE(HAVE_OMAP_GET_KEYS_2, 1, [Define if you have the `omap_get_keys2' function])) @@ -232,6 +248,9 @@ src/storage-rbox/Makefile src/librmb/tools/Makefile src/librmb/tools/rmb/Makefile src/tests/Makefile +src/storage-interface/Makefile +src/storage-interface/tools/Makefile +src/storage-interface/tools/rmb/Makefile ]) AC_OUTPUT diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index bf5df74e..3028da63 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,44 +1,17 @@ -version: '2' +version: '3' services: - ceph: - image: ceph/daemon:latest - command: "demo" - container_name: 'ceph_demo' - volumes: - - ceph-data:/etc/ceph/ - environment: - - MON_IP=192.168.100.2 - - CEPH_PUBLIC_NETWORK=192.168.100.0/24 - - CEPH_DEMO_UID=demo_uid - ports: - - 3300:3300 - networks: - - ceph_network - dovecot: - # image: cephdovecot/travis-build-master-2.3:latest - image: cephdovecot/travis-build:master-2.3 + build: + context: ../ + dockerfile: Dockerfile.dev command: tail -f /dev/null - container_name: 'build' + container_name: 'dovecot-dev' volumes: - - ceph-data:/etc/ceph/ - - ./../:/repo - environment: - - SOURCE_VERSION=master-2.3 + - mail-data:/usr/local/var/mail/ + - ../:/repo ports: - 10143:10143 - 1024:1024 - 10110:10110 - networks: - - ceph_network - volumes: - ceph-data: - external: false - -networks: - ceph_network: - ipam: - config: - - subnet: 192.168.100.0/24 - gateway: 192.168.100.1 + mail-data: diff --git a/setup.sh b/setup.sh new file mode 100755 index 00000000..682c4eb5 --- /dev/null +++ b/setup.sh @@ -0,0 +1,4 @@ +chmod 777 /etc/ceph/* +ceph tell mon.\* injectargs "--mon-allow-pool-delete=true" +chmod -R 777 /usr/local/var/ +ldconfig \ No newline at end of file diff --git a/src/dict-rados/dict-rados.cpp b/src/dict-rados/dict-rados.cpp index b1f66f53..ce0c3a1f 100644 --- a/src/dict-rados/dict-rados.cpp +++ b/src/dict-rados/dict-rados.cpp @@ -45,9 +45,10 @@ extern "C" { #include "libdict-rados-plugin.h" #include "../librmb/rados-cluster-impl.h" #include "../librmb/rados-dictionary-impl.h" -#include "../librmb/rados-cluster.h" -#include "../librmb/rados-guid-generator.h" -#include "../librmb/rados-util.h" +#include "../storage-interface/rados-cluster.h" +#include "../storage-interface/rados-guid-generator.h" +#include "../storage-interface/rados-util.h" +#include "../storage-engine/storage-backend-factory.h" #if DOVECOT_PREREQ(2, 3) #define dict_lookup(dict, pool, key, value_r, error_r) dict_lookup(dict, pool, key, value_r, error_r) @@ -70,9 +71,9 @@ using librados::AioCompletion; using librados::ObjectWriteOperation; using librados::completion_t; -using librmb::RadosCluster; -using librmb::RadosDictionary; -using librmb::RadosGuidGenerator; +using storage_interface::RadosCluster; +using storage_interface::RadosDictionary; +using storage_interface::RadosGuidGenerator; #define DICT_USERNAME_SEPARATOR '/' @@ -83,7 +84,7 @@ struct rados_dict { RadosGuidGenerator *guid_generator; }; -class DictGuidGenerator : public librmb::RadosGuidGenerator { +class DictGuidGenerator : public storage_interface::RadosGuidGenerator { void generate_guid(std::string *guid) override { guid_128_t namespace_guid; guid_128_generate(namespace_guid); @@ -216,7 +217,7 @@ int rados_dict_wait(struct dict *_dict) class rados_dict_lookup_context { public: - RadosDictionary *dict; + storage_interface::RadosDictionary *dict; ObjectReadOperation read_op; map result_map; int r_val = -1; @@ -289,7 +290,7 @@ void rados_dict_lookup_async(struct dict *_dict, const char *key, dict_lookup_ca lc->callback = callback; lc->read_op.omap_get_vals_by_keys(keys, &lc->result_map, &lc->r_val); - int err = d->get_io_ctx(key).aio_operate(d->get_full_oid(key), lc->completion, &lc->read_op, + int err = d->get_io_ctx_wrapper(key)->aio_operate(d->get_full_oid(key), lc->completion, &lc->read_op, LIBRADOS_OPERATION_NOFLAG, &lc->bl); if (err < 0) { @@ -303,7 +304,8 @@ void rados_dict_lookup_async(struct dict *_dict, const char *key, dict_lookup_ca delete lc; lc = nullptr; } else { - d->push_back_completion(lc->completion); + d->push_back_completion_wrapper->set_push_back_completion(*lc->completion); + d->push_back_completion(d->push_back_completion_wrapper); } } @@ -323,7 +325,7 @@ int rados_dict_lookup(struct dict *_dict, pool_t pool, const char *key, const ch *value_r = nullptr; *error_r = nullptr; - int err = d->get_io_ctx(key).omap_get_vals_by_keys(d->get_full_oid(key), keys, &result_map); + int err = d->get_io_ctx_wrapper(key)->omap_get_vals_by_keys(d->get_full_oid(key), keys, &result_map); if (err == 0) { auto value = result_map.find(key); if (value != result_map.end()) { @@ -452,7 +454,7 @@ class rados_dict_transaction_context { #ifdef DEBUG i_debug("deploy_set_map_value: %s , oid=%s", bl.to_str().c_str(), oid.c_str()); #endif - if ((is_private(key) ? d->get_private_io_ctx() : d->get_shared_io_ctx()).omap_set(oid, map) < 0) { + if ((is_private(key) ? d->get_private_io_ctx_wrapper()->get_io_ctx() : d->get_shared_io_ctx_wrapper()->get_io_ctx()).omap_set(oid, map) < 0) { i_error("unable to set key(%s), oid(%s), is_private(%d)", key.c_str(), oid.c_str(), is_private(key)); } } @@ -473,8 +475,13 @@ class rados_dict_transaction_context { const string key = it->first; std::string oid = is_private(key) ? d->get_private_oid() : d->get_shared_oid(); // it->second is a signed long int - librmb::RadosUtils::osd_add(&(is_private(key) ? d->get_private_io_ctx() : d->get_shared_io_ctx()), oid, key, + storage_interface::RadosUtils *rados_util= + storage_engine::StorageBackendFactory::create_rados_utils( + storage_engine::CEPH); + rados_util->osd_add(is_private(key) ? d->get_private_io_ctx_wrapper() : d->get_shared_io_ctx_wrapper(), oid, key, it->second); + delete rados_util; + rados_util=nullptr; } atomic_inc_map.clear(); } @@ -495,7 +502,7 @@ class rados_dict_transaction_context { #ifdef DEBUG i_debug("deploy_unset_map_value key: %s , oid=%s", key.c_str(), oid.c_str()); #endif - if ((is_private(key) ? d->get_private_io_ctx() : d->get_shared_io_ctx()).omap_rm_keys(oid, keys) < 0) { + if ((is_private(key) ? d->get_private_io_ctx_wrapper()->get_io_ctx() : d->get_shared_io_ctx_wrapper()->get_io_ctx()).omap_rm_keys(oid, keys) < 0) { i_error("unable to unset key(%s), oid(%s), is_private(%d)", key.c_str(), oid.c_str(), is_private(key)); } } @@ -715,7 +722,7 @@ struct dict_iterate_context *rados_dict_iterate_init(struct dict *_dict, const c } } - int err = d->get_private_io_ctx().aio_operate(d->get_private_oid(), private_read_completion, &private_read_op, + int err = d->get_private_io_ctx_wrapper()->aio_operate(d->get_private_oid(), private_read_completion, &private_read_op, &bl_private); #ifdef DEBUG i_debug("rados_dict_iterate_init(): private err=%d(%s)", err, strerror(-err)); @@ -746,7 +753,7 @@ struct dict_iterate_context *rados_dict_iterate_init(struct dict *_dict, const c } int err = - d->get_shared_io_ctx().aio_operate(d->get_shared_oid(), shared_read_completion, &shared_read_op, &bl_shared); + d->get_shared_io_ctx_wrapper()->aio_operate(d->get_shared_oid(), shared_read_completion, &shared_read_op, &bl_shared); #ifdef DEBUG i_debug("rados_dict_iterate_init(): shared err=%d(%s)", err, strerror(-err)); #endif diff --git a/src/librmb/Makefile.am b/src/librmb/Makefile.am index 60f1c1c2..cd85195f 100644 --- a/src/librmb/Makefile.am +++ b/src/librmb/Makefile.am @@ -12,30 +12,24 @@ lib_LTLIBRARIES = \ librmb.la headers = \ - rados-cluster.h \ - rados-dictionary.h \ - rados-storage.h \ encoding.h \ rados-cluster-impl.h \ rados-storage-impl.h \ rados-dictionary-impl.h \ - rados-mail.h \ - rados-util.h \ - rados-metadata.h \ - rados-types.h \ + rados-mail-impl.h \ + rados-util-impl.h \ + rados-metadata-impl.h \ rados-dovecot-config.h \ - rados-namespace-manager.h \ - rados-dovecot-ceph-cfg.h \ + rados-namespace-manager-impl.h\ rados-dovecot-ceph-cfg-impl.h \ - rados-ceph-config.h \ - rados-ceph-json-config.h \ - rados-guid-generator.h \ - rados-metadata-storage.h \ + rados-ceph-config-impl.h \ + rados-ceph-json-config-impl.h \ rados-metadata-storage-impl.h \ - rados-metadata-storage-module.h \ rados-metadata-storage-default.h \ rados-metadata-storage-ima.h \ - rados-save-log.h + rados-save-log-impl.h \ + rbox-io-ctx-impl.h + librmb_la_SOURCES = \ @@ -43,16 +37,16 @@ librmb_la_SOURCES = \ rados-cluster-impl.cpp \ rados-storage-impl.cpp \ rados-dictionary-impl.cpp \ - rados-mail.cpp \ - rados-util.cpp \ + rados-mail-impl.cpp \ + rados-util-impl.cpp \ rados-dovecot-config.cpp \ - rados-namespace-manager.cpp\ - rados-ceph-config.cpp \ + rados-namespace-manager-impl.cpp\ + rados-ceph-config-impl.cpp \ rados-dovecot-ceph-cfg-impl.cpp \ - rados-ceph-json-config.cpp \ + rados-ceph-json-config-impl.cpp \ rados-metadata-storage-default.cpp \ rados-metadata-storage-ima.cpp \ - rados-save-log.cpp + rados-save-log-impl.cpp AM_LDFLAGS = $(JANSSON_LIBS) AM_CFLAGS = $(JANSSON_CFLAGS) diff --git a/src/librmb/rados-ceph-config.cpp b/src/librmb/rados-ceph-config-impl.cpp similarity index 71% rename from src/librmb/rados-ceph-config.cpp rename to src/librmb/rados-ceph-config-impl.cpp index dae707a7..028374d1 100644 --- a/src/librmb/rados-ceph-config.cpp +++ b/src/librmb/rados-ceph-config-impl.cpp @@ -9,35 +9,41 @@ * Foundation. See file COPYING. */ -#include "rados-ceph-config.h" +#include "rados-ceph-config-impl.h" #include #include #include +#include "rados-ceph-json-config-impl.h" namespace librmb { -RadosCephConfig::RadosCephConfig(librados::IoCtx *io_ctx_) { io_ctx = io_ctx_; } +RadosCephConfigImpl::RadosCephConfigImpl(storage_interface::RboxIoCtx *io_ctx_) { + io_ctx = io_ctx_; + if(config==nullptr){ + config = new librmb::RadosCephJsonConfigImpl(); + } +} -int RadosCephConfig::save_cfg() { - ceph::bufferlist buffer; - bool success = config.to_json(&buffer) ? save_object(config.get_cfg_object_name(), buffer) >= 0 : false; +int RadosCephConfigImpl::save_cfg() { + void* buffer=(void*)new ceph::bufferlist(); + bool success = config->to_json(buffer) ? save_object(config->get_cfg_object_name(), buffer) >= 0 : false; return success ? 0 : -1; } -int RadosCephConfig::load_cfg() { - if (config.is_valid()) { +int RadosCephConfigImpl::load_cfg() { + if (config->is_valid()) { return 0; } - ceph::bufferlist buffer; - int ret = read_object(config.get_cfg_object_name(), &buffer); + void* buffer=(void*)new ceph::bufferlist(); + int ret = read_object(config->get_cfg_object_name(), buffer); if (ret < 0) { - return ret; + return ret; } - config.set_valid(true); - return config.from_json(&buffer) ? 0 : -1; + config->set_valid(true); + return config->from_json(buffer) ? 0 : -1; } -bool RadosCephConfig::is_valid_key_value(const std::string &key, const std::string &value) { +bool RadosCephConfigImpl::is_valid_key_value(const std::string &key, const std::string &value) { bool success = false; if (value.empty() || key.empty()) { return false; @@ -65,7 +71,7 @@ bool RadosCephConfig::is_valid_key_value(const std::string &key, const std::stri return success; } -bool RadosCephConfig::update_valid_key_value(const std::string &key, const std::string &value) { +bool RadosCephConfigImpl::update_valid_key_value(const std::string &key, const std::string &value) { bool success = false; if (value.empty() || key.empty()) { return false; @@ -101,13 +107,17 @@ bool RadosCephConfig::update_valid_key_value(const std::string &key, const std:: return success; } -int RadosCephConfig::save_object(const std::string &oid, librados::bufferlist &buffer) { +int RadosCephConfigImpl::save_object(const std::string &oid, void* buffer) { if (io_ctx == nullptr) { return -1; } - return io_ctx->write_full(oid, buffer); + ceph::bufferlist *bl=(ceph::bufferlist*)buffer; + ceph::bufferlist &bl_ref=*bl; + int ret= io_ctx->write_full(oid,bl_ref); + delete buffer; + return ret; } -int RadosCephConfig::read_object(const std::string &oid, librados::bufferlist *buffer) { +int RadosCephConfigImpl::read_object(const std::string &oid, void* buffer) { size_t max = INT_MAX; if (io_ctx == nullptr) { return -1; @@ -115,22 +125,22 @@ int RadosCephConfig::read_object(const std::string &oid, librados::bufferlist *b // retry max times to read the object. int max_retry = 10; int ret_read = -1; - + ceph::bufferlist *bl=(ceph::bufferlist*)buffer; + ceph::bufferlist &bl_ref=*bl; for(int i = 0;iread(oid, *buffer, max, 0); + ret_read = io_ctx->read(oid,bl_ref, max, 0); if(ret_read >= 0 || ret_read == -ENOENT ){ // exit here if the file does not exist, or we were successful break; } - buffer->clear(); + bl_ref.clear(); // wait random time before try again!! usleep(((rand() % 5) + 1) * 10000); } - return ret_read; } -void RadosCephConfig::set_io_ctx_namespace(const std::string &namespace_) { +void RadosCephConfigImpl::set_io_ctx_namespace(const std::string &namespace_) { if (io_ctx != nullptr) { io_ctx->set_namespace(namespace_); } diff --git a/src/librmb/rados-ceph-config-impl.h b/src/librmb/rados-ceph-config-impl.h new file mode 100644 index 00000000..4c909b88 --- /dev/null +++ b/src/librmb/rados-ceph-config-impl.h @@ -0,0 +1,94 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) override2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ +#ifndef SRC_LIBRMB_RADOS_CEPH_CONFIG_H_ +#define SRC_LIBRMB_RADOS_CEPH_CONFIG_H_ + +#include +#include "../storage-interface/rados-ceph-json-config.h" +#include "rados-ceph-json-config-impl.h" +#include "../storage-interface/rados-types.h" +#include +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-ceph-config.h" +namespace librmb { +/** + * RadosCephConfig + * + * read Plugin configuration from rados storage + */ +class RadosCephConfigImpl : public storage_interface::RadosCephConfig{ + public: + explicit RadosCephConfigImpl(storage_interface::RboxIoCtx *io_ctx_); + RadosCephConfigImpl() { + io_ctx = nullptr; + if(config==nullptr){ + config=new librmb::RadosCephJsonConfigImpl(); + } + } + virtual ~RadosCephConfigImpl() { + if(config!=nullptr){ + delete config; + config=nullptr; + } + } + + // load settings from rados cfg_object + int load_cfg() override; + int save_cfg() override; + + void set_io_ctx(storage_interface::RboxIoCtx *io_ctx_) override{ io_ctx = io_ctx_; } + bool is_config_valid() override{ return config->is_valid(); } + void set_config_valid(bool valid_) override{ config->set_valid(valid_); } + bool is_user_mapping() override{ return !config->get_user_mapping().compare("true"); } + void set_user_mapping(bool value_) override{ config->set_user_mapping(value_ ? "true" : "false"); } + void set_user_ns(const std::string &ns_) override{ config->set_user_ns(ns_); } + std::string &get_user_ns() override{ return config->get_user_ns(); } + void set_user_suffix(const std::string &ns_suffix_) override{ config->set_user_suffix(ns_suffix_); } + std::string &get_user_suffix() override{ return config->get_user_suffix(); } + const std::string &get_public_namespace()const override { return config->get_public_namespace(); } + void set_public_namespace(const std::string &public_namespace_) override{ config->set_public_namespace(public_namespace_); } + + void set_cfg_object_name(const std::string &cfg_object_name_) override{ config->set_cfg_object_name(cfg_object_name_); } + std::string get_cfg_object_name() override{ return config->get_cfg_object_name(); } + storage_interface::RadosCephJsonConfig *get_config() override{ return config; } + + bool is_valid_key_value(const std::string &key, const std::string &value) override; + bool update_valid_key_value(const std::string &key, const std::string &value) override; + // bool is_ceph_posix_bugfix_enabled() override; + bool is_mail_attribute(storage_interface::rbox_metadata_key key) override{ return config->is_mail_attribute(key); } + bool is_updateable_attribute(storage_interface::rbox_metadata_key key) override{ return config->is_updateable_attribute(key); } + bool is_update_attributes() override{ return config->is_update_attributes(); } + void set_update_attributes(const std::string &update_attributes_) override{ + config->set_update_attributes(update_attributes_); + } + + void update_mail_attribute(const char *value) override{ config->update_mail_attribute(value); } + void update_updateable_attribute(const char *value) override{ config->update_updateable_attribute(value); } + + const std::string &get_metadata_storage_module() override{ return config->get_metadata_storage_module(); } + const std::string &get_metadata_storage_attribute() override{ return config->get_metadata_storage_attribute(); } + + const std::string &get_mail_attribute_key() override{ return config->get_mail_attribute_key(); } + const std::string &get_updateable_attribute_key() override{ return config->get_updateable_attribute_key(); } + const std::string &get_update_attributes_key() override{ return config->get_update_attributes_key(); } + + int save_object(const std::string &oid, void* buffer) override; + int read_object(const std::string &oid, void* buffer) override; + void set_io_ctx_namespace(const std::string &namespace_) override; + + private: + storage_interface::RadosCephJsonConfig *config = nullptr; + storage_interface::RboxIoCtx *io_ctx = nullptr; +}; + +} /* namespace librmb */ + +#endif /* SRC_LIBRMB_RADOS_CEPH_CONFIG_H_ */ diff --git a/src/librmb/rados-ceph-config.h b/src/librmb/rados-ceph-config.h deleted file mode 100644 index e9cbccde..00000000 --- a/src/librmb/rados-ceph-config.h +++ /dev/null @@ -1,83 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ -#ifndef SRC_LIBRMB_RADOS_CEPH_CONFIG_H_ -#define SRC_LIBRMB_RADOS_CEPH_CONFIG_H_ - -#include -#include "rados-ceph-json-config.h" -#include "rados-types.h" -#include -#include "rados-storage.h" - -namespace librmb { -/** - * RadosCephConfig - * - * read Plugin configuration from rados storage - */ -class RadosCephConfig { - public: - explicit RadosCephConfig(librados::IoCtx *io_ctx_); - RadosCephConfig() { io_ctx = nullptr; } - virtual ~RadosCephConfig() {} - - // load settings from rados cfg_object - int load_cfg(); - int save_cfg(); - - void set_io_ctx(librados::IoCtx *io_ctx_) { io_ctx = io_ctx_; } - bool is_config_valid() { return config.is_valid(); } - void set_config_valid(bool valid_) { config.set_valid(valid_); } - bool is_user_mapping() { return !config.get_user_mapping().compare("true"); } - void set_user_mapping(bool value_) { config.set_user_mapping(value_ ? "true" : "false"); } - void set_user_ns(const std::string &ns_) { config.set_user_ns(ns_); } - std::string &get_user_ns() { return config.get_user_ns(); } - void set_user_suffix(const std::string &ns_suffix_) { config.set_user_suffix(ns_suffix_); } - std::string &get_user_suffix() { return config.get_user_suffix(); } - const std::string &get_public_namespace() const { return config.get_public_namespace(); } - void set_public_namespace(const std::string &public_namespace_) { config.set_public_namespace(public_namespace_); } - - void set_cfg_object_name(const std::string &cfg_object_name_) { config.set_cfg_object_name(cfg_object_name_); } - std::string get_cfg_object_name() { return config.get_cfg_object_name(); } - RadosCephJsonConfig *get_config() { return &config; } - - bool is_valid_key_value(const std::string &key, const std::string &value); - bool update_valid_key_value(const std::string &key, const std::string &value); - bool is_ceph_posix_bugfix_enabled(); - bool is_mail_attribute(enum rbox_metadata_key key) { return config.is_mail_attribute(key); } - bool is_updateable_attribute(enum rbox_metadata_key key) { return config.is_updateable_attribute(key); } - bool is_update_attributes() { return config.is_update_attributes(); } - void set_update_attributes(const std::string &update_attributes_) { - config.set_update_attributes(update_attributes_); - } - - void update_mail_attribute(const char *value) { config.update_mail_attribute(value); } - void update_updateable_attribute(const char *value) { config.update_updateable_attribute(value); } - - const std::string &get_metadata_storage_module() { return config.get_metadata_storage_module(); } - const std::string &get_metadata_storage_attribute() { return config.get_metadata_storage_attribute(); } - - const std::string &get_mail_attribute_key() { return config.get_mail_attribute_key(); } - const std::string &get_updateable_attribute_key() { return config.get_updateable_attribute_key(); } - const std::string &get_update_attributes_key() { return config.get_update_attributes_key(); } - - int save_object(const std::string &oid, librados::bufferlist &buffer); - int read_object(const std::string &oid, librados::bufferlist *buffer); - void set_io_ctx_namespace(const std::string &namespace_); - - private: - RadosCephJsonConfig config; - librados::IoCtx *io_ctx; -}; - -} /* namespace tallence */ - -#endif /* SRC_LIBRMB_RADOS_CEPH_CONFIG_H_ */ diff --git a/src/librmb/rados-ceph-json-config.cpp b/src/librmb/rados-ceph-json-config-impl.cpp similarity index 68% rename from src/librmb/rados-ceph-json-config.cpp rename to src/librmb/rados-ceph-json-config-impl.cpp index 3128a1d0..4e9040aa 100644 --- a/src/librmb/rados-ceph-json-config.cpp +++ b/src/librmb/rados-ceph-json-config-impl.cpp @@ -9,14 +9,14 @@ * Foundation. See file COPYING. */ -#include "rados-ceph-json-config.h" +#include "rados-ceph-json-config-impl.h" #include #include #include namespace librmb { -RadosCephJsonConfig::RadosCephJsonConfig() +RadosCephJsonConfigImpl::RadosCephJsonConfigImpl() : cfg_object_name("rbox_cfg"), valid(false), user_mapping("false"), @@ -39,28 +39,28 @@ RadosCephJsonConfig::RadosCephJsonConfig() set_default_updateable_attributes(); } -void RadosCephJsonConfig::set_default_mail_attributes() { - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_MAILBOX_GUID))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_GUID))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_POP3_UIDL))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_POP3_ORDER))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_RECEIVED_TIME))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_PHYSICAL_SIZE))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_VIRTUAL_SIZE))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_ORIG_MAILBOX))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_MAIL_UID))); - mail_attributes.append(std::string(1, static_cast(RBOX_METADATA_VERSION))); +void RadosCephJsonConfigImpl::set_default_mail_attributes() { + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_MAILBOX_GUID))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_GUID))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_POP3_UIDL))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_POP3_ORDER))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_RECEIVED_TIME))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_PHYSICAL_SIZE))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_VIRTUAL_SIZE))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_ORIG_MAILBOX))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_MAIL_UID))); + mail_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_VERSION))); } -void RadosCephJsonConfig::set_default_updateable_attributes() { - updateable_attributes.append(std::string(1, static_cast(RBOX_METADATA_ORIG_MAILBOX))); +void RadosCephJsonConfigImpl::set_default_updateable_attributes() { + updateable_attributes.append(std::string(1, static_cast(storage_interface::RBOX_METADATA_ORIG_MAILBOX))); } -bool RadosCephJsonConfig::from_json(librados::bufferlist *buffer) { +bool RadosCephJsonConfigImpl::from_json(void* buffer) { json_t *root; json_error_t error; bool ret = false; - root = json_loads(buffer->to_str().c_str(), 0, &error); + root = json_loads(((ceph::bufferlist*)buffer)->to_str().c_str(), 0, &error); if (root) { json_t *ns = json_object_get(root, key_user_mapping.c_str()); @@ -93,11 +93,11 @@ bool RadosCephJsonConfig::from_json(librados::bufferlist *buffer) { ret = valid = true; json_decref(root); } - + delete buffer; return ret; } -bool RadosCephJsonConfig::to_json(librados::bufferlist *buffer) { +bool RadosCephJsonConfigImpl::to_json(void* buffer) { json_t *root = json_object(); json_object_set_new(root, key_user_mapping.c_str(), json_string(user_mapping.c_str())); @@ -112,13 +112,13 @@ bool RadosCephJsonConfig::to_json(librados::bufferlist *buffer) { json_object_set_new(root, key_metadata_storage_attribute.c_str(), json_string(metadata_storage_attribute.c_str())); char *s = json_dumps(root, 0); - buffer->append(s); + ((ceph::bufferlist*)buffer)->append(s); free(s); json_decref(root); return true; } -std::string RadosCephJsonConfig::to_string() { +std::string RadosCephJsonConfigImpl::to_string() { std::ostringstream ss; ss << "Configuration : " << cfg_object_name << std::endl; ss << " " << key_user_mapping << "=" << user_mapping << std::endl; @@ -133,21 +133,21 @@ std::string RadosCephJsonConfig::to_string() { return ss.str(); } -bool RadosCephJsonConfig::is_mail_attribute(enum rbox_metadata_key key) { - return mail_attributes.find_first_of(librmb::rbox_metadata_key_to_char(key), 0) != std::string::npos; +bool RadosCephJsonConfigImpl::is_mail_attribute(storage_interface::rbox_metadata_key key) { + return mail_attributes.find_first_of(storage_interface::rbox_metadata_key_to_char(key), 0) != std::string::npos; } -bool RadosCephJsonConfig::is_updateable_attribute(enum rbox_metadata_key key) { - return updateable_attributes.find_first_of(librmb::rbox_metadata_key_to_char(key), 0) != std::string::npos; +bool RadosCephJsonConfigImpl::is_updateable_attribute(storage_interface::rbox_metadata_key key) { + return updateable_attributes.find_first_of(storage_interface::rbox_metadata_key_to_char(key), 0) != std::string::npos; } -void RadosCephJsonConfig::update_mail_attribute(const char *value) { +void RadosCephJsonConfigImpl::update_mail_attribute(const char *value) { if (value == NULL) { return; } mail_attributes = value; } -void RadosCephJsonConfig::update_updateable_attribute(const char *value) { +void RadosCephJsonConfigImpl::update_updateable_attribute(const char *value) { if (value == NULL) { return; } diff --git a/src/librmb/rados-ceph-json-config-impl.h b/src/librmb/rados-ceph-json-config-impl.h new file mode 100644 index 00000000..9191ee1e --- /dev/null +++ b/src/librmb/rados-ceph-json-config-impl.h @@ -0,0 +1,132 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_RADOS_CEPH_JSON_CONFIG_IMPL_H_ +#define SRC_LIBRMB_RADOS_CEPH_JSON_CONFIG_IMPL_H_ + +#include + +#include +#include "../storage-interface/rados-types.h" +#include "../storage-interface/rados-ceph-json-config.h" + +namespace librmb { +/** + * Rados CephJsonConfig + * + * Dovecot-Ceph-plugin configuration + * Ceph Configuration is json object, class + * provides access to json object / Serialize / Deserialize + * + */ +class RadosCephJsonConfigImpl : public storage_interface::RadosCephJsonConfig { + public: + RadosCephJsonConfigImpl(); + virtual ~RadosCephJsonConfigImpl() {} + + bool from_json(void* buffer) override; + bool to_json(void* buffer) override; + std::string to_string(); + + const std::string& get_cfg_object_name() const override{ return cfg_object_name; } + + void set_cfg_object_name(const std::string& cfgObjectName) override{ cfg_object_name = cfgObjectName; } + + const std::string& get_user_mapping() const override{ return user_mapping; } + + void set_user_mapping(const std::string& user_mapping_) override{ user_mapping = user_mapping_; } + + bool is_valid() const override{ return valid; } + + void set_valid(bool isValid) override{ valid = isValid; } + + std::string& get_user_ns() override{ return user_ns; } + + void set_user_ns(const std::string& user_ns_) override{ user_ns = user_ns_; } + + std::string& get_user_suffix() override{ return user_suffix; } + + void set_user_suffix(const std::string& user_suffix_) override{ user_suffix = user_suffix_; } + + const std::string& get_public_namespace() const override{ return public_namespace; } + + void set_public_namespace(const std::string& public_namespace_) override{ public_namespace = public_namespace_; } + + void set_mail_attributes(const std::string& mail_attributes_) override{ mail_attributes = mail_attributes_; } + void set_update_attributes(const std::string& update_attributes_) override{ update_attributes = update_attributes_; } + void set_updateable_attributes(const std::string& updateable_attributes_) override{ + updateable_attributes = updateable_attributes_; + } + + bool is_mail_attribute(storage_interface::rbox_metadata_key key) override; + bool is_updateable_attribute(storage_interface::rbox_metadata_key key) override; + bool is_update_attributes() override{ return update_attributes.compare("true") == 0; } + + void set_metadata_storage_module(const std::string& metadata_storage_module_) override{ + metadata_storage_module = metadata_storage_module_; + } + const std::string& get_metadata_storage_module() override{ return metadata_storage_module; } + + void set_metadata_storage_attribute(const std::string& metadata_storage_attribute_) override{ + metadata_storage_attribute = metadata_storage_attribute_; + } + const std::string& get_metadata_storage_attribute() override{ return metadata_storage_attribute; } + + void update_mail_attribute(const char* value) override; + void update_updateable_attribute(const char* value) override; + + const std::string& get_key_user_mapping() const override{ return key_user_mapping; } + const std::string& get_key_ns_cfg() const override{ return key_user_ns; } + const std::string& get_key_ns_suffix() const override{ return key_user_suffix; } + const std::string& get_key_public_namespace() const override{ return key_public_namespace; } + + const std::string& get_mail_attribute_key() override{ return key_mail_attributes; } + const std::string& get_updateable_attribute_key() override{ return key_updateable_attributes; } + const std::string& get_update_attributes_key() override{ return key_update_attributes; } + + const std::string& get_metadata_storage_module_key() override{ return key_metadata_storage_module; } + const std::string& get_metadata_storage_attribute_key() override{ return key_metadata_storage_attribute; } + + private: + void set_default_mail_attributes(); + void set_default_updateable_attributes(); + + private: + std::string cfg_object_name; + bool valid; + std::string user_mapping; + std::string user_ns; + std::string user_suffix; + std::string public_namespace; + + std::string mail_attributes; + std::string update_attributes; + std::string updateable_attributes; + + std::string metadata_storage_module; + std::string metadata_storage_attribute; + + std::string key_user_mapping; + std::string key_user_ns; + std::string key_user_suffix; + std::string key_public_namespace; + + std::string key_mail_attributes; + std::string key_update_attributes; + std::string key_updateable_attributes; + + std::string key_metadata_storage_module; + std::string key_metadata_storage_attribute; +}; + +} /* namespace librmb */ + +#endif // SRC_LIBRMB_RADOS_CEPH_JSON_CONFIG_IMPL_H_ diff --git a/src/librmb/rados-ceph-json-config.h b/src/librmb/rados-ceph-json-config.h deleted file mode 100644 index 8cdffbb6..00000000 --- a/src/librmb/rados-ceph-json-config.h +++ /dev/null @@ -1,131 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_RADOS_CEPH_JSON_CONFIG_H_ -#define SRC_LIBRMB_RADOS_CEPH_JSON_CONFIG_H_ - -#include - -#include -#include "rados-types.h" - -namespace librmb { -/** - * Rados CephJsonConfig - * - * Dovecot-Ceph-plugin configuration - * Ceph Configuration is json object, class - * provides access to json object / Serialize / Deserialize - * - */ -class RadosCephJsonConfig { - public: - RadosCephJsonConfig(); - virtual ~RadosCephJsonConfig() {} - - bool from_json(librados::bufferlist* buffer); - bool to_json(librados::bufferlist* buffer); - std::string to_string(); - - const std::string& get_cfg_object_name() const { return cfg_object_name; } - - void set_cfg_object_name(const std::string& cfgObjectName) { cfg_object_name = cfgObjectName; } - - const std::string& get_user_mapping() const { return user_mapping; } - - void set_user_mapping(const std::string& user_mapping_) { user_mapping = user_mapping_; } - - bool is_valid() const { return valid; } - - void set_valid(bool isValid) { valid = isValid; } - - std::string& get_user_ns() { return user_ns; } - - void set_user_ns(const std::string& user_ns_) { user_ns = user_ns_; } - - std::string& get_user_suffix() { return user_suffix; } - - void set_user_suffix(const std::string& user_suffix_) { user_suffix = user_suffix_; } - - const std::string& get_public_namespace() const { return public_namespace; } - - void set_public_namespace(const std::string& public_namespace_) { public_namespace = public_namespace_; } - - void set_mail_attributes(const std::string& mail_attributes_) { mail_attributes = mail_attributes_; } - void set_update_attributes(const std::string& update_attributes_) { update_attributes = update_attributes_; } - void set_updateable_attributes(const std::string& updateable_attributes_) { - updateable_attributes = updateable_attributes_; - } - - bool is_mail_attribute(enum rbox_metadata_key key); - bool is_updateable_attribute(enum rbox_metadata_key key); - bool is_update_attributes() { return update_attributes.compare("true") == 0; } - - void set_metadata_storage_module(const std::string& metadata_storage_module_) { - metadata_storage_module = metadata_storage_module_; - } - const std::string& get_metadata_storage_module() { return metadata_storage_module; } - - void set_metadata_storage_attribute(const std::string& metadata_storage_attribute_) { - metadata_storage_attribute = metadata_storage_attribute_; - } - const std::string& get_metadata_storage_attribute() { return metadata_storage_attribute; } - - void update_mail_attribute(const char* value); - void update_updateable_attribute(const char* value); - - const std::string& get_key_user_mapping() const { return key_user_mapping; } - const std::string& get_key_ns_cfg() const { return key_user_ns; } - const std::string& get_key_ns_suffix() const { return key_user_suffix; } - const std::string& get_key_public_namespace() const { return key_public_namespace; } - - const std::string& get_mail_attribute_key() { return key_mail_attributes; } - const std::string& get_updateable_attribute_key() { return key_updateable_attributes; } - const std::string& get_update_attributes_key() { return key_update_attributes; } - - const std::string& get_metadata_storage_module_key() { return key_metadata_storage_module; } - const std::string& get_metadata_storage_attribute_key() { return key_metadata_storage_attribute; } - - private: - void set_default_mail_attributes(); - void set_default_updateable_attributes(); - - private: - std::string cfg_object_name; - bool valid; - std::string user_mapping; - std::string user_ns; - std::string user_suffix; - std::string public_namespace; - - std::string mail_attributes; - std::string update_attributes; - std::string updateable_attributes; - - std::string metadata_storage_module; - std::string metadata_storage_attribute; - - std::string key_user_mapping; - std::string key_user_ns; - std::string key_user_suffix; - std::string key_public_namespace; - - std::string key_mail_attributes; - std::string key_update_attributes; - std::string key_updateable_attributes; - - std::string key_metadata_storage_module; - std::string key_metadata_storage_attribute; -}; - -} /* namespace librmb */ - -#endif // SRC_LIBRMB_RADOS_CEPH_JSON_CONFIG_H_ diff --git a/src/librmb/rados-cluster-impl.cpp b/src/librmb/rados-cluster-impl.cpp index 64d5347f..1daa52bf 100644 --- a/src/librmb/rados-cluster-impl.cpp +++ b/src/librmb/rados-cluster-impl.cpp @@ -17,7 +17,9 @@ #include "rados-dictionary-impl.h" #include "rados-storage-impl.h" -#include "rados-util.h" +#include "rados-util-impl.h" +#include "rbox-io-ctx-impl.h" + using std::list; using std::pair; using std::string; @@ -92,8 +94,8 @@ std::vector RadosClusterImpl::list_pgs_for_pool(std::string &pool_n librados::bufferlist inbl; librados::bufferlist outbl; RadosClusterImpl::cluster->mon_command(cmd, inbl, &outbl, nullptr); - - std::vector list = RadosUtils::extractPgs(std::string(outbl.c_str())); + librmb::RadosUtilsImpl rados_utils; + std::vector list = rados_utils.extractPgs(std::string(outbl.c_str())); for (auto const &token: list) { std::cout << token << std::endl; @@ -116,7 +118,8 @@ std::map> RadosClusterImpl::list_pgs_osd_f librados::bufferlist inbl; librados::bufferlist outbl; RadosClusterImpl::cluster->mon_command(cmd, inbl, &outbl, nullptr); - return RadosUtils::extractPgAndPrimaryOsd(std::string(outbl.c_str())); + librmb::RadosUtilsImpl rados_utils; + return rados_utils.extractPgAndPrimaryOsd(std::string(outbl.c_str())); } int RadosClusterImpl::initialize() { int ret = 0; @@ -200,11 +203,11 @@ int RadosClusterImpl::pool_create(const string &pool) { return ret; } -int RadosClusterImpl::io_ctx_create(const string &pool, librados::IoCtx *io_ctx) { +int RadosClusterImpl::io_ctx_create(const string &pool, storage_interface::RboxIoCtx *io_ctx_wrapper) { int ret = 0; - - assert(io_ctx != nullptr); - + librados::IoCtx *io_ctx_create=&io_ctx_wrapper->get_io_ctx(); + assert(io_ctx_create != nullptr); + if (RadosClusterImpl::cluster_ref_count == 0) { ret = -ENOENT; return ret; @@ -218,19 +221,20 @@ int RadosClusterImpl::io_ctx_create(const string &pool, librados::IoCtx *io_ctx) // pool exists? else create ret = pool_create(pool); if (ret == 0) { - ret = RadosClusterImpl::cluster->ioctx_create(pool.c_str(), *io_ctx); + ret = RadosClusterImpl::cluster->ioctx_create(pool.c_str(), *io_ctx_create); } return ret; } int RadosClusterImpl::recovery_index_io_ctx(const std::string &pool, - librados::IoCtx *io_ctx) { + storage_interface::RboxIoCtx *io_ctx_wrapper) { + librados::IoCtx *io_ctx_recovery=&io_ctx_wrapper->get_recovery_io_ctx(); if(!is_connected()) { return -1; } // pool exists? else create int ret = pool_create(pool); if (ret == 0) { - ret = RadosClusterImpl::cluster->ioctx_create(pool.c_str(), *io_ctx); + ret = RadosClusterImpl::cluster->ioctx_create(pool.c_str(), *io_ctx_recovery); } return ret; } diff --git a/src/librmb/rados-cluster-impl.h b/src/librmb/rados-cluster-impl.h index 5fd57016..977142d8 100644 --- a/src/librmb/rados-cluster-impl.h +++ b/src/librmb/rados-cluster-impl.h @@ -17,10 +17,11 @@ #include #include -#include "rados-cluster.h" +#include "../storage-interface/rados-cluster.h" +#include "../storage-interface/rbox-io-ctx.h" namespace librmb { -class RadosClusterImpl : public RadosCluster { +class RadosClusterImpl : public storage_interface::RadosCluster { public: RadosClusterImpl(); virtual ~RadosClusterImpl(); @@ -32,12 +33,12 @@ class RadosClusterImpl : public RadosCluster { void deinit() override; int pool_create(const std::string &pool) override; - int io_ctx_create(const std::string &pool, librados::IoCtx *io_ctx) override; - int recovery_index_io_ctx(const std::string &pool, librados::IoCtx *io_ctx) override; + int io_ctx_create(const std::string &pool, storage_interface::RboxIoCtx *io_ctx_wrapper) override; + int recovery_index_io_ctx(const std::string &pool, storage_interface::RboxIoCtx *io_ctx_wrapper) override; int get_config_option(const char *option, std::string *value) override; int dictionary_create(const std::string &pool, const std::string &username, const std::string &oid, - RadosDictionary **dictionary); + storage_interface::RadosDictionary **dictionary); bool is_connected() override; librados::Rados &get_cluster() { return *cluster; } void set_config_option(const char *option, const char *value); diff --git a/src/librmb/rados-dictionary-impl.cpp b/src/librmb/rados-dictionary-impl.cpp index f49aa28f..88f7b6d9 100644 --- a/src/librmb/rados-dictionary-impl.cpp +++ b/src/librmb/rados-dictionary-impl.cpp @@ -22,7 +22,9 @@ #include #include #include - +#include "../storage-interface/rbox-io-ctx.h" +#include "rbox-io-ctx-impl.h" +#include "rados-namespace-manager-impl.h" #include using std::string; @@ -37,8 +39,8 @@ using librmb::RadosDictionaryImpl; #define DICT_PATH_PRIVATE "priv/" #define DICT_PATH_SHARED "shared/" -RadosDictionaryImpl::RadosDictionaryImpl(RadosCluster *_cluster, const string &_poolname, const string &_username, - const string &_oid, librmb::RadosGuidGenerator *guid_generator_, +RadosDictionaryImpl::RadosDictionaryImpl(storage_interface::RadosCluster *_cluster, const string &_poolname, const string &_username, + const string &_oid, storage_interface::RadosGuidGenerator *guid_generator_, const std::string &cfg_object_name_) : cluster(_cluster), poolname(_poolname), @@ -52,6 +54,12 @@ RadosDictionaryImpl::RadosDictionaryImpl(RadosCluster *_cluster, const string &_ guid_generator = guid_generator_; shared_oid = oid; private_oid = oid; + + shared_io_ctx_wrapper=new librmb::RboxIoCtxImpl(); + private_io_ctx_wrapper=new librmb::RboxIoCtxImpl(); + + remove_completion_wrapper=new librmb::RboxIoCtxImpl(); + push_back_completion_wrapper=new librmb::RboxIoCtxImpl(); } RadosDictionaryImpl::~RadosDictionaryImpl() { @@ -59,6 +67,22 @@ RadosDictionaryImpl::~RadosDictionaryImpl() { delete namespace_mgr; namespace_mgr = nullptr; } + if(shared_io_ctx_wrapper!=nullptr){ + delete shared_io_ctx_wrapper; + shared_io_ctx_wrapper=nullptr; + } + if(private_io_ctx_wrapper!=nullptr){ + delete private_io_ctx_wrapper; + private_io_ctx_wrapper=nullptr; + } + if(remove_completion_wrapper!=nullptr){ + delete remove_completion_wrapper; + remove_completion_wrapper=nullptr; + } + if(push_back_completion_wrapper!=nullptr){ + delete push_back_completion_wrapper; + push_back_completion_wrapper=nullptr; + } if (cfg != nullptr) { delete cfg; cfg = nullptr; @@ -80,20 +104,20 @@ const string RadosDictionaryImpl::get_full_oid(const std::string &key) { return ""; } -librados::IoCtx &RadosDictionaryImpl::get_shared_io_ctx() { +storage_interface::RboxIoCtx *RadosDictionaryImpl::get_shared_io_ctx_wrapper() { if (!shared_io_ctx_created) { - shared_io_ctx_created = cluster->io_ctx_create(poolname, &shared_io_ctx) == 0; + shared_io_ctx_created = cluster->io_ctx_create(poolname, shared_io_ctx_wrapper) == 0; std::string ns; - if (load_configuration(&shared_io_ctx)) { + if (load_configuration(shared_io_ctx_wrapper)) { std::string user = cfg->get_public_namespace(); lookup_namespace(user, cfg, &ns); - shared_io_ctx.set_namespace(ns); + shared_io_ctx_wrapper->set_namespace(ns); } } - return shared_io_ctx; + return shared_io_ctx_wrapper; } -bool RadosDictionaryImpl::load_configuration(librados::IoCtx *io_ctx) { +bool RadosDictionaryImpl::load_configuration(storage_interface::RboxIoCtx *io_ctx) { bool loaded = true; if (cfg != nullptr) { return loaded; @@ -116,9 +140,9 @@ bool RadosDictionaryImpl::load_configuration(librados::IoCtx *io_ctx) { return loaded; } -bool RadosDictionaryImpl::lookup_namespace(std::string &username_, librmb::RadosDovecotCephCfg *cfg_, std::string *ns) { +bool RadosDictionaryImpl::lookup_namespace(std::string &username_, storage_interface::RadosDovecotCephCfg *cfg_, std::string *ns) { if (namespace_mgr == nullptr) { - namespace_mgr = new librmb::RadosNamespaceManager(cfg_); + namespace_mgr = new RadosNamespaceManagerImpl(cfg_); } if (!namespace_mgr->lookup_key(username_, ns)) { return namespace_mgr->add_namespace_entry(username_, ns, guid_generator) ? true : false; @@ -126,26 +150,26 @@ bool RadosDictionaryImpl::lookup_namespace(std::string &username_, librmb::Rados return 0; } -librados::IoCtx &RadosDictionaryImpl::get_private_io_ctx() { +storage_interface::RboxIoCtx *RadosDictionaryImpl::get_private_io_ctx_wrapper() { if (!private_io_ctx_created) { - if (cluster->io_ctx_create(poolname, &private_io_ctx) == 0) { - if (load_configuration(&private_io_ctx)) { + if (cluster->io_ctx_create(poolname, private_io_ctx_wrapper) == 0) { + if (load_configuration(private_io_ctx_wrapper)) { std::string ns; std::string user = username + cfg->get_user_suffix(); lookup_namespace(user, cfg, &ns); private_io_ctx_created = true; - private_io_ctx.set_namespace(ns); + private_io_ctx_wrapper->set_namespace(ns); } } } - return private_io_ctx; + return private_io_ctx_wrapper; } -librados::IoCtx &RadosDictionaryImpl::get_io_ctx(const std::string &key) { +storage_interface::RboxIoCtx *RadosDictionaryImpl::get_io_ctx_wrapper(const std::string &key) { if (!key.compare(0, strlen(DICT_PATH_PRIVATE), DICT_PATH_PRIVATE)) { - return get_private_io_ctx(); + return get_private_io_ctx_wrapper(); } else if (!key.compare(0, strlen(DICT_PATH_SHARED), DICT_PATH_SHARED)) { - return get_shared_io_ctx(); + return get_shared_io_ctx_wrapper(); } assert(false); // TODO(jrse): in the unlikely case (it's either private or public), io_ctx is not private and not // public, the return value is @@ -163,7 +187,7 @@ int RadosDictionaryImpl::get(const string &key, string *value_r) { oro.omap_get_vals_by_keys(keys, &map, &r_val); librados::bufferlist bl; - int err = get_io_ctx(key).operate(get_full_oid(key), &oro, &bl); + int err = get_io_ctx_wrapper(key)->operate(get_full_oid(key), &oro, &bl); if (err == 0) { if (r_val == 0) { @@ -181,12 +205,14 @@ int RadosDictionaryImpl::get(const string &key, string *value_r) { return err; } -void RadosDictionaryImpl::remove_completion(librados::AioCompletion *c) { +void RadosDictionaryImpl::remove_completion(storage_interface::RboxIoCtx *remove_completion_wrapper) { + librados::AioCompletion *c=&remove_completion_wrapper->get_remove_completion(); completions_mutex.lock(); completions.remove(c); completions_mutex.unlock(); } -void RadosDictionaryImpl::push_back_completion(librados::AioCompletion *c) { +void RadosDictionaryImpl::push_back_completion(storage_interface::RboxIoCtx *push_back_completion_wrapper_) { + librados::AioCompletion *c=&push_back_completion_wrapper_->get_push_back_completion(); completions_mutex.lock(); completions.push_back(c); completions_mutex.unlock(); @@ -196,7 +222,8 @@ void RadosDictionaryImpl::wait_for_completions() { while (!completions.empty()) { auto c = completions.front(); c->wait_for_complete_and_cb(); - remove_completion(c); + remove_completion_wrapper->set_remove_completion(*c); + remove_completion(remove_completion_wrapper); c->release(); } } diff --git a/src/librmb/rados-dictionary-impl.h b/src/librmb/rados-dictionary-impl.h index 2025db4c..36a2bea3 100644 --- a/src/librmb/rados-dictionary-impl.h +++ b/src/librmb/rados-dictionary-impl.h @@ -16,21 +16,22 @@ #include #include #include // NOLINT - +#include "../storage-interface/rbox-io-ctx.h" #include -#include "rados-cluster.h" -#include "rados-dictionary.h" -#include "rados-dovecot-ceph-cfg.h" +#include "../storage-interface/rados-cluster.h" +#include "../storage-interface/rados-dictionary.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" #include "rados-dovecot-ceph-cfg-impl.h" -#include "rados-namespace-manager.h" -#include "rados-guid-generator.h" +#include "../storage-interface/rados-namespace-manager.h" +#include "../storage-interface/rados-guid-generator.h" + namespace librmb { -class RadosDictionaryImpl : public RadosDictionary { +class RadosDictionaryImpl : public storage_interface::RadosDictionary { public: - RadosDictionaryImpl(RadosCluster* cluster, const std::string& poolname, const std::string& username, - const std::string& oid, librmb::RadosGuidGenerator* guid_generator_, + RadosDictionaryImpl(storage_interface::RadosCluster* cluster, const std::string& poolname, const std::string& username, + const std::string& oid, storage_interface::RadosGuidGenerator* guid_generator_, const std::string& cfg_object_name_); virtual ~RadosDictionaryImpl(); @@ -42,42 +43,41 @@ class RadosDictionaryImpl : public RadosDictionary { const std::string& get_username() override { return username; } const std::string& get_poolname() override { return poolname; } - librados::IoCtx& get_io_ctx(const std::string& key) override; - librados::IoCtx& get_shared_io_ctx() override; - librados::IoCtx& get_private_io_ctx() override; + storage_interface::RboxIoCtx* get_io_ctx_wrapper(const std::string& key) override; + storage_interface::RboxIoCtx* get_shared_io_ctx_wrapper() override; + storage_interface::RboxIoCtx* get_private_io_ctx_wrapper() override; - void remove_completion(librados::AioCompletion* c) override; - void push_back_completion(librados::AioCompletion* c) override; + void remove_completion(storage_interface::RboxIoCtx* remove_completion_wrapper) override; + void push_back_completion(storage_interface::RboxIoCtx* push_back_completion_wrapper_) override; void wait_for_completions() override; int get(const std::string& key, std::string* value_r) override; - + private: - bool load_configuration(librados::IoCtx* io_ctx); - - bool lookup_namespace(std::string& username_, librmb::RadosDovecotCephCfg* cfg_, std::string* ns); + bool load_configuration(storage_interface::RboxIoCtx* io_ctx); + bool lookup_namespace(std::string& username_, storage_interface::RadosDovecotCephCfg* cfg_, std::string* ns); private: - RadosCluster* cluster; + storage_interface::RadosCluster* cluster; std::string poolname; std::string username; std::string oid; std::string shared_oid; - librados::IoCtx shared_io_ctx; + storage_interface::RboxIoCtx* shared_io_ctx_wrapper; bool shared_io_ctx_created; std::string private_oid; - librados::IoCtx private_io_ctx; + storage_interface::RboxIoCtx* private_io_ctx_wrapper; bool private_io_ctx_created; std::list completions; std::mutex completions_mutex; - librmb::RadosDovecotCephCfg* cfg; - librmb::RadosNamespaceManager* namespace_mgr; + storage_interface::RadosDovecotCephCfg* cfg; + storage_interface::RadosNamespaceManager* namespace_mgr; - RadosGuidGenerator* guid_generator; + storage_interface::RadosGuidGenerator* guid_generator; std::string cfg_object_name; }; diff --git a/src/librmb/rados-dovecot-ceph-cfg-impl.cpp b/src/librmb/rados-dovecot-ceph-cfg-impl.cpp index cb2e8673..c54587b4 100644 --- a/src/librmb/rados-dovecot-ceph-cfg-impl.cpp +++ b/src/librmb/rados-dovecot-ceph-cfg-impl.cpp @@ -10,19 +10,26 @@ */ #include "rados-dovecot-ceph-cfg-impl.h" +#include "../storage-interface/rados-ceph-config.h" +#include "rados-ceph-config-impl.h" namespace librmb { -RadosDovecotCephCfgImpl::RadosDovecotCephCfgImpl(librados::IoCtx *io_ctx_) { - rados_cfg.set_io_ctx(io_ctx_); +RadosDovecotCephCfgImpl::RadosDovecotCephCfgImpl(storage_interface::RboxIoCtx *io_ctx_) { + if(rados_cfg!=nullptr){ + rados_cfg->set_io_ctx(io_ctx_); + }else{ + rados_cfg= + new librmb::RadosCephConfigImpl(io_ctx_); + } } -RadosDovecotCephCfgImpl::RadosDovecotCephCfgImpl(RadosConfig &dovecot_cfg_, RadosCephConfig &rados_cfg_) : dovecot_cfg(dovecot_cfg_), rados_cfg(rados_cfg_) {} +RadosDovecotCephCfgImpl::RadosDovecotCephCfgImpl(librmb::RadosConfig &dovecot_cfg_, storage_interface::RadosCephConfig *rados_cfg_) : dovecot_cfg(dovecot_cfg_), rados_cfg(rados_cfg_) {} int RadosDovecotCephCfgImpl::save_default_rados_config() { - bool valid = rados_cfg.save_cfg() == 0 ? true : false; - rados_cfg.set_config_valid(valid); + bool valid = rados_cfg->save_cfg() == 0 ? true : false; + rados_cfg->set_config_valid(valid); return valid ? 0 : -1; } diff --git a/src/librmb/rados-dovecot-ceph-cfg-impl.h b/src/librmb/rados-dovecot-ceph-cfg-impl.h index 56358877..a53cd741 100644 --- a/src/librmb/rados-dovecot-ceph-cfg-impl.h +++ b/src/librmb/rados-dovecot-ceph-cfg-impl.h @@ -12,8 +12,8 @@ #ifndef SRC_LIBRMB_RADOS_DOVECOT_CEPH_CFG_IMPL_H_ #define SRC_LIBRMB_RADOS_DOVECOT_CEPH_CFG_IMPL_H_ -#include "rados-dovecot-ceph-cfg.h" -#include "rados-ceph-config.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../storage-interface/rados-ceph-config.h" #include #include @@ -22,11 +22,16 @@ namespace librmb { -class RadosDovecotCephCfgImpl : public RadosDovecotCephCfg { +class RadosDovecotCephCfgImpl : public storage_interface::RadosDovecotCephCfg { public: - explicit RadosDovecotCephCfgImpl(librados::IoCtx *io_ctx_); - RadosDovecotCephCfgImpl(RadosConfig &dovecot_cfg_, RadosCephConfig &rados_cfg_); - virtual ~RadosDovecotCephCfgImpl() {} + explicit RadosDovecotCephCfgImpl(storage_interface::RboxIoCtx *io_ctx_); + RadosDovecotCephCfgImpl(RadosConfig &dovecot_cfg_, storage_interface::RadosCephConfig *rados_cfg_); + virtual ~RadosDovecotCephCfgImpl() { + if(rados_cfg!=nullptr){ + delete rados_cfg; + rados_cfg=nullptr; + } + } // dovecot config @@ -52,69 +57,78 @@ class RadosDovecotCephCfgImpl : public RadosDovecotCephCfg { bool is_write_chunks() override { return dovecot_cfg.is_write_chunks(); } // rados config - bool is_user_mapping() override { return rados_cfg.is_user_mapping(); } + bool is_user_mapping() override { return rados_cfg->is_user_mapping(); } void set_config_valid(bool is_valid_) override { dovecot_cfg.set_config_valid(is_valid_); if (is_valid_) { - rados_cfg.set_cfg_object_name(dovecot_cfg.get_rbox_cfg_object_name()); + rados_cfg->set_cfg_object_name(dovecot_cfg.get_rbox_cfg_object_name()); } } int get_object_search_method() override { return std::stoi(dovecot_cfg.get_object_search_method()); } int get_object_search_threads() override { return std::stoi(dovecot_cfg.get_object_search_threads()); } + const std::string &get_log_file_path() override { return dovecot_cfg.get_logfile_path();} + const std::string &get_log_level() override {return dovecot_cfg.get_loglevel();} + void set_rbox_cfg_object_name(const std::string &value) override { dovecot_cfg.set_rbox_cfg_object_name(value); } std::map *get_config() override { return dovecot_cfg.get_config(); } - void set_io_ctx(librados::IoCtx *io_ctx_) override { rados_cfg.set_io_ctx(io_ctx_); } + void set_io_ctx_wrapper(storage_interface::RboxIoCtx *io_ctx_wrapper) override { rados_cfg->set_io_ctx(io_ctx_wrapper); } int load_rados_config() override { - // return dovecot_cfg.is_config_valid() ? rados_cfg.load_cfg() : -1; - return rados_cfg.load_cfg(); + // return dovecot_cfg.is_config_valid() ? rados_cfg->load_cfg() : -1; + return rados_cfg->load_cfg(); } int save_default_rados_config() override; - void set_user_mapping(bool value_) override { rados_cfg.set_user_mapping(value_); } - void set_user_ns(const std::string &ns) override { rados_cfg.set_user_ns(ns); } - std::string &get_user_ns() override { return rados_cfg.get_user_ns(); } - void set_user_suffix(const std::string &ns_suffix) override { rados_cfg.set_user_suffix(ns_suffix); } - std::string &get_user_suffix() override { return rados_cfg.get_user_suffix(); } + void set_user_mapping(bool value_) override { rados_cfg->set_user_mapping(value_); } + void set_user_ns(const std::string &ns) override { rados_cfg->set_user_ns(ns); } + std::string &get_user_ns() override { return rados_cfg->get_user_ns(); } + void set_user_suffix(const std::string &ns_suffix) override { rados_cfg->set_user_suffix(ns_suffix); } + std::string &get_user_suffix() override { return rados_cfg->get_user_suffix(); } void set_update_attributes(const std::string &update_attributes_) override { - rados_cfg.set_update_attributes(update_attributes_); + rados_cfg->set_update_attributes(update_attributes_); } - bool is_mail_attribute(enum rbox_metadata_key key) override { return rados_cfg.is_mail_attribute(key); } - bool is_updateable_attribute(enum rbox_metadata_key key) override { return rados_cfg.is_updateable_attribute(key); } - void update_mail_attributes(const char *value) override { rados_cfg.update_mail_attribute(value); } - void update_updatable_attributes(const char *value) override { rados_cfg.update_updateable_attribute(value); } - bool is_update_attributes() override { return rados_cfg.is_update_attributes(); } + bool is_mail_attribute(storage_interface::rbox_metadata_key key) override { return rados_cfg->is_mail_attribute(key); } + bool is_updateable_attribute(storage_interface::rbox_metadata_key key) override { return rados_cfg->is_updateable_attribute(key); } + void update_mail_attributes(const char *value) override { rados_cfg->update_mail_attribute(value); } + void update_updatable_attributes(const char *value) override { rados_cfg->update_updateable_attribute(value); } + bool is_update_attributes() override { return rados_cfg->is_update_attributes(); } - const std::string &get_metadata_storage_module() override { return rados_cfg.get_metadata_storage_module(); }; - const std::string &get_metadata_storage_attribute() override { return rados_cfg.get_metadata_storage_attribute(); }; + const std::string &get_metadata_storage_module() override { return rados_cfg->get_metadata_storage_module(); }; + const std::string &get_metadata_storage_attribute() override { return rados_cfg->get_metadata_storage_attribute(); }; - const std::string &get_mail_attributes_key() override { return rados_cfg.get_mail_attribute_key(); } - const std::string &get_updateable_attributes_key() override { return rados_cfg.get_updateable_attribute_key(); } - const std::string &get_update_attributes_key() override { return rados_cfg.get_update_attributes_key(); } + const std::string &get_mail_attributes_key() override { return rados_cfg->get_mail_attribute_key(); } + const std::string &get_updateable_attributes_key() override { return rados_cfg->get_updateable_attribute_key(); } + const std::string &get_update_attributes_key() override { return rados_cfg->get_update_attributes_key(); } - bool is_config_valid() override { return dovecot_cfg.is_config_valid() && rados_cfg.is_config_valid(); } - const std::string &get_public_namespace() override { return rados_cfg.get_public_namespace(); } + bool is_config_valid() override { return dovecot_cfg.is_config_valid() && rados_cfg->is_config_valid(); } + const std::string &get_public_namespace() override { return rados_cfg->get_public_namespace(); } void update_mail_attributes(const std::string &mail_attributes) { - rados_cfg.update_mail_attribute(mail_attributes.c_str()); + rados_cfg->update_mail_attribute(mail_attributes.c_str()); } void update_updatable_attributes(const std::string &updateable_attributes) { - rados_cfg.update_updateable_attribute(updateable_attributes.c_str()); - } - int save_object(const std::string &oid, librados::bufferlist &buffer) override { - return rados_cfg.save_object(oid, buffer); + rados_cfg->update_updateable_attribute(updateable_attributes.c_str()); } - int read_object(const std::string &oid, librados::bufferlist *buffer) override { - return rados_cfg.read_object(oid, buffer); + int save_object(const std::string &oid, std::istream &stream_buffer) override { + ceph::bufferlist buffer; + buffer.append(stream_buffer); + return rados_cfg->save_object(oid,(void*)(&buffer)); + } + int read_object(const std::string &oid,std::ostream &stream_buffer) override { + ceph::bufferlist *buffer; + std::stringstream ss; + ss << stream_buffer.rdbuf(); + buffer->append(ss.str()); + return rados_cfg->read_object(oid, (void*)buffer); } - void set_io_ctx_namespace(const std::string &namespace_) override { rados_cfg.set_io_ctx_namespace(namespace_); } + void set_io_ctx_namespace(const std::string &namespace_) override { rados_cfg->set_io_ctx_namespace(namespace_); } RadosConfig *get_dovecot_cfg() { return &dovecot_cfg; } - RadosCephConfig *get_rados_ceph_cfg() { return &rados_cfg; } + storage_interface::RadosCephConfig *get_rados_ceph_cfg() override { return rados_cfg; } private: RadosConfig dovecot_cfg; - RadosCephConfig rados_cfg; + storage_interface::RadosCephConfig *rados_cfg = nullptr; }; } /* namespace librmb */ diff --git a/src/librmb/rados-dovecot-config.cpp b/src/librmb/rados-dovecot-config.cpp index 81f84a08..392e8ba0 100644 --- a/src/librmb/rados-dovecot-config.cpp +++ b/src/librmb/rados-dovecot-config.cpp @@ -13,7 +13,7 @@ #include #include -#include "rados-types.h" +#include "../storage-interface/rados-types.h" namespace librmb { @@ -32,7 +32,9 @@ RadosConfig::RadosConfig() rbox_chunk_size("rbox_chunk_size"), rbox_write_method("rbox_write_method"), rbox_object_search_method("rbox_object_search_method"), - rbox_object_search_threads("rbox_object_search_threads") { + rbox_object_search_threads("rbox_object_search_threads"), + rbox_log_file_name("rbox_log_file_name"), + rbox_log_level("rbox_log_level") { config[pool_name] = "mail_storage"; config[index_pool_name] = "object_recovery"; @@ -48,12 +50,14 @@ RadosConfig::RadosConfig() config[rbox_write_method] = "0"; config[rbox_object_search_method] = "0"; config[rbox_object_search_threads] = "4"; + config[rbox_log_file_name] = "/var/log/dovecot_ceph_plugin.log"; + config[rbox_log_level] = "error"; is_valid = false; } -bool RadosConfig::string_contains_key(const std::string &str, enum rbox_metadata_key key) { - std::string value(librmb::rbox_metadata_key_to_char(key)); +bool RadosConfig::string_contains_key(const std::string &str, storage_interface::rbox_metadata_key key) { + std::string value(storage_interface::rbox_metadata_key_to_char(key)); return str.find(value) != std::string::npos; } diff --git a/src/librmb/rados-dovecot-config.h b/src/librmb/rados-dovecot-config.h index a9785911..5e33f253 100644 --- a/src/librmb/rados-dovecot-config.h +++ b/src/librmb/rados-dovecot-config.h @@ -15,7 +15,7 @@ #include #include -#include "rados-types.h" +#include "../storage-interface/rados-types.h" namespace librmb { /** @@ -55,6 +55,10 @@ class RadosConfig { const std::string &get_object_search_method() { return config[rbox_object_search_method]; } const std::string &get_object_search_threads() { return config[rbox_object_search_threads]; } + + const std::string &get_logfile_path() { return config[rbox_log_file_name];} + const std::string &get_loglevel() {return config[rbox_log_level];} + void update_metadata(const std::string &key, const char *value_); bool is_ceph_posix_bugfix_enabled() { return config[bugfix_cephfs_posix_hardlinks].compare("true") == 0 ? true : false; @@ -76,7 +80,7 @@ class RadosConfig { std::string to_string(); private: - bool string_contains_key(const std::string &str, enum rbox_metadata_key key); + bool string_contains_key(const std::string &str, storage_interface::rbox_metadata_key key); private: std::map config; @@ -95,6 +99,8 @@ class RadosConfig { std::string rbox_write_method; std::string rbox_object_search_method; std::string rbox_object_search_threads; + std::string rbox_log_file_name; + std::string rbox_log_level; bool is_valid; }; diff --git a/src/librmb/rados-mail-impl.cpp b/src/librmb/rados-mail-impl.cpp new file mode 100644 index 00000000..425982a5 --- /dev/null +++ b/src/librmb/rados-mail-impl.cpp @@ -0,0 +1,169 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#include "rados-mail-impl.h" + +#include + +#include +#include +#include "rados-util-impl.h" + +using std::endl; +using std::ostringstream; + +using librmb::RadosMailImpl; + +RadosMailImpl::RadosMailImpl() + : object_size(-1), + mail_buffer(nullptr), + save_date_rados(-1), + valid(true), + index_ref(false), + deprecated_uid(false), + restored(false), + lost_object(false) {} + +RadosMailImpl::~RadosMailImpl() { + /*NOTE: This means all metadata objects must be created with new!*/ + for(std::list::iterator + it=metadata_list.begin(); it!=metadata_list.end(); ++it){ + delete *it; + *it=nullptr; + } + for(std::list::iterator + it=extended_metadata_list.begin(); it!=extended_metadata_list.end(); ++it){ + delete *it; + *it=nullptr; + } +} + +std::string RadosMailImpl::to_string(const string& padding) { + librmb::RadosUtilsImpl rados_utils; + char* uid = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, this->get_metadata(), &uid); + char* recv_time_str = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, this->get_metadata(), &recv_time_str); + char* p_size = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, this->get_metadata(), &p_size); + char* v_size = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, this->get_metadata(), &v_size); + + char* rbox_version = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VERSION, this->get_metadata(), &rbox_version); + char* mailbox_guid = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, this->get_metadata(), &mailbox_guid); + char* mail_guid = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_GUID, this->get_metadata(), &mail_guid); + char* mb_orig_name = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, this->get_metadata(), &mb_orig_name); + + // string keywords = get_metadata(RBOX_METADATA_OLDV1_KEYWORDS); + char* flags = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, this->get_metadata(), &flags); + char* pvt_flags = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PVT_FLAGS, this->get_metadata(), &pvt_flags); + char* from_envelope = NULL; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_FROM_ENVELOPE, this->get_metadata(), &from_envelope); + + time_t ts = -1; + if (recv_time_str != NULL) { + try { + std::string recv(recv_time_str); + ts = static_cast(stol(recv)); + } catch (std::exception& ex) { + ts = -1; + } + } + ostringstream ss; + ss << endl; + if (!valid) { + ss << padding << "<<< MAIL OBJECT IS NOT VALID <<<<" << endl; + } + if (!index_ref) { + ss << padding << "<<< MAIL OBJECT HAS NO INDEX REFERENCE <<<< oid: " << oid << endl; + } + ss << padding << "MAIL: "; + if (uid != NULL) { + ss << static_cast(storage_interface::RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl; + ss << padding << " "; + } + ss << "oid = " << oid << endl; + string recv_time; + if (rados_utils.convert_time_t_to_str(ts, &recv_time) >= 0) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_RECEIVED_TIME) << "(receive_time)=" << recv_time + << "\n"; + } else { + if (recv_time_str != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_RECEIVED_TIME) + << "(receive_time)= INVALID DATE : '" << recv_time_str << "'" + << "\n"; + } + } + string save_time; + if (rados_utils.convert_time_t_to_str(save_date_rados, &save_time) >= 0) { + ss << padding << " " + << "save_time=" << save_time << "\n"; + } else { + ss << padding << " " + << "save_time= UNKNOWN '" << save_date_rados << "'\n"; + } + if (p_size != NULL && v_size != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_PHYSICAL_SIZE) << "(phy_size)=" << p_size << " " + << static_cast(storage_interface::RBOX_METADATA_VIRTUAL_SIZE) << "(v_size) = " << v_size << " stat_size=" << object_size + << endl; + } + if (mailbox_guid != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << mailbox_guid + << endl; + } + if (mb_orig_name != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_ORIG_MAILBOX) + << "(mailbox_orig_name)=" << mb_orig_name << endl; + } + if (mail_guid != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_GUID) << "(mail_guid)=" << mail_guid << endl; + } + if (rbox_version != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_VERSION) << "(rbox_version): " << rbox_version + << endl; + } + + if (extended_attrset.size() > 0) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS) << "(keywords): " << std::endl; + for (std::map::iterator iter = extended_attrset.begin(); iter != extended_attrset.end(); + ++iter) { + ss << " " << iter->first << " : " << ((ceph::bufferlist*)iter->second)->to_str() << endl; + } + } + + if (flags != NULL) { + uint8_t flags_; + if (rados_utils.string_to_flags(flags, &flags_)) { + std::string resolved_flags; + rados_utils.resolve_flags(flags_, &resolved_flags); + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_OLDV1_FLAGS) << "(flags): " << resolved_flags + << std::endl; + } + } + + if (pvt_flags != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_PVT_FLAGS) << "(private flags): " << pvt_flags + << endl; + } + + if (from_envelope != NULL) { + ss << padding << " " << static_cast(storage_interface::RBOX_METADATA_FROM_ENVELOPE) + << "(from envelope): " << from_envelope << endl; + } + + return ss.str(); +} diff --git a/src/librmb/rados-mail-impl.h b/src/librmb/rados-mail-impl.h new file mode 100644 index 00000000..c1851f90 --- /dev/null +++ b/src/librmb/rados-mail-impl.h @@ -0,0 +1,116 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_RADOS_MAIL_IMPL_H_ +#define SRC_LIBRMB_RADOS_MAIL_IMPL_H_ + +#include +#include +#include +#include +#include +#include "../storage-interface/rados-metadata.h" +#include "../storage-interface/rados-types.h" +#include "../storage-interface/rados-mail.h" + +namespace librmb { + +using std::map; +using std::string; + +/** + * Rados mail object + * + * ceph mail representation. + * + */ +class RadosMailImpl : public storage_interface::RadosMail { + public: + RadosMailImpl(); + virtual ~RadosMailImpl(); + void set_oid(const char* _oid) override { this->oid = _oid; } + void set_oid(const string& _oid) override { this->oid = _oid; } + void set_mail_size(const int _size) override { object_size = _size; } + void set_rados_save_date(const time_t& _save_date) override { this->save_date_rados = _save_date; } + string* get_oid() override { return &this->oid; } + int get_mail_size() override { return this->object_size; } + time_t get_rados_save_date() override { return this->save_date_rados; } + uint8_t get_guid_ref() override { return *this->guid; } + + /*! + * @return ptr to internal buffer . + */ + void set_mail_buffer(void* buffer) override { this->mail_buffer = buffer;} + void* get_mail_buffer() override { return this->mail_buffer;} + map* get_metadata() override { + return &attrset; + } + bool is_index_ref() override { return index_ref; } + void set_index_ref(bool ref) override { this->index_ref = ref; } + bool is_valid() override { return valid; } + void set_valid(bool valid_) override { valid = valid_; } + + bool is_restored() override { return restored; } + void set_restored(bool restored_) override { restored = restored_; } + + bool is_lost_object() override { return lost_object; } + void set_lost_object(bool is_lost_object) override { lost_object = is_lost_object; } + + string to_string(const string& padding) override; + void add_metadata(const storage_interface::RadosMetadata* metadata_) override { + metadata_list.push_back(const_cast(metadata_)); + attrset[metadata_->get_key()] = metadata_->get_buffer(); + } + bool is_deprecated_uid() override {return deprecated_uid;} + void set_deprecated_uid(bool deprecated_uid_) override {deprecated_uid = deprecated_uid_;} + /*! + * Some metadata isn't saved as xattribute (default). To access those, get_extended_metadata can + * be used. + */ + map* get_extended_metadata() override { + return &this->extended_attrset; + } + /*! + * Save metadata to extended metadata store currently omap + * @param[in] metadata valid radosMetadata. + */ + void add_extended_metadata(const storage_interface::RadosMetadata *ext_metadata) override { + extended_metadata_list.push_back(const_cast(ext_metadata)); + extended_attrset[ext_metadata->get_key()] = ext_metadata->get_buffer(); + } + + const string get_extended_metadata(const string& key) override { + if (extended_attrset.find(key) != extended_attrset.end()) { + return ((ceph::bufferlist*)extended_attrset[key])->to_str(); + } + return nullptr; + } + + private: + string oid; + uint8_t guid[GUID_128_SIZE] = {}; + int object_size; // byte + void* mail_buffer; + time_t save_date_rados; + std::list metadata_list; + std::list extended_metadata_list; + map attrset; + map extended_attrset; + bool valid; + bool index_ref; + bool deprecated_uid; + bool restored; + bool lost_object; // is this a lost object for re-sync. +}; + +} // namespace librmb + +#endif // SRC_LIBRMB_RADOS_MAIL_IMPL_H_ diff --git a/src/librmb/rados-mail.cpp b/src/librmb/rados-mail.cpp deleted file mode 100644 index 836d56a2..00000000 --- a/src/librmb/rados-mail.cpp +++ /dev/null @@ -1,159 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#include "rados-mail.h" - -#include - -#include -#include -#include "rados-util.h" - -using std::endl; -using std::ostringstream; - -using librmb::RadosMail; - -RadosMail::RadosMail() - : object_size(-1), - completion(nullptr), - write_operation(nullptr), - active_op(0), - mail_buffer(nullptr), - save_date_rados(-1), - valid(true), - index_ref(false), - deprecated_uid(false), - restored(false), - lost_object(false) {} - -RadosMail::~RadosMail() {} - -std::string RadosMail::to_string(const string& padding) { - char* uid = NULL; - RadosUtils::get_metadata(RBOX_METADATA_MAIL_UID, &attrset, &uid); - char* recv_time_str = NULL; - RadosUtils::get_metadata(RBOX_METADATA_RECEIVED_TIME, &attrset, &recv_time_str); - char* p_size = NULL; - RadosUtils::get_metadata(RBOX_METADATA_PHYSICAL_SIZE, &attrset, &p_size); - char* v_size = NULL; - RadosUtils::get_metadata(RBOX_METADATA_VIRTUAL_SIZE, &attrset, &v_size); - - char* rbox_version = NULL; - RadosUtils::get_metadata(RBOX_METADATA_VERSION, &attrset, &rbox_version); - char* mailbox_guid = NULL; - RadosUtils::get_metadata(RBOX_METADATA_MAILBOX_GUID, &attrset, &mailbox_guid); - char* mail_guid = NULL; - RadosUtils::get_metadata(RBOX_METADATA_GUID, &attrset, &mail_guid); - char* mb_orig_name = NULL; - RadosUtils::get_metadata(RBOX_METADATA_ORIG_MAILBOX, &attrset, &mb_orig_name); - - // string keywords = get_metadata(RBOX_METADATA_OLDV1_KEYWORDS); - char* flags = NULL; - RadosUtils::get_metadata(RBOX_METADATA_OLDV1_FLAGS, &attrset, &flags); - char* pvt_flags = NULL; - RadosUtils::get_metadata(RBOX_METADATA_PVT_FLAGS, &attrset, &pvt_flags); - char* from_envelope = NULL; - RadosUtils::get_metadata(RBOX_METADATA_FROM_ENVELOPE, &attrset, &from_envelope); - - time_t ts = -1; - if (recv_time_str != NULL) { - try { - std::string recv(recv_time_str); - ts = static_cast(stol(recv)); - } catch (std::exception& ex) { - ts = -1; - } - } - ostringstream ss; - ss << endl; - if (!valid) { - ss << padding << "<<< MAIL OBJECT IS NOT VALID <<<<" << endl; - } - if (!index_ref) { - ss << padding << "<<< MAIL OBJECT HAS NO INDEX REFERENCE <<<< oid: " << oid << endl; - } - ss << padding << "MAIL: "; - if (uid != NULL) { - ss << static_cast(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl; - ss << padding << " "; - } - ss << "oid = " << oid << endl; - string recv_time; - if (RadosUtils::convert_time_t_to_str(ts, &recv_time) >= 0) { - ss << padding << " " << static_cast(RBOX_METADATA_RECEIVED_TIME) << "(receive_time)=" << recv_time - << "\n"; - } else { - if (recv_time_str != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_RECEIVED_TIME) - << "(receive_time)= INVALID DATE : '" << recv_time_str << "'" - << "\n"; - } - } - string save_time; - if (RadosUtils::convert_time_t_to_str(save_date_rados, &save_time) >= 0) { - ss << padding << " " - << "save_time=" << save_time << "\n"; - } else { - ss << padding << " " - << "save_time= UNKNOWN '" << save_date_rados << "'\n"; - } - if (p_size != NULL && v_size != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_PHYSICAL_SIZE) << "(phy_size)=" << p_size << " " - << static_cast(RBOX_METADATA_VIRTUAL_SIZE) << "(v_size) = " << v_size << " stat_size=" << object_size - << endl; - } - if (mailbox_guid != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << mailbox_guid - << endl; - } - if (mb_orig_name != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_ORIG_MAILBOX) - << "(mailbox_orig_name)=" << mb_orig_name << endl; - } - if (mail_guid != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_GUID) << "(mail_guid)=" << mail_guid << endl; - } - if (rbox_version != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_VERSION) << "(rbox_version): " << rbox_version - << endl; - } - - if (extended_attrset.size() > 0) { - ss << padding << " " << static_cast(RBOX_METADATA_OLDV1_KEYWORDS) << "(keywords): " << std::endl; - for (std::map::iterator iter = extended_attrset.begin(); iter != extended_attrset.end(); - ++iter) { - ss << " " << iter->first << " : " << iter->second.to_str() << endl; - } - } - - if (flags != NULL) { - uint8_t flags_; - if (RadosUtils::string_to_flags(flags, &flags_)) { - std::string resolved_flags; - RadosUtils::resolve_flags(flags_, &resolved_flags); - ss << padding << " " << static_cast(RBOX_METADATA_OLDV1_FLAGS) << "(flags): " << resolved_flags - << std::endl; - } - } - - if (pvt_flags != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_PVT_FLAGS) << "(private flags): " << pvt_flags - << endl; - } - - if (from_envelope != NULL) { - ss << padding << " " << static_cast(RBOX_METADATA_FROM_ENVELOPE) - << "(from envelope): " << from_envelope << endl; - } - - return ss.str(); -} diff --git a/src/librmb/rados-mail.h b/src/librmb/rados-mail.h deleted file mode 100644 index 62d4fb25..00000000 --- a/src/librmb/rados-mail.h +++ /dev/null @@ -1,139 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_RADOS_MAIL_H_ -#define SRC_LIBRMB_RADOS_MAIL_H_ - -#include -#include -#include -#include -#include "rados-metadata.h" -#include "rados-types.h" -#include - -namespace librmb { - -using std::map; -using std::string; - -using librados::AioCompletion; -using librados::ObjectWriteOperation; - -/** - * Rados mail object - * - * ceph mail representation. - * - */ -class RadosMail { - public: - RadosMail(); - virtual ~RadosMail(); - void set_oid(const char* _oid) { this->oid = _oid; } - void set_oid(const string& _oid) { this->oid = _oid; } - void set_mail_size(const int _size) { object_size = _size; } - void set_active_op(int num_write_op) { this->active_op = num_write_op; } - void set_rados_save_date(const time_t& _save_date) { this->save_date_rados = _save_date; } - - string* get_oid() { return &this->oid; } - int get_mail_size() { return this->object_size; } - - time_t get_rados_save_date() { return this->save_date_rados; } - uint8_t get_guid_ref() { return *this->guid; } - /*! - * @return ptr to internal buffer . - */ - librados::bufferlist* get_mail_buffer() { return this->mail_buffer; } - void set_mail_buffer(librados::bufferlist* buffer) { this->mail_buffer = buffer; } - - map* get_metadata() { return &this->attrset; } - - AioCompletion* get_completion() { return completion; } - - ObjectWriteOperation* get_write_operation() { return write_operation; } - void set_write_operation(ObjectWriteOperation* write_operation_) { this->write_operation = write_operation_; } - void set_completion(AioCompletion* completion_) { this->completion = completion_; } - - /*! - * @return reference to all write operations related with this object - */ - - /* void get_metadata(const std::string& key, char** value) { - if (attrset.find(key) != attrset.end()) { - *value = attrset[key].c_str(); - return; - } - *value = NULL; - } - void get_metadata(rbox_metadata_key key, char** value) { - string str_key(librmb::rbox_metadata_key_to_char(key)); - get_metadata(str_key, value); - }*/ - - bool is_index_ref() { return index_ref; } - void set_index_ref(bool ref) { this->index_ref = ref; } - bool is_valid() { return valid; } - void set_valid(bool valid_) { valid = valid_; } - - bool is_restored() { return restored; } - void set_restored(bool restored_) { restored = restored_; } - - bool is_lost_object() { return lost_object; } - void set_lost_object(bool is_lost_object) { lost_object = is_lost_object; } - - bool has_active_op() { return active_op > 0; } - int get_num_active_op() { return active_op; } - string to_string(const string& padding); - void add_metadata(const RadosMetadata& metadata) { attrset[metadata.key] = metadata.bl; } - bool is_deprecated_uid() {return deprecated_uid;} - void set_deprecated_uid(bool deprecated_uid_) {deprecated_uid = deprecated_uid_;} - /*! - * Some metadata isn't saved as xattribute (default). To access those, get_extended_metadata can - * be used. - */ - map* get_extended_metadata() { return &this->extended_attrset; } - /*! - * Save metadata to extended metadata store currently omap - * @param[in] metadata valid radosMetadata. - */ - void add_extended_metadata(const RadosMetadata& metadata) { extended_attrset[metadata.key] = metadata.bl; } - - const string get_extended_metadata(const string& key) { - if (extended_attrset.find(key) != extended_attrset.end()) { - return extended_attrset[key].to_str(); - } - return nullptr; - } - - private: - string oid; - uint8_t guid[GUID_128_SIZE] = {}; - int object_size; // byte - AioCompletion* completion; - ObjectWriteOperation* write_operation; - - int active_op; - ceph::bufferlist* mail_buffer; - time_t save_date_rados; - - map attrset; - map extended_attrset; - bool valid; - bool index_ref; - bool deprecated_uid; - bool restored; - bool lost_object; // is this a lost object for re-sync. -}; - -} // namespace librmb - -#endif // SRC_LIBRMB_RADOS_MAIL_H_ diff --git a/src/librmb/rados-metadata-impl.h b/src/librmb/rados-metadata-impl.h new file mode 100644 index 00000000..dbb3062b --- /dev/null +++ b/src/librmb/rados-metadata-impl.h @@ -0,0 +1,136 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_RADOS_METADATA_H_ +#define SRC_LIBRMB_RADOS_METADATA_H_ + +#include "../storage-interface/rados-types.h" +#include +#include "time.h" +#include +#include +#include +#include "../storage-interface/rados-metadata.h" + +namespace librmb { + +class RadosMetadataImpl : public storage_interface::RadosMetadata{ + public: + RadosMetadataImpl() {} + RadosMetadataImpl(std::string& key_, std::string& value_){ + this->key = key_; + (*bl).clear(); + (*bl).append(value_.c_str(), value_.length() + 1); + } + RadosMetadataImpl(storage_interface::rbox_metadata_key _key, const std::string& val) { convert(_key, val); } + + RadosMetadataImpl(storage_interface::rbox_metadata_key _key, const time_t& val) { convert(_key, val); } + + RadosMetadataImpl(storage_interface::rbox_metadata_key _key, const char* val) { convert(_key, val); } + + RadosMetadataImpl(storage_interface::rbox_metadata_key _key, const uint& val) { convert(_key, val); } + + RadosMetadataImpl(storage_interface::rbox_metadata_key _key, const size_t& val) { convert(_key, val); } + RadosMetadataImpl(storage_interface::rbox_metadata_key _key, const int val) { convert(_key, val); } + ~RadosMetadataImpl() { + if(bl != nullptr){ + delete bl; + bl = nullptr; + } + } + + void convert(const char* value, time_t* t) override { + if (t != NULL) { + std::istringstream stream(value); + stream >> *t; + } + } + + bool from_string(const std::string& str) override { + std::stringstream ss(str); + std::string item; + std::vector token; + while (std::getline(ss, item, '=')) { + token.push_back(item); + } + if (token.size() != 2 || this == nullptr) { + return false; + } + this->key = token[0]; + this->bl->append(token[1]); + return true; + } + + std::string to_string() override { + std::stringstream str; + str << key << "=" << (*bl).to_str().substr(0, (*bl).length() - 1); + return str.str(); + } + + public: + ceph::bufferlist *bl= new ceph::bufferlist(); + std::string key; + + void* get_buffer() override{ + return (void*)bl; + }; + + std::string& get_key() override{ + return key; + }; + + void set_key(std::string& key_) override{ + this->key=key_; + }; + + void convert(storage_interface::rbox_metadata_key _key, const std::string& val) override { + (*bl).clear(); + key = storage_interface::rbox_metadata_key_to_char(_key); + (*bl).append(val.c_str(), val.length() + 1); + } + + void convert(storage_interface::rbox_metadata_key _key, const time_t& time) override { + (*bl).clear(); + key = storage_interface::rbox_metadata_key_to_char(_key); + std::string time_ = std::to_string(time); + (*bl).append(time_.c_str(), time_.length() + 1); + } + + void convert(storage_interface::rbox_metadata_key _key, const char* value) override { + (*bl).clear(); + key = storage_interface::rbox_metadata_key_to_char(_key); + std::string str = value; + (*bl).append(str.c_str(), str.length() + 1); + } + + void convert(storage_interface::rbox_metadata_key _key, const uint& value) override { + (*bl).clear(); + key = storage_interface::rbox_metadata_key_to_char(_key); + std::string val = std::to_string(value); + (*bl).append(val.c_str(), val.length() + 1); + } + + void convert(storage_interface::rbox_metadata_key _key, const size_t& value) override { + (*bl).clear(); + key = storage_interface::rbox_metadata_key_to_char(_key); + std::string val = std::to_string(static_cast(value)); + (*bl).append(val.c_str(), val.length() + 1); + } + + void convert(storage_interface::rbox_metadata_key _key, const int value) override { + (*bl).clear(); + key = storage_interface::rbox_metadata_key_to_char(_key); + std::string val = std::to_string(value); + (*bl).append(val.c_str(), val.length() + 1); + } +}; +} // end namespace +#endif /* SRC_LIBRMB_RADOS_METADATA_H_ */ diff --git a/src/librmb/rados-metadata-storage-default.cpp b/src/librmb/rados-metadata-storage-default.cpp index 31c92cc2..56ca91fa 100644 --- a/src/librmb/rados-metadata-storage-default.cpp +++ b/src/librmb/rados-metadata-storage-default.cpp @@ -10,17 +10,20 @@ */ #include "rados-metadata-storage-default.h" -#include "rados-util.h" +#include "rados-util-impl.h" #include +#include "../storage-interface/rados-mail.h" +#include "rados-metadata-impl.h" +#include "rados-mail-impl.h" namespace librmb { std::string RadosMetadataStorageDefault::module_name = "default"; -RadosMetadataStorageDefault::RadosMetadataStorageDefault(librados::IoCtx *io_ctx_) { this->io_ctx = io_ctx_; } +RadosMetadataStorageDefault::RadosMetadataStorageDefault(storage_interface::RboxIoCtx *io_ctx_wrapper) { this->io_ctx_wrapper = io_ctx_wrapper; } RadosMetadataStorageDefault::~RadosMetadataStorageDefault() {} -int RadosMetadataStorageDefault::load_metadata(RadosMail *mail) { +int RadosMetadataStorageDefault::load_metadata(storage_interface::RadosMail *mail) { int ret = -1; if (mail == nullptr) { return ret; @@ -28,70 +31,92 @@ int RadosMetadataStorageDefault::load_metadata(RadosMail *mail) { if (mail->get_metadata()->size() > 0) { mail->get_metadata()->clear(); } - ret = io_ctx->getxattrs(*mail->get_oid(), *mail->get_metadata()); + std::map ceph_metadata; - if (ret >= 0) { - ret = RadosUtils::get_all_keys_and_values(io_ctx, *mail->get_oid(), mail->get_extended_metadata()); + ret = io_ctx_wrapper->getxattrs(*mail->get_oid(),ceph_metadata); + if(ret >= 0){ + for(std::map::iterator it=ceph_metadata.begin(); it!=ceph_metadata.end(); ++it){ + std::string key = (*it).first; + std::string value = std::string((*it).second.c_str()); + storage_interface::RadosMetadata *xattr= new RadosMetadataImpl(key,value); + + mail->add_metadata(xattr); + } + librmb::RadosUtilsImpl rados_utils; + ret = rados_utils.get_all_keys_and_values(io_ctx_wrapper, *mail->get_oid(), mail); } return ret; } -int RadosMetadataStorageDefault::set_metadata(RadosMail *mail, RadosMetadata &xattr) { - mail->add_metadata(xattr); - return io_ctx->setxattr(*mail->get_oid(), xattr.key.c_str(), xattr.bl); -} +// xattr is deprecated +int RadosMetadataStorageDefault::set_metadata(storage_interface::RadosMail *mail) { + librados::ObjectWriteOperation write_op_xattr; + save_metadata(&write_op_xattr, mail); + return io_ctx_wrapper->operate(*mail->get_oid(), &write_op_xattr); -int RadosMetadataStorageDefault::set_metadata(RadosMail *mail, RadosMetadata &xattr, - librados::ObjectWriteOperation *write_op) { - mail->add_metadata(xattr); - if (mail->get_completion() == nullptr) { - mail->set_completion(librados::Rados::aio_create_completion()); - mail->set_active_op(1); - } - return io_ctx->aio_operate(*mail->get_oid(), mail->get_completion(), write_op); } -void RadosMetadataStorageDefault::save_metadata(librados::ObjectWriteOperation *write_op, RadosMail *mail) { +void RadosMetadataStorageDefault::save_metadata(librados::ObjectWriteOperation *write_op, storage_interface::RadosMail *mail) { // update metadata - for (std::map::iterator it = mail->get_metadata()->begin(); - it != mail->get_metadata()->end(); ++it) { - write_op->setxattr((*it).first.c_str(), (*it).second); + ceph::bufferlist bl; + std::string buff_str; + + for (std::map::iterator it = mail->get_metadata()->begin(); + it != mail->get_metadata()->end(); ++it) { + buff_str = std::string(((ceph::bufferlist*)(*it).second)->c_str()); + bl.append(buff_str.c_str(), buff_str.size() + 1); + const char* key = (*it).first.c_str(); + std::cout << bl.c_str() <setxattr(key,bl); + bl.clear(); } + if (mail->get_extended_metadata()->size() > 0) { - write_op->omap_set(*mail->get_extended_metadata()); + std::map ceph_metadata; + for(std::map::iterator it=mail->get_extended_metadata()->begin(); + it!=mail->get_extended_metadata()->end(); ++it){ + buff_str = std::string(((ceph::bufferlist*)(*it).second)->c_str()); + bl.append(buff_str.c_str(), buff_str.size() +1); + ceph_metadata[(*it).first]=bl; + bl.clear(); + } + write_op->omap_set(ceph_metadata); } } -bool RadosMetadataStorageDefault::update_metadata(const std::string &oid, std::list &to_update) { - librados::ObjectWriteOperation write_op; - librados::AioCompletion *completion = librados::Rados::aio_create_completion(); +bool RadosMetadataStorageDefault::update_metadata(const std::string &oid, std::list &to_update) { + librados::ObjectWriteOperation write_op; + ceph::bufferlist bl; + std::string buff_str; // update metadata - for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { - write_op.setxattr((*it).key.c_str(), (*it).bl); + for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { + buff_str = std::string(((ceph::bufferlist*)(*it)->get_buffer())->c_str()); + bl.append(buff_str.c_str(), buff_str.size() + 1); + write_op.setxattr((*it)->get_key().c_str(), bl); + bl.clear(); } - int ret = io_ctx->aio_operate(oid, completion, &write_op); - completion->wait_for_complete(); - completion->release(); + int ret = io_ctx_wrapper->operate(oid, &write_op); return ret == 0; } -int RadosMetadataStorageDefault::update_keyword_metadata(const std::string &oid, RadosMetadata *metadata) { +int RadosMetadataStorageDefault::update_keyword_metadata(const std::string &oid, storage_interface::RadosMetadata *metadata) { int ret = -1; + ceph::bufferlist bl; + std::string buff_str; if (metadata != nullptr) { std::map map; - map.insert(std::pair(metadata->key, metadata->bl)); - ret = io_ctx->omap_set(oid, map); + buff_str = std::string(((ceph::bufferlist*)metadata->get_buffer())->c_str()); + bl.append(buff_str.c_str(), buff_str.size() +1); + map.insert(std::pair(metadata->get_key(),bl)); + ret = io_ctx_wrapper->omap_set(map,oid); + bl.clear(); } return ret; } int RadosMetadataStorageDefault::remove_keyword_metadata(const std::string &oid, std::string &key) { std::set keys; keys.insert(key); - return io_ctx->omap_rm_keys(oid, keys); + return io_ctx_wrapper->omap_rm_keys(oid, keys); } -int RadosMetadataStorageDefault::load_keyword_metadata(const std::string &oid, std::set &keys, - std::map *metadata) { - return io_ctx->omap_get_vals_by_keys(oid, keys, metadata); -} - } /* namespace librmb */ diff --git a/src/librmb/rados-metadata-storage-default.h b/src/librmb/rados-metadata-storage-default.h index 1d4b9040..25420c1c 100644 --- a/src/librmb/rados-metadata-storage-default.h +++ b/src/librmb/rados-metadata-storage-default.h @@ -15,37 +15,36 @@ #include #include #include -#include "rados-metadata-storage-module.h" +#include "../storage-interface/rados-metadata-storage-module.h" +#include "../storage-interface/rados-mail.h" +#include "../storage-interface/rados-metadata.h" +#include "../storage-interface/rbox-io-ctx.h" namespace librmb { /** * Implements the default storage of mail metadata. * - * Each metadata attribute is saved as single xattribute. + * Each metadata attribute is saved as single xattribute.io_ctx_wrapper_wrapper * */ -class RadosMetadataStorageDefault : public RadosStorageMetadataModule { +class RadosMetadataStorageDefault : public storage_interface::RadosStorageMetadataModule { public: - explicit RadosMetadataStorageDefault(librados::IoCtx *io_ctx_); + explicit RadosMetadataStorageDefault(storage_interface::RboxIoCtx *io_ctx_wrapper); virtual ~RadosMetadataStorageDefault(); - void set_io_ctx(librados::IoCtx *io_ctx_) override { this->io_ctx = io_ctx_; } + void set_io_ctx(storage_interface::RboxIoCtx *io_ctx_wrapper) override { this->io_ctx_wrapper =io_ctx_wrapper; } - int load_metadata(RadosMail *mail) override; - int set_metadata(RadosMail *mail, RadosMetadata &xattr) override; - bool update_metadata(const std::string &oid, std::list &to_update) override; - void save_metadata(librados::ObjectWriteOperation *write_op, RadosMail *mail) override; - int set_metadata(RadosMail *mail, RadosMetadata &xattr, librados::ObjectWriteOperation *write_op) override; - - int update_keyword_metadata(const std::string &oid, RadosMetadata *metadata) override; + int load_metadata(storage_interface::RadosMail *mail) override; + int set_metadata(storage_interface::RadosMail *mail) override; + bool update_metadata(const std::string &oid, std::list &to_update) override; + void save_metadata(librados::ObjectWriteOperation *write_op, storage_interface::RadosMail *mail); + int update_keyword_metadata(const std::string &oid, storage_interface::RadosMetadata *metadata) override; int remove_keyword_metadata(const std::string &oid, std::string &key) override; - int load_keyword_metadata(const std::string &oid, std::set &keys, - std::map *metadata) override; public: static std::string module_name; private: - librados::IoCtx *io_ctx; + storage_interface::RboxIoCtx *io_ctx_wrapper; }; } /* namespace librmb */ diff --git a/src/librmb/rados-metadata-storage-ima.cpp b/src/librmb/rados-metadata-storage-ima.cpp index 7aa98c8c..3d307771 100644 --- a/src/librmb/rados-metadata-storage-ima.cpp +++ b/src/librmb/rados-metadata-storage-ima.cpp @@ -10,23 +10,26 @@ */ #include "rados-metadata-storage-ima.h" -#include "rados-util.h" +#include "rados-util-impl.h" #include #include #include +#include "../storage-interface/rados-mail.h" +#include "rados-mail-impl.h" +#include"rados-metadata-impl.h" std::string librmb::RadosMetadataStorageIma::module_name = "ima"; std::string librmb::RadosMetadataStorageIma::keyword_key = "K"; namespace librmb { -RadosMetadataStorageIma::RadosMetadataStorageIma(librados::IoCtx *io_ctx_, RadosDovecotCephCfg *cfg_) { - this->io_ctx = io_ctx_; +RadosMetadataStorageIma::RadosMetadataStorageIma(storage_interface::RboxIoCtx *io_ctx_wrapper_, storage_interface::RadosDovecotCephCfg *cfg_) { + this->io_ctx_wrapper = io_ctx_wrapper_; this->cfg = cfg_; } RadosMetadataStorageIma::~RadosMetadataStorageIma() {} -int RadosMetadataStorageIma::parse_attribute(RadosMail *mail, json_t *root) { +int RadosMetadataStorageIma::parse_attribute(storage_interface::RadosMail *mail, json_t *root) { std::string key; void *iter = json_object_iter(root); @@ -43,21 +46,21 @@ int RadosMetadataStorageIma::parse_attribute(RadosMail *mail, json_t *root) { json_t *keyword_value = json_object_iter_value(keyword_iter); bl.append(json_string_value(keyword_value)); - (*mail->get_extended_metadata())[_keyword_key] = bl; + (*mail->get_extended_metadata())[_keyword_key] = (void*)&bl; keyword_iter = json_object_iter_next(value, keyword_iter); } } else { librados::bufferlist bl; bl.append(json_string_value(value)); - (*mail->get_metadata())[key] = bl; + (*mail->get_metadata())[key] = (void*)&bl; } iter = json_object_iter_next(root, iter); } return 0; } -int RadosMetadataStorageIma::load_metadata(RadosMail *mail) { +int RadosMetadataStorageIma::load_metadata(storage_interface::RadosMail *mail) { if (mail == nullptr) { return -1; } @@ -65,12 +68,12 @@ int RadosMetadataStorageIma::load_metadata(RadosMail *mail) { return 0; } - std::map attr; + std::map attr; // retry mechanism .. int max_retry = 10; int ret = -1; for(int i=0;igetxattrs(*mail->get_oid(), attr); + ret = io_ctx_wrapper->getxattrs(*mail->get_oid(), attr); if(ret >= 0){ break; } @@ -93,76 +96,74 @@ int RadosMetadataStorageIma::load_metadata(RadosMail *mail) { } // load other attributes - for (std::map::iterator it = attr.begin(); it != attr.end(); ++it) { + for (std::map::iterator it = attr.begin(); it != attr.end(); ++it) { if ((*it).first.compare(cfg->get_metadata_storage_attribute()) != 0) { - (*mail->get_metadata())[(*it).first] = (*it).second; + std::string key = (*it).first; + std::string value = std::string((*it).second.c_str()); + storage_interface::RadosMetadata *xattr= new librmb::RadosMetadataImpl(key,value); + mail->add_metadata(xattr); } } // load other omap values. - if (cfg->is_updateable_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS)) { - ret = RadosUtils::get_all_keys_and_values(io_ctx, *mail->get_oid(), mail->get_extended_metadata()); + if (cfg->is_updateable_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS)) { + librmb::RadosUtilsImpl rados_utils; + ret = rados_utils.get_all_keys_and_values(io_ctx_wrapper, *mail->get_oid(), mail); } - return ret; } -// it is required that mail->get_metadata is up to date before update. -int RadosMetadataStorageIma::set_metadata(RadosMail *mail, RadosMetadata &xattr) { - enum rbox_metadata_key k = static_cast(*xattr.key.c_str()); - if (!cfg->is_updateable_attribute(k)) { - mail->add_metadata(xattr); - librados::ObjectWriteOperation op; - save_metadata(&op, mail); - return io_ctx->operate(*mail->get_oid(), &op); - } else { - return io_ctx->setxattr(*mail->get_oid(), xattr.key.c_str(), xattr.bl); - } +int RadosMetadataStorageIma::set_metadata(storage_interface::RadosMail *mail) { + librados::ObjectWriteOperation write_op_xattr; + save_metadata(&write_op_xattr, mail); + return io_ctx_wrapper->operate(*mail->get_oid(), &write_op_xattr); } -// TODO(jrse): make set metadata async! -int RadosMetadataStorageIma::set_metadata(RadosMail *mail, RadosMetadata &xattr, - librados::ObjectWriteOperation *write_op) { - enum rbox_metadata_key k = static_cast(*xattr.key.c_str()); - if (!cfg->is_updateable_attribute(k)) { - mail->add_metadata(xattr); - librados::ObjectWriteOperation op; - save_metadata(&op, mail); - return io_ctx->operate(*mail->get_oid(), &op); - } else { - return io_ctx->setxattr(*mail->get_oid(), xattr.key.c_str(), xattr.bl); - } - -} // namespace librmb - -void RadosMetadataStorageIma::save_metadata(librados::ObjectWriteOperation *write_op, RadosMail *mail) { +void RadosMetadataStorageIma::save_metadata(librados::ObjectWriteOperation *write_op, storage_interface::RadosMail *mail) { char *s = NULL; json_t *root = json_object(); librados::bufferlist bl; + std::string buff_str; + ceph::bufferlist temp_bl; + if (mail->get_metadata()->size() > 0) { - for (std::map::iterator it = mail->get_metadata()->begin(); + for (std::map::iterator it = mail->get_metadata()->begin(); it != mail->get_metadata()->end(); ++it) { - enum rbox_metadata_key k = static_cast(*(*it).first.c_str()); + storage_interface::rbox_metadata_key k = static_cast(*(*it).first.c_str()); + buff_str = std::string(((ceph::bufferlist*)(*it).second)->c_str()); + temp_bl.append(buff_str.c_str(), buff_str.size() + 1); if (!cfg->is_updateable_attribute(k) || !cfg->is_update_attributes()) { - json_object_set_new(root, (*it).first.c_str(), json_string((*it).second.to_str().c_str())); + json_object_set_new(root, (*it).first.c_str(), json_string(temp_bl.to_str().c_str())); + temp_bl.clear(); } else { - write_op->setxattr((*it).first.c_str(), (*it).second); + write_op->setxattr((*it).first.c_str(), temp_bl); + temp_bl.clear(); } } } json_t *keyword = json_object(); // build extended Metadata object if (mail->get_extended_metadata()->size() > 0) { - if (!cfg->is_updateable_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS) || !cfg->is_update_attributes()) { - for (std::map::iterator it = mail->get_extended_metadata()->begin(); - it != mail->get_extended_metadata()->end(); ++it) { - json_object_set_new(keyword, (*it).first.c_str(), json_string((*it).second.to_str().c_str())); - } - json_object_set_new(root, RadosMetadataStorageIma::keyword_key.c_str(), keyword); - } else { - write_op->omap_set(*mail->get_extended_metadata()); + bool json = (!cfg->is_updateable_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS) || !cfg->is_update_attributes()); + std::map ceph_metadata; + for (std::map::iterator it = mail->get_extended_metadata()->begin(); + it != mail->get_extended_metadata()->end(); ++it) { + buff_str = std::string(((ceph::bufferlist*)(*it).second)->c_str()); + temp_bl.append(buff_str.c_str(), buff_str.size() +1); + if(json){ + json_object_set_new(keyword, (*it).first.c_str(), json_string(temp_bl.to_str().c_str())); + temp_bl.clear(); + }else{ + ceph_metadata[(*it).first]=temp_bl; + temp_bl.clear(); + } } - } + if(json){ + json_object_set_new(root, RadosMetadataStorageIma::keyword_key.c_str(), keyword); + }else{ + write_op->omap_set(ceph_metadata); + } + } s = json_dumps(root, 0); bl.append(s); @@ -173,40 +174,35 @@ void RadosMetadataStorageIma::save_metadata(librados::ObjectWriteOperation *writ write_op->setxattr(cfg->get_metadata_storage_attribute().c_str(), bl); } -bool RadosMetadataStorageIma::update_metadata(const std::string &oid, std::list &to_update) { +bool RadosMetadataStorageIma::update_metadata(const std::string &oid, std::list &to_update) { librados::ObjectWriteOperation write_op; - - if (to_update.empty()) { - return true; - } - - RadosMail obj; - obj.set_oid(oid); - load_metadata(&obj); + ceph::bufferlist bl; + std::string buff_str; // update metadata - for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { - (*obj.get_extended_metadata())[(*it).key] = (*it).bl; + for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { + buff_str = std::string(((ceph::bufferlist*)(*it)->get_buffer())->c_str()); + bl.append(buff_str.c_str(), buff_str.size() + 1); + write_op.setxattr((*it)->get_key().c_str(), bl); + bl.clear(); } - // write update - save_metadata(&write_op, &obj); - librados::AioCompletion *completion = librados::Rados::aio_create_completion(); - - //TODO: do we need a retry mechanism here? - int ret = io_ctx->aio_operate(oid, completion, &write_op); - completion->wait_for_complete(); - completion->release(); + int ret = io_ctx_wrapper->operate(oid, &write_op); return ret == 0; } -int RadosMetadataStorageIma::update_keyword_metadata(const std::string &oid, RadosMetadata *metadata) { +int RadosMetadataStorageIma::update_keyword_metadata(const std::string &oid, storage_interface::RadosMetadata *metadata) { int ret = -1; + ceph::bufferlist bl; + std::string buff_str; if (metadata != nullptr) { - if (!cfg->is_updateable_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS) || !cfg->is_update_attributes()) { + if (!cfg->is_updateable_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS) || !cfg->is_update_attributes()) { } else { - std::map map; - map.insert(std::pair(metadata->key, metadata->bl)); - ret = io_ctx->omap_set(oid, map); + std::map map; + buff_str = std::string(((ceph::bufferlist*)metadata->get_buffer())->c_str()); + bl.append(buff_str.c_str(), buff_str.size() +1); + map.insert(std::pair(metadata->get_key(),bl)); + ret = io_ctx_wrapper->omap_set(map,oid); + bl.clear(); } } return ret; @@ -214,12 +210,6 @@ int RadosMetadataStorageIma::update_keyword_metadata(const std::string &oid, Rad int RadosMetadataStorageIma::remove_keyword_metadata(const std::string &oid, std::string &key) { std::set keys; keys.insert(key); - return io_ctx->omap_rm_keys(oid, keys); -} - -int RadosMetadataStorageIma::load_keyword_metadata(const std::string &oid, std::set &keys, - std::map *metadata) { - return io_ctx->omap_get_vals_by_keys(oid, keys, metadata); + return io_ctx_wrapper->omap_rm_keys(oid, keys); } - } /* namespace librmb */ diff --git a/src/librmb/rados-metadata-storage-ima.h b/src/librmb/rados-metadata-storage-ima.h index 3b8151da..fdbb7530 100644 --- a/src/librmb/rados-metadata-storage-ima.h +++ b/src/librmb/rados-metadata-storage-ima.h @@ -18,10 +18,11 @@ #include #include -#include "rados-ceph-config.h" -#include "rados-dovecot-ceph-cfg.h" -#include "rados-metadata-storage-module.h" - +#include "../storage-interface/rados-ceph-config.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../storage-interface/rados-metadata-storage-module.h" +#include "../storage-interface/rados-mail.h" +#include "../storage-interface/rbox-io-ctx.h" namespace librmb { /** * All immutable mail attributes are saved in one rados @@ -32,32 +33,29 @@ namespace librmb { * immutable value. * */ -class RadosMetadataStorageIma : public RadosStorageMetadataModule { +class RadosMetadataStorageIma : public storage_interface::RadosStorageMetadataModule { private: - int parse_attribute(RadosMail *mail, json_t *root); + int parse_attribute(storage_interface::RadosMail *mail, json_t *root); public: - RadosMetadataStorageIma(librados::IoCtx *io_ctx_, RadosDovecotCephCfg *cfg_); + RadosMetadataStorageIma(storage_interface::RboxIoCtx *io_ctx_wrapper, storage_interface::RadosDovecotCephCfg *cfg_); virtual ~RadosMetadataStorageIma(); - void set_io_ctx(librados::IoCtx *io_ctx_) override { this->io_ctx = io_ctx_; } - int load_metadata(RadosMail *mail) override; - int set_metadata(RadosMail *mail, RadosMetadata &xattr) override; - int set_metadata(RadosMail *mail, RadosMetadata &xattr, librados::ObjectWriteOperation *write_op) override; - bool update_metadata(const std::string &oid, std::list &to_update) override; - void save_metadata(librados::ObjectWriteOperation *write_op, RadosMail *mail) override; + void set_io_ctx(storage_interface::RboxIoCtx *io_ctx_wrapper) override { this->io_ctx_wrapper = io_ctx_wrapper; } + int load_metadata(storage_interface::RadosMail *mail) override; + int set_metadata(storage_interface::RadosMail *mail) override; + bool update_metadata(const std::string &oid, std::list &to_update) override; + void save_metadata(librados::ObjectWriteOperation *write_op, storage_interface::RadosMail *mail); - int update_keyword_metadata(const std::string &oid, RadosMetadata *metadata) override; + int update_keyword_metadata(const std::string &oid, storage_interface::RadosMetadata *metadata) override; int remove_keyword_metadata(const std::string &oid, std::string &key) override; - int load_keyword_metadata(const std::string &oid, std::set &keys, - std::map *metadata) override; public: static std::string module_name; static std::string keyword_key; private: - librados::IoCtx *io_ctx; - RadosDovecotCephCfg *cfg; + storage_interface::RboxIoCtx *io_ctx_wrapper = nullptr; + storage_interface::RadosDovecotCephCfg *cfg = nullptr; }; } /* namespace librmb */ diff --git a/src/librmb/rados-metadata-storage-impl.h b/src/librmb/rados-metadata-storage-impl.h index 868193bc..c3137e9c 100644 --- a/src/librmb/rados-metadata-storage-impl.h +++ b/src/librmb/rados-metadata-storage-impl.h @@ -13,19 +13,27 @@ #include /* assert */ #include -#include "rados-metadata-storage-module.h" +#include "../storage-interface/rados-metadata-storage-module.h" #include "rados-metadata-storage-default.h" #include "rados-metadata-storage-ima.h" -#include "rados-metadata-storage.h" +#include "../storage-interface/rados-metadata-storage.h" namespace librmb { -class RadosMetadataStorageImpl : public RadosMetadataStorage { +class RadosMetadataStorageImpl : public storage_interface::RadosMetadataStorage { public: - RadosMetadataStorageImpl() { - storage = nullptr; - io_ctx = nullptr; - cfg = nullptr; + + RadosMetadataStorageImpl(storage_interface::RboxIoCtx *io_ctx_, storage_interface::RadosDovecotCephCfg *cfg_) { + io_ctx_wrapper = io_ctx_; + cfg = cfg_; + + std::string storage_module_name = cfg->get_metadata_storage_module(); + if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) { + storage = new librmb::RadosMetadataStorageIma(io_ctx_wrapper, cfg); + } else { + storage = new librmb::RadosMetadataStorageDefault(io_ctx_wrapper); + } } + virtual ~RadosMetadataStorageImpl() { if (storage != nullptr) { delete storage; @@ -33,30 +41,15 @@ class RadosMetadataStorageImpl : public RadosMetadataStorage { } } - RadosStorageMetadataModule *create_metadata_storage(librados::IoCtx *io_ctx_, RadosDovecotCephCfg *cfg_) override { - this->io_ctx = io_ctx_; - this->cfg = cfg_; - if (storage == nullptr) { - // decide metadata storage! - std::string storage_module_name = cfg_->get_metadata_storage_module(); - if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) { - storage = new librmb::RadosMetadataStorageIma(io_ctx, cfg_); - } else { - storage = new librmb::RadosMetadataStorageDefault(io_ctx); - } - } - return storage; - } - - RadosStorageMetadataModule *get_storage() override { + storage_interface::RadosStorageMetadataModule *get_storage() override { assert(storage != nullptr); return storage; } private: - librados::IoCtx *io_ctx; - RadosDovecotCephCfg *cfg; - RadosStorageMetadataModule *storage; + storage_interface::RboxIoCtx *io_ctx_wrapper = nullptr; + storage_interface::RadosDovecotCephCfg *cfg = nullptr; + storage_interface::RadosStorageMetadataModule *storage = nullptr; }; } // namespace librmb diff --git a/src/librmb/rados-metadata-storage-module.h b/src/librmb/rados-metadata-storage-module.h deleted file mode 100644 index 833549f7..00000000 --- a/src/librmb/rados-metadata-storage-module.h +++ /dev/null @@ -1,44 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_RADOS_METADATA_STORAGE_MODULE_H_ -#define SRC_LIBRMB_RADOS_METADATA_STORAGE_MODULE_H_ - -#include - -#include "rados-mail.h" - -namespace librmb { -class RadosStorageMetadataModule { - public: - virtual ~RadosStorageMetadataModule(){}; - /* update io_ctx */ - virtual void set_io_ctx(librados::IoCtx *io_ctx){}; - /* load the metadta into RadosMail */ - virtual int load_metadata(RadosMail *mail) = 0; - /* set a new metadata attribute to a mail object */ - virtual int set_metadata(RadosMail *mail, RadosMetadata &xattr) = 0; - /* set a new metadata attribute to a mail object */ - virtual int set_metadata(RadosMail *mail, RadosMetadata &xattr, librados::ObjectWriteOperation *write_op) = 0; - /* update the given metadata attributes */ - virtual bool update_metadata(const std::string &oid, std::list &to_update) = 0; - /* add all metadata of RadosMail to write_operation */ - virtual void save_metadata(librados::ObjectWriteOperation *write_op, RadosMail *mail) = 0; - /* manage keywords */ - virtual int update_keyword_metadata(const std::string &oid, RadosMetadata *metadata) = 0; - virtual int remove_keyword_metadata(const std::string &oid, std::string &key) = 0; - virtual int load_keyword_metadata(const std::string &oid, std::set &keys, - std::map *metadata) = 0; -}; - -} // namespace librmb - -#endif /* SRC_LIBRMB_RADOS_METADATA_STORAGE_MODULE_H_ */ diff --git a/src/librmb/rados-metadata.h b/src/librmb/rados-metadata.h deleted file mode 100644 index 7bbc1a4d..00000000 --- a/src/librmb/rados-metadata.h +++ /dev/null @@ -1,115 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_RADOS_METADATA_H_ -#define SRC_LIBRMB_RADOS_METADATA_H_ - -#include "rados-types.h" -#include -#include "time.h" -#include -#include -#include - -namespace librmb { - -class RadosMetadata { - public: - RadosMetadata() {} - RadosMetadata(std::string& key_, std::string& value_) : key(key_) { bl.append(value_.c_str(), value_.length() + 1); } - RadosMetadata(enum rbox_metadata_key _key, const std::string& val) { convert(_key, val); } - - RadosMetadata(enum rbox_metadata_key _key, const time_t& val) { convert(_key, val); } - - RadosMetadata(enum rbox_metadata_key _key, const char* val) { convert(_key, val); } - - RadosMetadata(enum rbox_metadata_key _key, const uint& val) { convert(_key, val); } - - RadosMetadata(enum rbox_metadata_key _key, const size_t& val) { convert(_key, val); } - RadosMetadata(enum rbox_metadata_key _key, const int val) { convert(_key, val); } - ~RadosMetadata() {} - - public: - ceph::bufferlist& get_bl(); - std::string& get_key(); - - void convert(const char* value, time_t* t) { - if (t != NULL) { - std::istringstream stream(value); - stream >> *t; - } - } - static bool from_string(const std::string& str, RadosMetadata* metadata) { - std::stringstream ss(str); - std::string item; - std::vector token; - while (std::getline(ss, item, '=')) { - token.push_back(item); - } - if (token.size() != 2 || metadata == nullptr) { - return false; - } - metadata->key = token[0]; - metadata->bl.append(token[1]); - return true; - } - std::string to_string() { - std::stringstream str; - str << key << "=" << bl.to_str().substr(0, bl.length() - 1); - return str.str(); - } - - public: - ceph::bufferlist bl; - std::string key; - - void convert(enum rbox_metadata_key _key, const std::string& val) { - bl.clear(); - key = librmb::rbox_metadata_key_to_char(_key); - bl.append(val.c_str(), val.length() + 1); - } - - void convert(enum rbox_metadata_key _key, const time_t& time) { - bl.clear(); - key = librmb::rbox_metadata_key_to_char(_key); - std::string time_ = std::to_string(time); - bl.append(time_.c_str(), time_.length() + 1); - } - - void convert(enum rbox_metadata_key _key, char* value) { - bl.clear(); - key = librmb::rbox_metadata_key_to_char(_key); - std::string str = value; - bl.append(str.c_str(), str.length() + 1); - } - - void convert(enum rbox_metadata_key _key, const uint& value) { - bl.clear(); - key = librmb::rbox_metadata_key_to_char(_key); - std::string val = std::to_string(value); - bl.append(val.c_str(), val.length() + 1); - } - - void convert(enum rbox_metadata_key _key, const size_t& value) { - bl.clear(); - key = librmb::rbox_metadata_key_to_char(_key); - std::string val = std::to_string(static_cast(value)); - bl.append(val.c_str(), val.length() + 1); - } - void convert(enum rbox_metadata_key _key, const int value) { - bl.clear(); - key = librmb::rbox_metadata_key_to_char(_key); - std::string val = std::to_string(value); - bl.append(val.c_str(), val.length() + 1); - } -}; -} // end namespace -#endif /* SRC_LIBRMB_RADOS_METADATA_H_ */ diff --git a/src/librmb/rados-namespace-manager.cpp b/src/librmb/rados-namespace-manager-impl.cpp similarity index 67% rename from src/librmb/rados-namespace-manager.cpp rename to src/librmb/rados-namespace-manager-impl.cpp index a6f02be5..42d88e90 100644 --- a/src/librmb/rados-namespace-manager.cpp +++ b/src/librmb/rados-namespace-manager-impl.cpp @@ -8,16 +8,16 @@ * License version 2.1, as published by the Free Software * Foundation. See file COPYING. */ -#include "rados-namespace-manager.h" +#include "rados-namespace-manager-impl.h" + #include namespace librmb { -RadosNamespaceManager::~RadosNamespaceManager() { -} +RadosNamespaceManagerImpl::~RadosNamespaceManagerImpl() {} -bool RadosNamespaceManager::lookup_key(const std::string &uid, std::string *value) { +bool RadosNamespaceManagerImpl::lookup_key(const std::string &uid, std::string *value) { if (uid.empty()) { *value = uid; return true; @@ -41,15 +41,16 @@ bool RadosNamespaceManager::lookup_key(const std::string &uid, std::string *valu return true; } - ceph::bufferlist bl; + + std::ostringstream stream_buffer; bool retval = false; // temporarily set storage namespace to config namespace config->set_io_ctx_namespace(config->get_user_ns()); // storage->set_namespace(config->get_user_ns()); - int err = config->read_object(uid, &bl); - if (err >= 0 && !bl.to_str().empty()) { - *value = bl.to_str(); + int err = config->read_object(uid,stream_buffer); + if (err >= 0 && !stream_buffer.str().empty()) { + *value = stream_buffer.str(); cache[uid] = *value; retval = true; } @@ -58,8 +59,8 @@ bool RadosNamespaceManager::lookup_key(const std::string &uid, std::string *valu return retval; } -bool RadosNamespaceManager::add_namespace_entry(const std::string &uid, std::string *value, - RadosGuidGenerator *guid_generator_) { +bool RadosNamespaceManagerImpl::add_namespace_entry(const std::string &uid, std::string *value, + storage_interface::RadosGuidGenerator *guid_generator_) { if (config == nullptr) { return false; } @@ -74,16 +75,17 @@ bool RadosNamespaceManager::add_namespace_entry(const std::string &uid, std::str guid_generator_->generate_guid(value); // temporarily set storage namespace to config namespace config->set_io_ctx_namespace(config->get_user_ns()); - - ceph::bufferlist bl; - bl.append(*value); bool retval = false; - if (config->save_object(uid, bl) >= 0) { - cache[uid] = *value; + + std::istringstream i_value(*value); + if (config->save_object(uid,i_value) >= 0) { + cache[uid] = i_value.str(); retval = true; } // reset namespace config->set_io_ctx_namespace(""); + delete guid_generator_; + guid_generator_ = nullptr; return retval; } diff --git a/src/librmb/rados-namespace-manager-impl.h b/src/librmb/rados-namespace-manager-impl.h new file mode 100644 index 00000000..0364c99d --- /dev/null +++ b/src/librmb/rados-namespace-manager-impl.h @@ -0,0 +1,51 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ +#ifndef SRC_LIBRMB_RADOS_NAMESPACE_MANAGER_H_ +#define SRC_LIBRMB_RADOS_NAMESPACE_MANAGER_H_ + +#include +#include +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../storage-interface/rados-guid-generator.h" +#include"../storage-interface/rados-namespace-manager.h" + +namespace librmb { + +/** + * Rados Namespace Manager + * + * Get Users Namespace based on configuration + * + */ +class RadosNamespaceManagerImpl :public storage_interface::RadosNamespaceManager { + public: + /*! + * @param[in] config_ valid radosDovecotCephCfg. + */ + explicit RadosNamespaceManagerImpl(storage_interface::RadosDovecotCephCfg *config_) : oid_suffix("_namespace"), config(config_) {} + virtual ~RadosNamespaceManagerImpl(); + void set_config(storage_interface::RadosDovecotCephCfg *config_) override { config = config_; } + storage_interface::RadosDovecotCephCfg *get_config() override { return config; } + + void set_namespace_oid(std::string &namespace_oid_) override{ this->oid_suffix = namespace_oid_; } + bool lookup_key(const std::string &uid, std::string *value) override; + bool add_namespace_entry(const std::string &uid, std::string *value, storage_interface::RadosGuidGenerator *guid_generator_) override; + + private: + std::map cache; + std::string oid_suffix; + storage_interface::RadosDovecotCephCfg *config; +}; + +} /* namespace librmb */ + +#endif /* SRC_LIBRMB_RADOS_NAMESPACE_MANAGER_H_ */ diff --git a/src/librmb/rados-namespace-manager.h b/src/librmb/rados-namespace-manager.h deleted file mode 100644 index 542c4070..00000000 --- a/src/librmb/rados-namespace-manager.h +++ /dev/null @@ -1,49 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ -#ifndef SRC_LIBRMB_RADOS_NAMESPACE_MANAGER_H_ -#define SRC_LIBRMB_RADOS_NAMESPACE_MANAGER_H_ - -#include -#include -#include "rados-storage.h" -#include "rados-dovecot-ceph-cfg.h" -#include "rados-guid-generator.h" -namespace librmb { - -/** - * Rados Namespace Manager - * - * Get Users Namespace based on configuration - * - */ -class RadosNamespaceManager { - public: - /*! - * @param[in] config_ valid radosDovecotCephCfg. - */ - explicit RadosNamespaceManager(RadosDovecotCephCfg *config_) : oid_suffix("_namespace"), config(config_) {} - virtual ~RadosNamespaceManager(); - void set_config(RadosDovecotCephCfg *config_) { config = config_; } - RadosDovecotCephCfg *get_config() { return config; } - - void set_namespace_oid(std::string &namespace_oid_) { this->oid_suffix = namespace_oid_; } - bool lookup_key(const std::string &uid, std::string *value); - bool add_namespace_entry(const std::string &uid, std::string *value, RadosGuidGenerator *guid_generator_); - - private: - std::map cache; - std::string oid_suffix; - RadosDovecotCephCfg *config; -}; - -} /* namespace librmb */ - -#endif /* SRC_LIBRMB_RADOS_NAMESPACE_MANAGER_H_ */ diff --git a/src/librmb/rados-save-log.cpp b/src/librmb/rados-save-log-impl.cpp similarity index 70% rename from src/librmb/rados-save-log.cpp rename to src/librmb/rados-save-log-impl.cpp index 08859d01..fda20a75 100644 --- a/src/librmb/rados-save-log.cpp +++ b/src/librmb/rados-save-log-impl.cpp @@ -9,23 +9,25 @@ * Foundation. See file COPYING. */ -#include "rados-save-log.h" +#include "rados-save-log-impl.h" namespace librmb { -bool RadosSaveLog::open() { +bool RadosSaveLogImpl::open() { if (this->log_active && !ofs.is_open()) { ofs.open(this->logfile, std::ofstream::out | std::ofstream::app); this->log_active = ofs.is_open(); } return this->log_active; } -void RadosSaveLog::append(const RadosSaveLogEntry &entry) { +void RadosSaveLogImpl::append(const storage_interface::RadosSaveLogEntry *entry) { if (this->log_active && ofs.is_open()) { - ofs << entry; + const RadosSaveLogEntryImpl &entry_impl= + *(dynamic_cast(entry)); + ofs << entry_impl; } } -bool RadosSaveLog::close() { +bool RadosSaveLogImpl::close() { if (this->log_active && ofs.is_open()) { ofs.close(); return !ofs.is_open(); diff --git a/src/librmb/rados-save-log-impl.h b/src/librmb/rados-save-log-impl.h new file mode 100644 index 00000000..f6d61fc0 --- /dev/null +++ b/src/librmb/rados-save-log-impl.h @@ -0,0 +1,197 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_RADOS_SAVE_LOG_IMPL_H_ +#define SRC_LIBRMB_RADOS_SAVE_LOG_IMPL_H_ + + +#include "../storage-interface/rados-save-log.h" +#include "../librmb/rados-metadata-impl.h" + +namespace librmb { +/** + * RadosSaveLogEntryImpl + * + * Class provides access to the savelog.format. + * + */ +class RadosSaveLogEntryImpl : public storage_interface::RadosSaveLogEntry{ + public: + RadosSaveLogEntryImpl() {} + RadosSaveLogEntryImpl(const std::string &oid_, const std::string &ns_, const std::string &pool_, const std::string &op_) + : oid(oid_), ns(ns_), pool(pool_), op(op_), metadata(0) {} + ~RadosSaveLogEntryImpl(){ + std::list::iterator it; + for(it = metadata.begin(); it != metadata.end(); ++it){ + delete (*it); + (*it)=nullptr; + } + }; + + // format: mv|cp|save:src_ns,src_oid;metadata_key=metadata_value:metadata_key=metadata_value:.... + // e.g.: // mv:ns_src:src_oid;M=ABCDEFG:B=INBOX:U=1 + bool parse_mv_op() override{ + int pos = op.find(";"); + if (pos <= 0) { + return false; + } + + std::stringstream left(op.substr(0, pos)); + std::vector left_tokens; + std::string item; + while (std::getline(left, item, ':')) { + left_tokens.push_back(item); + } + if (left_tokens.size() < 4) { + return false; + } + + if (left_tokens[0].compare("mv") != 0) { + return true; // not a move cmd. + } + // mv specific + src_ns = left_tokens[1]; + src_oid = left_tokens[2]; + src_user = left_tokens[3]; + // parsing metadata. + std::stringstream right(op.substr(pos + 1, op.size())); + std::vector right_tokens; + while (std::getline(right, item, ':')) { + storage_interface::RadosMetadata *m= new librmb::RadosMetadataImpl(); + if (!m->from_string(item)) { + delete m; + m=nullptr; + return false; + } + metadata.push_back(m); + } + return true; + } + std::string op_save() override{ return "save"; } + std::string op_cpy() override{ return "cpy"; } + std::string op_mv(const std::string &src_ns, const std::string &src_oid, const std::string &src_user, + std::list &metadata) override{ + std::stringstream mv; + mv << "mv:" << src_ns << ":" << src_oid << ":" << src_user << ";" << convert_metadata(metadata, ":"); + return mv.str(); + } + + friend std::ostream &operator<<(std::ostream &os, const librmb::RadosSaveLogEntryImpl &obj) { + os << obj.get_op() << "," << obj.get_pool() << "," << obj.get_ns() << "," << obj.get_oid()<< std::endl; + return os; + } + + friend std::istream &operator>>(std::istream &is, librmb::RadosSaveLogEntryImpl &obj) { + std::string line; + std::string item; + std::vector csv_items; + std::getline(is, line); + + std::stringstream line_to_parse(line); + while (std::getline(line_to_parse, item, ',')) { + csv_items.push_back(item); + } + + // read obj from stream + if (csv_items.size() == 4) { + obj.get_op() = csv_items[0]; + obj.get_pool() = csv_items[1]; + obj.get_ns() = csv_items[2]; + obj.get_oid() = csv_items[3]; + obj.parse_mv_op(); + + } else { + is.setstate(std::ios::failbit); + } + return is; + } + + static std::string convert_metadata(std::list &metadata, const std::string &separator) { + std::stringstream metadata_str; + std::list::iterator list_it; + list_it = metadata.begin(); + if (list_it != metadata.end()) { + metadata_str << (*list_it)->to_string(); + list_it++; + } + + for (; list_it != metadata.end(); list_it++) { + metadata_str << separator; + metadata_str << (*list_it)->to_string(); + } + return metadata_str.str(); + } + + std::string& get_oid() override{ + return oid; + } + + std::string& get_ns() override{ + return ns; + } + + std::string& get_pool() override{ + return pool; + } + + std::string& get_op() override{ + return op; + } + + std::string& get_src_oid() override{ + return src_oid; + } + std::string& get_src_ns() override{ + return src_ns; + } + + std::string& get_src_user() override{ + return src_user; + } + + std::list& get_metadata() override{ + return metadata; + } + + private: + std::string oid; // oid + std::string ns; // namespace + std::string pool; // storage pool + std::string op; // operation: save, cp (copy), mv (move) + std::string src_oid; + std::string src_ns; + std::string src_user; + std::list metadata; +}; + +class RadosSaveLogImpl : public storage_interface::RadosSaveLog{ + public: + explicit RadosSaveLogImpl(const std::string &logfile_) : logfile(logfile_) { log_active = !logfile.empty(); } + RadosSaveLogImpl() { log_active = false; } + virtual ~RadosSaveLogImpl(){}; + void set_save_log_file (const std::string &logfile_) override{ + this->logfile = logfile_; + this->log_active = !logfile.empty(); + } + bool open() override; + void append(const storage_interface::RadosSaveLogEntry *entry) override; + bool close() override; + bool is_open() override{ return ofs.is_open(); } + + private: + std::string logfile; + bool log_active; + std::ofstream ofs; +}; + +} /* namespace librmb */ + +#endif /* SRC_LIBRMB_RADOS_SAVE_LOG_IMPL_H_ */ diff --git a/src/librmb/rados-save-log.h b/src/librmb/rados-save-log.h deleted file mode 100644 index c1112b9b..00000000 --- a/src/librmb/rados-save-log.h +++ /dev/null @@ -1,165 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_RADOS_SAVE_LOG_H_ -#define SRC_LIBRMB_RADOS_SAVE_LOG_H_ - -#include // std::ofstream -//#include -#include -#include -#include -#include -#include - -#include "rados-metadata.h" - -namespace librmb { -/** - * RadosSaveLogEntry - * - * Class provides access to the savelog.format. - * - */ -class RadosSaveLogEntry { - public: - RadosSaveLogEntry() {} - RadosSaveLogEntry(const std::string &oid_, const std::string &ns_, const std::string &pool_, const std::string &op_) - : oid(oid_), ns(ns_), pool(pool_), op(op_), metadata(0) {} - ~RadosSaveLogEntry(){}; - - // format: mv|cp|save:src_ns,src_oid;metadata_key=metadata_value:metadata_key=metadata_value:.... - // e.g.: // mv:ns_src:src_oid;M=ABCDEFG:B=INBOX:U=1 - bool parse_mv_op() { - int pos = op.find(";"); - if (pos <= 0) { - return false; - } - - std::stringstream left(op.substr(0, pos)); - std::vector left_tokens; - std::string item; - while (std::getline(left, item, ':')) { - left_tokens.push_back(item); - } - if (left_tokens.size() < 4) { - return false; - } - - if (left_tokens[0].compare("mv") != 0) { - return true; // not a move cmd. - } - // mv specific - src_ns = left_tokens[1]; - src_oid = left_tokens[2]; - src_user = left_tokens[3]; - // parsing metadata. - std::stringstream right(op.substr(pos + 1, op.size())); - std::vector right_tokens; - while (std::getline(right, item, ':')) { - librmb::RadosMetadata m; - if (!librmb::RadosMetadata::from_string(item, &m)) { - return false; - } - metadata.push_back(m); - } - return true; - } - static std::string op_save() { return "save"; } - static std::string op_cpy() { return "cpy"; } - static std::string op_mv(const std::string &src_ns, const std::string &src_oid, const std::string &src_user, - std::list &metadata) { - std::stringstream mv; - mv << "mv:" << src_ns << ":" << src_oid << ":" << src_user << ";" << convert_metadata(metadata, ":"); - return mv.str(); - } - - friend std::ostream &operator<<(std::ostream &os, const RadosSaveLogEntry &obj) { - os << obj.op << "," << obj.pool << "," << obj.ns << "," << obj.oid << std::endl; - return os; - } - - friend std::istream &operator>>(std::istream &is, RadosSaveLogEntry &obj) { - std::string line; - std::string item; - std::vector csv_items; - std::getline(is, line); - - std::stringstream line_to_parse(line); - while (std::getline(line_to_parse, item, ',')) { - csv_items.push_back(item); - } - - // read obj from stream - if (csv_items.size() == 4) { - obj.op = csv_items[0]; - obj.pool = csv_items[1]; - obj.ns = csv_items[2]; - obj.oid = csv_items[3]; - - obj.parse_mv_op(); - - } else { - is.setstate(std::ios::failbit); - } - return is; - } - - static std::string convert_metadata(std::list &metadata, const std::string &separator) { - std::stringstream metadata_str; - std::list::iterator list_it; - list_it = metadata.begin(); - if (list_it != metadata.end()) { - metadata_str << (*list_it)->to_string(); - list_it++; - } - - for (; list_it != metadata.end(); list_it++) { - metadata_str << separator; - metadata_str << (*list_it)->to_string(); - } - return metadata_str.str(); - } - - public: - std::string oid; // oid - std::string ns; // namespace - std::string pool; // storage pool - std::string op; // operation: save, cp (copy), mv (move) - std::string src_oid; - std::string src_ns; - std::string src_user; - std::list metadata; -}; - -class RadosSaveLog { - public: - explicit RadosSaveLog(const std::string &logfile_) : logfile(logfile_) { log_active = !logfile.empty(); } - RadosSaveLog() { log_active = false; } - void set_save_log_file(const std::string &logfile_) { - this->logfile = logfile_; - this->log_active = !logfile.empty(); - } - virtual ~RadosSaveLog(){}; - bool open(); - void append(const RadosSaveLogEntry &entry); - bool close(); - bool is_open() { return ofs.is_open(); } - - private: - std::string logfile; - bool log_active; - std::ofstream ofs; -}; - -} /* namespace librmb */ - -#endif /* SRC_LIBRMB_RADOS_SAVE_LOG_H_ */ diff --git a/src/librmb/rados-storage-impl.cpp b/src/librmb/rados-storage-impl.cpp index f2b38993..eeed670d 100644 --- a/src/librmb/rados-storage-impl.cpp +++ b/src/librmb/rados-storage-impl.cpp @@ -18,13 +18,22 @@ #include #include #include - -#include "rados-util.h" +#include +#include "rados-util-impl.h" +#include "../storage-interface/rados-plugin-logger.h" #include +#include #include "encoding.h" #include "limits.h" +#include "rados-metadata-storage-impl.h" +#include "../storage-interface/rbox-io-ctx.h" +#include "rbox-io-ctx-impl.h" +#include "rbox-io-ctx-impl.h" +#include "rados-metadata-impl.h" +#include "rados-mail-impl.h" + using std::pair; using std::string; @@ -37,18 +46,27 @@ using librmb::RadosStorageImpl; const char *RadosStorageImpl::CFG_OSD_MAX_WRITE_SIZE = "osd_max_write_size"; const char *RadosStorageImpl::CFG_OSD_MAX_OBJECT_SIZE= "osd_max_object_size"; -RadosStorageImpl::RadosStorageImpl(RadosCluster *_cluster) { +RadosStorageImpl::RadosStorageImpl(storage_interface::RadosCluster *_cluster) { cluster = _cluster; max_write_size = 10; max_object_size = 134217728; //ceph default 128MB io_ctx_created = false; - wait_method = WAIT_FOR_COMPLETE_AND_CB; + wait_method = storage_interface::WAIT_FOR_COMPLETE_AND_CB; + io_ctx_wrapper = new librmb::RboxIoCtxImpl(); } -RadosStorageImpl::~RadosStorageImpl() {} +RadosStorageImpl::~RadosStorageImpl() { + if(io_ctx_wrapper != nullptr){ + delete io_ctx_wrapper; + } + io_ctx_wrapper=nullptr; +} +storage_interface::RboxIoCtx* RadosStorageImpl::get_io_ctx_wrapper(){ + return io_ctx_wrapper; +} //DEPRECATED!!!!! -> moved to rbox-save.cpp -int RadosStorageImpl::split_buffer_and_exec_op(RadosMail *current_object, +int RadosStorageImpl::split_buffer_and_exec_op(storage_interface::RadosMail *current_object, librados::ObjectWriteOperation *write_op_xattr, const uint64_t &max_write) { @@ -57,7 +75,7 @@ int RadosStorageImpl::split_buffer_and_exec_op(RadosMail *current_object, } int ret_val = 0; - uint64_t write_buffer_size = current_object->get_mail_size() -1; + uint64_t write_buffer_size = current_object->get_mail_size(); assert(max_write > 0); @@ -67,7 +85,7 @@ int RadosStorageImpl::split_buffer_and_exec_op(RadosMail *current_object, return ret_val; } - ret_val = get_io_ctx().operate(*current_object->get_oid(), write_op_xattr); + ret_val = io_ctx_wrapper->operate(*current_object->get_oid(), write_op_xattr); if(ret_val< 0){ ret_val = -1; @@ -76,6 +94,7 @@ int RadosStorageImpl::split_buffer_and_exec_op(RadosMail *current_object, uint64_t rest = write_buffer_size % max_write; int div = write_buffer_size / max_write + (rest > 0 ? 1 : 0); + librados::bufferlist* mail_buffer = (librados::bufferlist*) current_object->get_mail_buffer(); for (int i = 0; i < div; ++i) { // split the buffer. @@ -95,100 +114,131 @@ int RadosStorageImpl::split_buffer_and_exec_op(RadosMail *current_object, write_op.set_alloc_hint(write_buffer_size, length); #endif if (div == 1) { - write_op.write(0, *current_object->get_mail_buffer()); + write_op.write(0, *mail_buffer); } else { tmp_buffer.clear(); - tmp_buffer.substr_of(*current_object->get_mail_buffer(), offset, length); + tmp_buffer.substr_of(*mail_buffer, offset, length); write_op.write(offset, tmp_buffer); } - ret_val = get_io_ctx().operate(*current_object->get_oid(), &write_op); + ret_val = io_ctx_wrapper->operate(*current_object->get_oid(), &write_op); if(ret_val < 0){ ret_val = -1; break; } } - // deprecated unused - current_object->set_write_operation(nullptr); - current_object->set_completion(nullptr); - current_object->set_active_op(0); - // free mail's buffer cause we don't need it anymore - librados::bufferlist *mail_buffer = current_object->get_mail_buffer(); delete mail_buffer; return ret_val; } -int RadosStorageImpl::save_mail(const std::string &oid, librados::bufferlist &buffer) { - return get_io_ctx().write_full(oid, buffer); -} - -int RadosStorageImpl::read_mail(const std::string &oid, librados::bufferlist *buffer) { +int RadosStorageImpl::read_mail(const std::string &oid, storage_interface::RadosMail* mail,int try_counter){ if (!cluster->is_connected() || !io_ctx_created) { return -1; } - size_t max = INT_MAX; - return get_io_ctx().read(oid, *buffer, max, 0); -} - -int RadosStorageImpl::delete_mail(RadosMail *mail) { - int ret = -1; - - if (cluster->is_connected() && io_ctx_created && mail != nullptr) { - ret = delete_mail(*mail->get_oid()); + + int ret=0; + int stat_err = 0; + int read_err = 0; + uint64_t psize; + time_t save_date; + librados::bufferlist* buffer = (librados::bufferlist*) mail->get_mail_buffer(); + librados::ObjectReadOperation read_op; + read_op.read(0, INT_MAX, buffer, &read_err); + read_op.stat(&psize, &save_date, &stat_err); + ret=io_ctx_wrapper->operate(oid, &read_op, buffer); + if(ret == -ETIMEDOUT) { + int max_retry = 10; //TODO FIX + if(try_counter < max_retry){ + usleep(((rand() % 5) + 1) * 10000); + try_counter++; + ret=read_mail(oid,mail,try_counter); + } } + if(ret < 0 || ret == -ETIMEDOUT){ + return ret; + } + mail->set_mail_size((const int)psize); + mail->set_rados_save_date(&save_date); return ret; } + int RadosStorageImpl::delete_mail(const std::string &oid) { if (!cluster->is_connected() || oid.empty() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"Delete mail {} failed ", oid); return -1; } - return get_io_ctx().remove(oid); + return io_ctx_wrapper->remove(oid); +} + +bool RadosStorageImpl::execute_operation(std::string &oid, librados::ObjectWriteOperation *write_op_xattr) { + if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"operation can't be executed as there is no cluster connection available "); + return false; + } + return io_ctx_wrapper->operate(oid, write_op_xattr) >=0 ? true : false; +} + +bool RadosStorageImpl::append_to_object(std::string &oid, librados::bufferlist &bufferlist, int length) { + if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"appending to {} is not possible as there is no cluster connection available ", oid); + return false; + } + return io_ctx_wrapper->append(oid, bufferlist, length) >=0 ? true : false; } int RadosStorageImpl::aio_operate(librados::IoCtx *io_ctx_, const std::string &oid, librados::AioCompletion *c, librados::ObjectWriteOperation *op) { if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"execting async operation for {} is not possible as there is no cluster connection available ", oid); return -1; } if (io_ctx_ != nullptr) { return io_ctx_->aio_operate(oid, c, op); } else { - return get_io_ctx().aio_operate(oid, c, op); + return io_ctx_wrapper->aio_operate(oid, c, op); } } int RadosStorageImpl::stat_mail(const std::string &oid, uint64_t *psize, time_t *pmtime) { if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"stat for {} is not possible as there is no cluster connection available ", oid); return -1; } - return get_io_ctx().stat(oid, psize, pmtime); + return io_ctx_wrapper->stat(oid, psize, pmtime); } void RadosStorageImpl::set_namespace(const std::string &_nspace) { - get_io_ctx().set_namespace(_nspace); + io_ctx_wrapper->set_namespace(_nspace); this->nspace = _nspace; } -librados::NObjectIterator RadosStorageImpl::find_mails(const RadosMetadata *attr) { +std::set RadosStorageImpl::find_mails(const storage_interface::RadosMetadata *attr) { if (!cluster->is_connected() || !io_ctx_created) { - return librados::NObjectIterator::__EndObjectIterator; + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"find_mails {} is not possible as there is no cluster connection available "); + return ; } - + std::set mail_list; + librados::NObjectIterator iter_guid; if (attr != nullptr) { std::string filter_name = PLAIN_FILTER_NAME; ceph::bufferlist filter_bl; encode(filter_name, filter_bl); - encode("_" + attr->key, filter_bl); - encode(attr->bl.to_str(), filter_bl); + encode("_" + attr->get_key(), filter_bl); + encode(((ceph::bufferlist*)attr->get_buffer())->to_str(), filter_bl); - return get_io_ctx().nobjects_begin(filter_bl); + iter_guid=io_ctx_wrapper->nobjects_begin(filter_bl); } else { - return get_io_ctx().nobjects_begin(); + iter_guid=io_ctx_wrapper->nobjects_begin(); + } + while (iter_guid != librados::NObjectIterator::__EndObjectIterator) { + mail_list.insert((*iter_guid).get_oid()); + iter_guid++; } + return mail_list; } /** * POC Implementation: @@ -196,7 +246,7 @@ librados::NObjectIterator RadosStorageImpl::find_mails(const RadosMetadata *attr * see in prod how it behaves. * **/ -std::set RadosStorageImpl::find_mails_async(const RadosMetadata *attr, +std::set RadosStorageImpl::find_mails_async(const storage_interface::RadosMetadata *attr, std::string &pool_name, int num_threads, void (*ptr)(std::string&)){ @@ -249,8 +299,8 @@ std::set RadosStorageImpl::find_mails_async(const RadosMetadata *at ceph::bufferlist filter_bl; encode(filter_name, filter_bl); - encode("_" + attr->key, filter_bl); - encode(attr->bl.to_str(), filter_bl); + encode("_" + attr->get_key(), filter_bl); + encode(((ceph::bufferlist*)attr->get_buffer())->to_str(), filter_bl); std::string msg_1 = "buffer set"; (*ptr)(msg_1); @@ -276,37 +326,45 @@ std::set RadosStorageImpl::find_mails_async(const RadosMetadata *at } return oid_list; } -librados::IoCtx &RadosStorageImpl::get_io_ctx() { return io_ctx; } -librados::IoCtx &RadosStorageImpl::get_recovery_io_ctx() { return recovery_io_ctx; } +librados::IoCtx& RadosStorageImpl::get_io_ctx() { return io_ctx_wrapper->get_io_ctx(); } + +librados::IoCtx& RadosStorageImpl::get_recovery_io_ctx() { return io_ctx_wrapper->get_recovery_io_ctx(); } int RadosStorageImpl::open_connection(const std::string &poolname, const std::string &index_pool, const std::string &clustername, const std::string &rados_username) { if (cluster->is_connected() && io_ctx_created) { // cluster is already connected! + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::DEBUG_LEVEL,"connection to {}->{} is is already open", poolname, clustername); return 1; } if (cluster->init(clustername, rados_username) < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"creating connection to {} with user {} is not possible", clustername, rados_username); return -1; } return create_connection(poolname, index_pool); } + int RadosStorageImpl::open_connection(const std::string &poolname, const std::string &clustername, const std::string &rados_username) { if (cluster->is_connected() && io_ctx_created) { // cluster is already connected! + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::DEBUG_LEVEL,"connection to {}->{} is is already open", poolname, clustername); return 1; } if (cluster->init(clustername, rados_username) < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"creating connection to {} with user {} is not possible", clustername, rados_username); return -1; } return create_connection(poolname, poolname); } + int RadosStorageImpl::open_connection(const string &poolname, const string &index_pool) { if (cluster->init() < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"cluser init failed"); return -1; } return create_connection(poolname, index_pool); @@ -314,35 +372,48 @@ int RadosStorageImpl::open_connection(const string &poolname, const string &inde int RadosStorageImpl::open_connection(const string &poolname) { if (cluster->init() < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"cluser init failed"); return -1; } return create_connection(poolname, poolname); } int RadosStorageImpl::create_connection(const std::string &poolname, const std::string &index_pool){ + // pool exists? else create - int err = cluster->io_ctx_create(poolname, &io_ctx); + int err = cluster->io_ctx_create(poolname,io_ctx_wrapper); if (err < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"Io Ctx creation failed for pool {}", poolname); return err; } - err = cluster->recovery_index_io_ctx(index_pool, &recovery_io_ctx); + err = cluster->recovery_index_io_ctx(index_pool,io_ctx_wrapper); if (err < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"Recovery Io Ctx creation failed for pool {}", index_pool); return err; } + string max_write_size_str; err = cluster->get_config_option(RadosStorageImpl::CFG_OSD_MAX_WRITE_SIZE, &max_write_size_str); if (err < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"getting cluster option failed for {}", RadosStorageImpl::CFG_OSD_MAX_WRITE_SIZE); return err; } - max_write_size = std::stoi(max_write_size_str); + + std::stringstream ss_write; + ss_write << max_write_size_str; + ss_write >> max_write_size; string max_object_size_str; err = cluster->get_config_option(RadosStorageImpl::CFG_OSD_MAX_OBJECT_SIZE, &max_object_size_str); if (err < 0) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"getting cluster option failed for {}", RadosStorageImpl::CFG_OSD_MAX_OBJECT_SIZE); return err; } - max_object_size = std::stoi(max_object_size_str); + + std::stringstream ss; + ss << max_object_size_str; + ss >> max_object_size; if (err == 0) { io_ctx_created = true; @@ -352,69 +423,16 @@ int RadosStorageImpl::create_connection(const std::string &poolname, const std:: pool_name = poolname; return 0; } - void RadosStorageImpl::close_connection() { if (cluster != nullptr && io_ctx_created) { cluster->deinit(); } } -bool RadosStorageImpl::wait_for_write_operations_complete(librados::AioCompletion *completion, - librados::ObjectWriteOperation *write_operation) { - if (completion == nullptr) { - return true; // failed! - } - - bool failed = false; - - switch (wait_method) { - case WAIT_FOR_COMPLETE_AND_CB: - completion->wait_for_complete_and_cb(); - break; - case WAIT_FOR_SAFE_AND_CB: - completion->wait_for_safe_and_cb(); - break; - default: - completion->wait_for_complete_and_cb(); - break; - } - failed = completion->get_return_value() < 0 || failed ? true : false; - // clean up - completion->release(); - - return failed; -} -//DEPRECATED and buggy -bool RadosStorageImpl::wait_for_rados_operations(const std::list &object_list) { - bool ctx_failed = false; - if(object_list.size() == 0){ - return ctx_failed; - } - // wait for all writes to finish! - // imaptest shows it's possible that begin -> continue -> finish cycle is invoked several times before - // rbox_transaction_save_commit_pre is called. - for (std::list::const_iterator it_cur_obj = object_list.begin(); it_cur_obj != object_list.end(); - ++it_cur_obj) { - // if we come from copy mail, there is no operation to wait for. - if ((*it_cur_obj)->has_active_op()) { - bool op_failed = - wait_for_write_operations_complete((*it_cur_obj)->get_completion(), (*it_cur_obj)->get_write_operation()); - - ctx_failed = ctx_failed ? ctx_failed : op_failed; - (*it_cur_obj)->set_active_op(0); - // (*it_cur_obj)->set_completion(nullptr); - (*it_cur_obj)->set_write_operation(nullptr); - } - // free mail's buffer cause we don't need it anymore - librados::bufferlist *mail_buffer = (*it_cur_obj)->get_mail_buffer(); - delete mail_buffer; - } - return ctx_failed; -} - // assumes that destination io ctx is current io_ctx; int RadosStorageImpl::move(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update, bool delete_source) { + std::list &to_update, bool delete_source) { if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"move ns({}) {} to {} is not possible as there is no cluster connection available ",src_ns, src_oid, dest_oid); return -1; } @@ -425,9 +443,10 @@ int RadosStorageImpl::move(std::string &src_oid, const char *src_ns, std::string librados::AioCompletion *completion = librados::Rados::aio_create_completion(); // destination io_ctx is current io_ctx - dest_io_ctx = io_ctx; + dest_io_ctx = get_io_ctx(); if (strcmp(src_ns, dest_ns) != 0) { + std::cout<<"strcmp from move method"< save_date of copy/move will be same as source. // write_op.mtime(&ctx->data.save_date); time_t save_time = time(NULL); - write_op.mtime(&save_time); - + write_op.mtime(&save_time); // update metadata - for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { - write_op.setxattr((*it).key.c_str(), (*it).bl); - } - ret = aio_operate(&dest_io_ctx, dest_oid, completion, &write_op); + ceph::bufferlist bl; + std::string bl_str; + for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { + bl_str = std::string(((ceph::bufferlist*)(*it)->get_buffer())->c_str()); + bl.append(bl_str.c_str(), bl_str.size() + 1); + write_op.setxattr((*it)->get_key().c_str(), bl); + bl.clear(); + } + ret = aio_operate(&dest_io_ctx, dest_oid, completion, &write_op); if (ret >= 0) { completion->wait_for_complete(); ret = completion->get_return_value(); @@ -474,8 +498,9 @@ int RadosStorageImpl::move(std::string &src_oid, const char *src_ns, std::string // assumes that destination io ctx is current io_ctx; int RadosStorageImpl::copy(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update) { + std::list &to_update) { if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"copy ns({}) {} to {} is not possible as there is no cluster connection available ",src_ns, src_oid, dest_oid); return -1; } @@ -483,7 +508,7 @@ int RadosStorageImpl::copy(std::string &src_oid, const char *src_ns, std::string librados::IoCtx src_io_ctx, dest_io_ctx; // destination io_ctx is current io_ctx - dest_io_ctx = io_ctx; + dest_io_ctx = get_io_ctx(); if (strcmp(src_ns, dest_ns) != 0) { src_io_ctx.dup(dest_io_ctx); @@ -507,8 +532,13 @@ int RadosStorageImpl::copy(std::string &src_oid, const char *src_ns, std::string write_op.mtime(&save_time); // update metadata - for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { - write_op.setxattr((*it).key.c_str(), (*it).bl); + ceph::bufferlist bl; + std::string bl_str; + for (std::list::iterator it = to_update.begin(); it != to_update.end(); ++it) { + bl_str = std::string(((ceph::bufferlist*)(*it)->get_buffer())->c_str()); + bl.append(bl_str.c_str(), bl_str.size() + 1); + write_op.setxattr((*it)->get_key().c_str(), bl); + bl.clear(); } int ret = 0; librados::AioCompletion *completion = librados::Rados::aio_create_completion(); @@ -524,52 +554,68 @@ int RadosStorageImpl::copy(std::string &src_oid, const char *src_ns, std::string return ret; } -// DEPRECATED => MOVED to rbox-save.cpp -// if save_async = true, don't forget to call wait_for_rados_operations e.g. wait_for_write_operations_complete -// to wait for completion and free resources. -bool RadosStorageImpl::save_mail(librados::ObjectWriteOperation *write_op_xattr, - RadosMail *mail) { - +bool RadosStorageImpl::save_mail(storage_interface::RadosMail *current_object){ + if (!cluster->is_connected() || !io_ctx_created) { + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"save_mail ({}) is not possible as there is no cluster connection available ",*current_object->get_oid()); return false; } - if (write_op_xattr == nullptr || mail == nullptr) { + bool ret_val=false; + int object_size = current_object->get_mail_size(); + int max_object_size = this->get_max_object_size(); + if( max_object_size < object_size ||object_size<0||max_object_size==0){ + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"save_mail ({}) is not possible as the object exceeds the max object size {}",*current_object->get_oid(),max_object_size); return false; } - time_t save_date = mail->get_rados_save_date(); - write_op_xattr->mtime(&save_date); - uint32_t max_op_size = get_max_write_size_bytes() - 1024; - //TODO: make this configurable - int ret = split_buffer_and_exec_op(mail, write_op_xattr, 10240); - if (ret != 0) { - mail->set_active_op(0); - } - return ret == 0; -} -// if save_async = true, don't forget to call wait_for_rados_operations e.g. wait_for_write_operations_complete -// to wait for completion and free resources. -bool RadosStorageImpl::save_mail(RadosMail *mail) { - if (!cluster->is_connected() || !io_ctx_created) { - return false; - } - // delete write_op_xattr is called after operation completes (wait_for_rados_operations) - librados::ObjectWriteOperation write_op_xattr; // = new librados::ObjectWriteOperation(); - // set metadata - for (std::map::iterator it = mail->get_metadata()->begin(); - it != mail->get_metadata()->end(); ++it) { - write_op_xattr.setxattr(it->first.c_str(), it->second); - } + int max_write = get_max_write_size_bytes(); + uint64_t rest = object_size % max_write; + int div = object_size / max_write + (rest > 0 ? 1 : 0); + librados::bufferlist *mail_buffer = (librados::bufferlist*) current_object->get_mail_buffer(); + for (int i = 0; i < div; ++i) { + + librados::bufferlist tmp_buffer; + int offset = i * max_write; + + uint64_t length = max_write; + if (object_size < ((i + 1) * length)) { + length = rest; + } - if (mail->get_extended_metadata()->size() > 0) { - write_op_xattr.omap_set(*mail->get_extended_metadata()); + if (div == 1) { + librados::ObjectWriteOperation write_op; + write_op.write(0,*mail_buffer); + ret_val=execute_operation(*current_object->get_oid(), &write_op); + } + else { + if(offset + length >object_size){ + return false; + }else{ + if(offset + length > mail_buffer->length() ){ + tmp_buffer.substr_of(*mail_buffer, offset,mail_buffer->length() - offset ); + }else{ + tmp_buffer.substr_of(*mail_buffer, offset, length); + } + } + ret_val = append_to_object(*current_object->get_oid(), tmp_buffer, length); + } + if(!ret_val){ + RadosPluginLogger::getInstance().log(librmb::LOG_LEVEL::ERROR,"save_mail ({}) failed while writing to cluster",*current_object->get_oid()); + return ret_val; + } } - return save_mail(&write_op_xattr, mail); + // delete rados_metadata_storage; + // rados_metadata_storage=nullptr; + return ret_val; +} + +storage_interface::RadosMail *RadosStorageImpl::alloc_rados_mail() { + storage_interface::RadosMail *mail=new librmb::RadosMailImpl(); + mail->set_mail_buffer((void*) new librados::bufferlist()); + return mail; } -librmb::RadosMail *RadosStorageImpl::alloc_rados_mail() { return new librmb::RadosMail(); } - -void RadosStorageImpl::free_rados_mail(librmb::RadosMail *mail) { +void RadosStorageImpl::free_rados_mail(storage_interface::RadosMail *mail) { if (mail != nullptr) { delete mail; mail = nullptr; @@ -585,20 +631,25 @@ uint64_t RadosStorageImpl::ceph_index_size(){ int RadosStorageImpl::ceph_index_append(const std::string &oid) { librados::bufferlist bl; - bl.append(RadosUtils::convert_to_ceph_index(oid)); + librmb::RadosUtilsImpl rados_utils; + bl.append(rados_utils.convert_to_ceph_index(oid)); return get_recovery_io_ctx().append( get_namespace(),bl, bl.length()); } int RadosStorageImpl::ceph_index_append(const std::set &oids) { librados::bufferlist bl; - bl.append(RadosUtils::convert_to_ceph_index(oids)); + librmb::RadosUtilsImpl rados_utils; + bl.append(rados_utils.convert_to_ceph_index(oids)); return get_recovery_io_ctx().append( get_namespace(),bl, bl.length()); } + int RadosStorageImpl::ceph_index_overwrite(const std::set &oids) { librados::bufferlist bl; - bl.append(RadosUtils::convert_to_ceph_index(oids)); + librmb::RadosUtilsImpl rados_utils; + bl.append(rados_utils.convert_to_ceph_index(oids)); return get_recovery_io_ctx().write_full( get_namespace(),bl); } + std::set RadosStorageImpl::ceph_index_read() { std::set index; librados::bufferlist bl; @@ -609,17 +660,36 @@ std::set RadosStorageImpl::ceph_index_read() { if(psize <=0){ return index; } - //std::cout << " NAMESPACE: " << get_namespace() << " exist? " << exist << " size : " << psize << std::endl; int ret = get_recovery_io_ctx().read(get_namespace(),bl, max,0); - - if(ret < 0){ return index; } - index = RadosUtils::ceph_index_to_set(bl.c_str()); + librmb::RadosUtilsImpl rados_utils; + index = rados_utils.ceph_index_to_set(bl.c_str()); return index; } + int RadosStorageImpl::ceph_index_delete() { return get_recovery_io_ctx().remove(get_namespace()); } +void* RadosStorageImpl::alloc_mail_buffer(){ + return (void*)new librados::bufferlist(); +} + +const char* RadosStorageImpl::get_mail_buffer(void *buffer,int *mail_buff_size){ + *mail_buff_size=((librados::bufferlist*)buffer)->length(); + return ((librados::bufferlist*)buffer)->c_str(); +} + +void RadosStorageImpl::free_mail_buffer(void* mail_buffer_){ + if(mail_buffer_!=nullptr){ + librados::bufferlist *buffer=(librados::bufferlist*)mail_buffer_; + delete buffer; + mail_buffer_=nullptr; + } +} + +void RadosStorageImpl::append_to_buffer(void *buff,const unsigned char * chunk, size_t size){ + ((librados::bufferlist*)buff)->append(chunk,size); +} diff --git a/src/librmb/rados-storage-impl.h b/src/librmb/rados-storage-impl.h index c4320859..490854cd 100644 --- a/src/librmb/rados-storage-impl.h +++ b/src/librmb/rados-storage-impl.h @@ -23,39 +23,42 @@ #include #include -#include "rados-mail.h" -#include "rados-storage.h" -namespace librmb { +#include "../storage-interface/rados-mail.h" +#include "../storage-interface/rbox-io-ctx.h" +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-metadata.h" + -class RadosStorageImpl : public RadosStorage { +namespace librmb { +class RadosStorageImpl : public storage_interface::RadosStorage { public: - explicit RadosStorageImpl(RadosCluster *cluster); + explicit RadosStorageImpl(storage_interface::RadosCluster *cluster); virtual ~RadosStorageImpl(); - - librados::IoCtx &get_io_ctx() override; - librados::IoCtx &get_recovery_io_ctx() override; - + librados::IoCtx &get_io_ctx(); + storage_interface::RboxIoCtx* get_io_ctx_wrapper()override; + void set_io_ctx_wrapper(storage_interface::RboxIoCtx* io_ctx_){ + if(io_ctx_wrapper!=nullptr){ + delete io_ctx_wrapper; + io_ctx_wrapper=io_ctx_; + } + } + int stat_mail(const std::string &oid, uint64_t *psize, time_t *pmtime) override; void set_namespace(const std::string &_nspace) override; std::string get_namespace() override { return nspace; } std::string get_pool_name() override { return pool_name; } - void set_ceph_wait_method(enum rbox_ceph_aio_wait_method wait_method_) { this->wait_method = wait_method_; } + void set_ceph_wait_method(storage_interface::rbox_ceph_aio_wait_method wait_method_) override { this->wait_method = wait_method_; } int get_max_write_size() override { return max_write_size; } int get_max_write_size_bytes() override { return max_write_size * 1024 * 1024; } int get_max_object_size() override {return max_object_size;} - int split_buffer_and_exec_op(RadosMail *current_object, librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write) override; - - int delete_mail(RadosMail *mail) override; int delete_mail(const std::string &oid) override; - int aio_operate(librados::IoCtx *io_ctx_, const std::string &oid, librados::AioCompletion *c, - librados::ObjectWriteOperation *op) override; - librados::NObjectIterator find_mails(const RadosMetadata *attr) override; + + std::set find_mails(const storage_interface::RadosMetadata *attr) override; - std::set find_mails_async(const RadosMetadata *attr, std::string &pool_name, int num_threads, void (*ptr)(std::string&)) override; + std::set find_mails_async(const storage_interface::RadosMetadata *attr, std::string &pool_name, int num_threads, void (*ptr)(std::string&)) override; int open_connection(const std::string &poolname) override; int open_connection(const std::string &poolname, const std::string &index_pool) override; @@ -67,23 +70,16 @@ class RadosStorageImpl : public RadosStorage { int open_connection(const std::string &poolname, const std::string &clustername, const std::string &rados_username) override; void close_connection() override; - bool wait_for_write_operations_complete(librados::AioCompletion *completion, - librados::ObjectWriteOperation *write_operation) override; - - bool wait_for_rados_operations(const std::list &object_list) override; - - int read_mail(const std::string &oid, librados::bufferlist *buffer) override; + // int read_mail(const std::string &oid, librados::bufferlist *buffer) override; + int read_mail(const std::string &oid, storage_interface::RadosMail* mail,int try_counter) override; int move(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update, bool delete_source) override; + std::list &to_update, bool delete_source) override; int copy(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update) override; - - int save_mail(const std::string &oid, librados::bufferlist &buffer) override; - bool save_mail(RadosMail *mail) override; - bool save_mail(librados::ObjectWriteOperation *write_op_xattr, RadosMail *mail) override; - librmb::RadosMail *alloc_rados_mail() override; + std::list &to_update) override; + bool save_mail(storage_interface::RadosMail *mail) override; + storage_interface::RadosMail *alloc_rados_mail() override; - void free_rados_mail(librmb::RadosMail *mail) override; + void free_rados_mail(storage_interface::RadosMail *mail) override; uint64_t ceph_index_size() override; int ceph_index_append(const std::string &oid) override; @@ -91,28 +87,36 @@ class RadosStorageImpl : public RadosStorage { int ceph_index_overwrite(const std::set &oids) override; std::set ceph_index_read() override; int ceph_index_delete() override; + void* alloc_mail_buffer() override; + const char* get_mail_buffer(void *buffer,int *mail_buff_size) override; + void free_mail_buffer(void* mail_buffer) override; + void append_to_buffer(void *buff,const unsigned char * chunk, size_t size) override; + + bool execute_operation(std::string &oid, librados::ObjectWriteOperation *write_op_xattr); + bool append_to_object(std::string &oid, librados::bufferlist &bufferlist, int length); private: int create_connection(const std::string &poolname,const std::string &index_pool); - + librados::IoCtx &get_recovery_io_ctx(); + int split_buffer_and_exec_op(storage_interface::RadosMail *current_object, librados::ObjectWriteOperation *write_op_xattr, + const uint64_t &max_write); + int aio_operate(librados::IoCtx *io_ctx_, const std::string &oid, librados::AioCompletion *c, + librados::ObjectWriteOperation *op); + private: - RadosCluster *cluster; + storage_interface::RadosCluster *cluster; int max_write_size; int max_object_size; std::string nspace; - librados::IoCtx io_ctx; - librados::IoCtx recovery_io_ctx; - + storage_interface::RboxIoCtx* io_ctx_wrapper; + bool io_ctx_created; std::string pool_name; - enum rbox_ceph_aio_wait_method wait_method; + storage_interface::rbox_ceph_aio_wait_method wait_method; static const char *CFG_OSD_MAX_WRITE_SIZE; static const char *CFG_OSD_MAX_OBJECT_SIZE; }; - - - } // namespace librmb #endif // SRC_LIBRMB_RADOS_MAIL_STORAGE_IMPL_H_ diff --git a/src/librmb/rados-util.cpp b/src/librmb/rados-util-impl.cpp similarity index 56% rename from src/librmb/rados-util.cpp rename to src/librmb/rados-util-impl.cpp index 2668bbb9..5d681ee3 100644 --- a/src/librmb/rados-util.cpp +++ b/src/librmb/rados-util-impl.cpp @@ -13,7 +13,7 @@ #include "dovecot-ceph-plugin-config.h" #endif -#include "rados-util.h" +#include "rados-util-impl.h" #include #include #include @@ -23,14 +23,16 @@ #include #include #include "encoding.h" +#include "rados-mail-impl.h" +#include "rados-metadata-impl.h" namespace librmb { - RadosUtils::RadosUtils() {} + RadosUtilsImpl::RadosUtilsImpl() {} - RadosUtils::~RadosUtils() {} + RadosUtilsImpl::~RadosUtilsImpl() {} - bool RadosUtils::convert_str_to_time_t(const std::string &date, time_t *val) { + bool RadosUtilsImpl::convert_str_to_time_t(const std::string &date, time_t *val) { struct tm tm = {0}; if (strptime(date.c_str(), "%Y-%m-%d %H:%M:%S", &tm)) { tm.tm_isdst = -1; @@ -42,7 +44,7 @@ namespace librmb { return false; } - bool RadosUtils::convert_string_to_date(const std::string &date_string, std::string *date) { + bool RadosUtilsImpl::convert_string_to_date(const std::string &date_string, std::string *date) { time_t t; if (convert_str_to_time_t(date_string, &t)) { *date = std::to_string(t); @@ -51,7 +53,7 @@ namespace librmb { return false; } - bool RadosUtils::is_numeric(const char *s) { + bool RadosUtilsImpl::is_numeric(const char *s) { if (s == NULL) { return false; } @@ -63,15 +65,14 @@ namespace librmb { break; } } - return is_numeric; } - bool RadosUtils::is_date_attribute(const rbox_metadata_key &key) { - return (key == RBOX_METADATA_OLDV1_SAVE_TIME || key == RBOX_METADATA_RECEIVED_TIME); + bool RadosUtilsImpl::is_date_attribute(const storage_interface::rbox_metadata_key &key) { + return (key == storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME || key == storage_interface::RBOX_METADATA_RECEIVED_TIME); } - int RadosUtils::convert_time_t_to_str(const time_t &t, std::string *ret_val) { + int RadosUtilsImpl::convert_time_t_to_str(const time_t &t, std::string *ret_val) { char buffer[256]; if (t == -1) { *ret_val = "invalid date"; @@ -83,14 +84,14 @@ namespace librmb { *ret_val = std::string(buffer); return 0; } - bool RadosUtils::flags_to_string(const uint8_t &flags_, std::string *flags_str) { + bool RadosUtilsImpl::flags_to_string(const uint8_t &flags_, std::string *flags_str) { std::stringstream sstream; sstream << std::hex << flags_; sstream >> *flags_str; return true; } - bool RadosUtils::string_to_flags(const std::string &flags_, uint8_t *flags) { + bool RadosUtilsImpl::string_to_flags(const std::string &flags_, uint8_t *flags) { std::istringstream in(flags_); if (in >> std::hex >> *flags) { @@ -99,15 +100,15 @@ namespace librmb { return false; } - void RadosUtils::find_and_replace(std::string *source, std::string const &find, std::string const &replace) { + void RadosUtilsImpl::find_and_replace(std::string *source, std::string const &find, std::string const &replace) { for (std::string::size_type i = 0; source != nullptr && (i = source->find(find, i)) != std::string::npos;) { source->replace(i, find.length(), replace); i += replace.length(); } } - int RadosUtils::get_all_keys_and_values(librados::IoCtx *io_ctx, const std::string &oid, - std::map *kv_map) { + int RadosUtilsImpl::get_all_keys_and_values(storage_interface::RboxIoCtx *io_ctx, const std::string &oid, + storage_interface::RadosMail* mail) { int err = 0; librados::ObjectReadOperation first_read; std::set extended_keys; @@ -120,10 +121,18 @@ namespace librmb { if (ret < 0) { return ret; } - return io_ctx->omap_get_vals_by_keys(oid, extended_keys, kv_map); + std::map ceph_metadata; + ret= io_ctx->omap_get_vals_by_keys(oid, extended_keys, &ceph_metadata); + for(std::map::iterator it=ceph_metadata.begin(); it!=ceph_metadata.end(); ++it){ + std::string key = (*it).first; + std::string value = std::string((*it).second.c_str()); + storage_interface::RadosMetadata *xattr= new librmb::RadosMetadataImpl(key,value); + mail->add_extended_metadata(xattr); + } + return ret; } - void RadosUtils::resolve_flags(const uint8_t &flags, std::string *flat) { + void RadosUtilsImpl::resolve_flags(const uint8_t &flags, std::string *flat) { std::stringbuf buf; std::ostream os(&buf); @@ -148,7 +157,7 @@ namespace librmb { *flat = buf.str(); } - int RadosUtils::osd_add(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, + int RadosUtilsImpl::osd_add(storage_interface::RboxIoCtx *ioctx, const std::string &oid, const std::string &key, long long value_to_add) { librados::bufferlist in, out; encode(key, in); @@ -161,7 +170,7 @@ namespace librmb { return ioctx->exec(oid, "numops", "add", in, out); } - int RadosUtils::osd_sub(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, + int RadosUtilsImpl::osd_sub(storage_interface::RboxIoCtx *ioctx, const std::string &oid, const std::string &key, long long value_to_subtract) { return osd_add(ioctx, oid, key, -value_to_subtract); } @@ -170,49 +179,48 @@ namespace librmb { * @return reference to all write operations related with this object */ - void RadosUtils::get_metadata(const std::string &key, std::map *metadata, char **value) { + void RadosUtilsImpl::get_metadata(const std::string &key, std::map *metadata, char **value) { if (metadata->find(key) != metadata->end()) { - *value = (*metadata)[key].c_str(); + *value = ((ceph::bufferlist*)(*metadata)[key])->c_str(); return; } *value = NULL; } - void RadosUtils::get_metadata(rbox_metadata_key key, std::map *metadata, char **value) { - string str_key(librmb::rbox_metadata_key_to_char(key)); + void RadosUtilsImpl::get_metadata(storage_interface::rbox_metadata_key key, std::map *metadata, char **value) { + std::string str_key(storage_interface::rbox_metadata_key_to_char(key)); get_metadata(str_key, metadata, value); } - bool RadosUtils::is_numeric_optional(const char *text) { + bool RadosUtilsImpl::is_numeric_optional(const char *text) { if (text == NULL) { return true; // optional } return is_numeric(text); } - bool RadosUtils::validate_metadata(map *metadata) { + bool RadosUtilsImpl::validate_metadata(std::map *metadata) { char *uid = NULL; - get_metadata(RBOX_METADATA_MAIL_UID, metadata, &uid); + get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, metadata, &uid); char *recv_time_str = NULL; - get_metadata(RBOX_METADATA_RECEIVED_TIME, metadata, &recv_time_str); + get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, metadata, &recv_time_str); char *p_size = NULL; - get_metadata(RBOX_METADATA_PHYSICAL_SIZE, metadata, &p_size); + get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, metadata, &p_size); char *v_size = NULL; - get_metadata(RBOX_METADATA_VIRTUAL_SIZE, metadata, &v_size); - + get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, metadata, &v_size); char *rbox_version; - get_metadata(RBOX_METADATA_VERSION, metadata, &rbox_version); + get_metadata(storage_interface::RBOX_METADATA_VERSION, metadata, &rbox_version); char *mailbox_guid = NULL; - get_metadata(RBOX_METADATA_MAILBOX_GUID, metadata, &mailbox_guid); + get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, metadata, &mailbox_guid); char *mail_guid = NULL; - get_metadata(RBOX_METADATA_GUID, metadata, &mail_guid); + get_metadata(storage_interface::RBOX_METADATA_GUID, metadata, &mail_guid); char *mb_orig_name = NULL; - get_metadata(RBOX_METADATA_ORIG_MAILBOX, metadata, &mb_orig_name); + get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, metadata, &mb_orig_name); char *flags = NULL; - get_metadata(RBOX_METADATA_OLDV1_FLAGS, metadata, &flags); + get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, metadata, &flags); char *pvt_flags = NULL; - get_metadata(RBOX_METADATA_PVT_FLAGS, metadata, &pvt_flags); + get_metadata(storage_interface::RBOX_METADATA_PVT_FLAGS, metadata, &pvt_flags); char *from_envelope = NULL; - get_metadata(RBOX_METADATA_FROM_ENVELOPE, metadata, &from_envelope); + get_metadata(storage_interface::RBOX_METADATA_FROM_ENVELOPE, metadata, &from_envelope); int test = 0; test += is_numeric(uid) ? 0 : 1; @@ -222,27 +230,28 @@ namespace librmb { test += is_numeric_optional(flags) ? 0 : 1; test += is_numeric_optional(pvt_flags) ? 0 : 1; - + test += mailbox_guid == NULL ? 1 : 0; test += mail_guid == NULL ? 1 : 0; return test == 0; } // assumes that destination is open and initialized with uses namespace - int RadosUtils::move_to_alt(std::string &oid, RadosStorage *primary, RadosStorage *alt_storage, - RadosMetadataStorage *metadata, bool inverse) { + int RadosUtilsImpl::move_to_alt(std::string &oid, storage_interface::RadosStorage *primary, storage_interface::RadosStorage *alt_storage, + storage_interface::RadosMetadataStorage *metadata, bool inverse) { int ret = -1; ret = copy_to_alt(oid, oid, primary, alt_storage, metadata, inverse); if (ret > 0) { if (inverse) { - ret = alt_storage->get_io_ctx().remove(oid); + ret = alt_storage->get_io_ctx_wrapper()->remove(oid); } else { - ret = primary->get_io_ctx().remove(oid); + ret = primary->get_io_ctx_wrapper()->remove(oid); } } return ret; } - int RadosUtils::copy_to_alt(std::string &src_oid, std::string &dest_oid, RadosStorage *primary, - RadosStorage *alt_storage, RadosMetadataStorage *metadata, bool inverse) { + int RadosUtilsImpl::copy_to_alt(std::string &src_oid, std::string &dest_oid, storage_interface::RadosStorage *primary, + storage_interface::RadosStorage *alt_storage, + storage_interface::RadosMetadataStorage *metadata, bool inverse) { int ret = 0; // TODO(jrse) check that storage is connected and open. @@ -250,52 +259,42 @@ namespace librmb { return 0; } - RadosMail mail; - mail.set_oid(src_oid); - - librados::bufferlist *bl = new librados::bufferlist(); - mail.set_mail_buffer(bl); - + storage_interface::RadosMail* mail; if (inverse) { - ret = alt_storage->read_mail(src_oid, mail.get_mail_buffer()); - metadata->get_storage()->set_io_ctx(&alt_storage->get_io_ctx()); + mail=alt_storage->alloc_rados_mail(); + ret = alt_storage->read_mail(src_oid,mail,0); + metadata->get_storage()->set_io_ctx(alt_storage->get_io_ctx_wrapper()); } else { - ret = primary->read_mail(src_oid, mail.get_mail_buffer()); + mail=primary->alloc_rados_mail(); + ret = primary->read_mail(src_oid,mail,0); + metadata->get_storage()->set_io_ctx(primary->get_io_ctx_wrapper()); } if (ret < 0) { - metadata->get_storage()->set_io_ctx(&primary->get_io_ctx()); - return ret; + return ret; } - mail.set_mail_size(mail.get_mail_buffer()->length()); + mail->set_mail_size(((librados::bufferlist*)mail->get_mail_buffer())->length()); // load the metadata; - ret = metadata->get_storage()->load_metadata(&mail); + mail->set_oid(dest_oid); + ret = metadata->get_storage()->load_metadata(mail); if (ret < 0) { return ret; } - - mail.set_oid(dest_oid); - - librados::ObjectWriteOperation write_op; // = new librados::ObjectWriteOperation(); - metadata->get_storage()->save_metadata(&write_op, &mail); - + ret = metadata->get_storage()->set_metadata(mail); + if(ret<0){ + return ret; + } bool success; if (inverse) { - success = primary->save_mail(&write_op, &mail); + success = primary->save_mail(mail); } else { - success = alt_storage->save_mail(&write_op, &mail); + success = alt_storage->save_mail(mail); } - - if (!success) { - return 0; - } - return success ? 0 : 1; } - static std::vector RadosUtils::extractPgs(const std::string& str) - { + std::vector RadosUtilsImpl::extractPgs(const std::string& str){ std::vector tokens; std::stringstream ss(str); @@ -314,8 +313,7 @@ namespace librmb { return tokens; } - static std::map> RadosUtils::extractPgAndPrimaryOsd(const std::string& str) - { + std::map> RadosUtilsImpl::extractPgAndPrimaryOsd(const std::string& str){ std::map> tokens; std::stringstream ss(str); @@ -348,7 +346,7 @@ namespace librmb { return tokens; } - static std::vector RadosUtils::split(std::string str_to_split, char delimiter) { + std::vector RadosUtilsImpl::split(std::string str_to_split, char delimiter) { std::vector tokens; std::stringstream stream(str_to_split); std::string token; @@ -364,17 +362,17 @@ namespace librmb { return tokens; } - static std::string RadosUtils::convert_to_ceph_index(const std::set &list){ + std::string RadosUtilsImpl::convert_to_ceph_index(const std::set &list){ std::ostringstream str; - std::copy(list.begin(), list.end(), std::ostream_iterator(str, ",")); + std::copy(list.begin(), list.end(), std::ostream_iterator(str, ",")); return str.str(); } - static std::string RadosUtils::convert_to_ceph_index(const std::string &str) { + std::string RadosUtilsImpl::convert_to_ceph_index(const std::string &str) { return str + ","; } - static std::set RadosUtils::ceph_index_to_set(const std::string &str) { + std::set RadosUtilsImpl::ceph_index_to_set(const std::string &str) { std::set index; std::stringstream ss(str); diff --git a/src/librmb/rados-util-impl.h b/src/librmb/rados-util-impl.h new file mode 100644 index 00000000..c557a29f --- /dev/null +++ b/src/librmb/rados-util-impl.h @@ -0,0 +1,207 @@ +// -*- mode:C++ ; tab-width:8 ; c-basic-offset:2 ; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software ; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_RADOS_UTIL_IMPL_H_ +#define SRC_LIBRMB_RADOS_UTIL_IMPL_H_ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-metadata-storage.h" +#include "../storage-interface/rados-types.h" +#include "../storage-interface/rados-util.h" + +namespace librmb { + +/** + * Rados Utils + * + * Utility class with usefull helper functions. + * + */ +class RadosUtilsImpl : public storage_interface::RadosUtils { + public: + RadosUtilsImpl() ; + virtual ~RadosUtilsImpl(); + /*! + * convert given date string to time_t + * + * @param[in] date Date format: %Y-%m-%d %H:%M:%S + * @param[out] val ptr to time_t. + * @return true if sucessfull. + */ + bool convert_str_to_time_t(const std::string &date, time_t *val) override; + /*! + * check if given string is a numeric value. + * @param[in] s string if s is empty => false + * @return true if given string is numeric. + */ + bool is_numeric(const char *s) override; + /*! + * check if given string is a numeric value. + * @param[in] text string, if string is empty => true + * @return true if given string is numeric. + */ + bool is_numeric_optional(const char *text) override; + /*! + * checks if key is a data attribute + */ + bool is_date_attribute(const storage_interface::rbox_metadata_key &key) override; + /*! + * converts given data_string to numeric string + * @param[in] date_string Date format: %Y-%m-%d %H:%M:%S + * @param[out] date : unix timestamp. + */ + bool convert_string_to_date(const std::string &date_string, std::string *date) override; + /*! + * converts given time_to to string %Y-%m-%d %H:%M:%S + * @param[in] t time_t + * @param[out] ret_val : ptr to valid string buffer. + * @return <0 error + */ + int convert_time_t_to_str(const time_t &t, std::string *ret_val) override; + /*! + * converts flags to hex string + * @param[in] flags flags + * @param[out] ptr to string buffer: + * @return false if not sucessful + */ + bool flags_to_string(const uint8_t &flags, std::string *flags_str) override; + + /*! + * converts hex string to uint8_t + * @param[in] flags_str flags (e.g. 0x03 + * @param[out] flags to uint8_t + * @return false if not sucessful + */ + bool string_to_flags(const std::string &flags_str, uint8_t *flags) override; + + /*! + * replace string in text. + * @param[in,out] source: source string + * @param[in] find : text to find. + * @param[in] replace: text to replace. + */ + void find_and_replace(std::string *source, std::string const &find, std::string const &replace) override; + + /*! + * get a list of key value pairs + * @param[in] io_ctx valid io_ctx + * @param[in] oid: unique identifier + * @param[out] kv_map valid ptr to key value map. + */ + int get_all_keys_and_values(storage_interface::RboxIoCtx *io_ctx, const std::string &oid, + storage_interface::RadosMail* mail) override; + /*! + * get the text representation of uint flags. + * @param[in] flags + * @param[out] flat : string representation + */ + void resolve_flags(const uint8_t &flags, std::string *flat) override; + /*! + * copy object to alternative storage + * @param[in] src_oid + * @param[in] dest_oid + * @param[in] primary rados primary storage + * @param[in] alt_storage rados alternative storage + * @param[in] metadata storage + * @param[in] bool inverse if true, copy from alt to primary. + * @return linux error code or 0 if sucessful + */ + int copy_to_alt(std::string &src_oid, std::string &dest_oid, storage_interface::RadosStorage *primary, storage_interface::RadosStorage *alt_storage, + storage_interface::RadosMetadataStorage *metadata, bool inverse) override; + /*! + * move object to alternative storage + * @param[in] src_oid + * @param[in] dest_oid + * @param[in] primary rados primary storage + * @param[in] alt_storage rados alternative storage + * @param[in] metadata storage + * @param[in] bool inverse if true, move from alt to primary. + * @return linux error code or 0 if sucessful + */ + int move_to_alt(std::string &oid, storage_interface::RadosStorage *primary, storage_interface::RadosStorage *alt_storage, + storage_interface::RadosMetadataStorage *metadata, bool inverse) override; + /*! + * increment (add) value directly on osd + * @param[in] ioctx + * @param[in] oid + * @param[in] key + * @param[in] value_to_add + * + * @return linux error code or 0 if sucessful + */ + int osd_add(storage_interface::RboxIoCtx *ioctx, const std::string &oid, const std::string &key, long long value_to_add) override; + /*! + * decrement (sub) value directly on osd + * @param[in] ioctx + * @param[in] oid + * @param[in] key + * @param[in] value_to_subtract + * + * @return linux error code or 0 if sucessful + */ + int osd_sub(storage_interface::RboxIoCtx *ioctx, const std::string &oid, const std::string &key, + long long value_to_subtract) override; + + /*! + * check all given metadata key is valid + * + * @param[in] metadata + * @return true if all keys and value are correct. (type, name, value) + */ + bool validate_metadata(std::map *metadata) override; + /*! + * get metadata + * + * @param[in] key + * @param[int] valid pointer to metadata map + * @return the metadata value + */ + void get_metadata(const std::string &key, std::map *metadata, char **value) override; + + /*! + * get metadata + * + * @param[in] key + * @param[int] valid pointer to metadata map + * @return the metadata value + */ + void get_metadata(storage_interface::rbox_metadata_key key, std::map *metadata, char **value) override; + + + /** + * POC Implemnentation to extract pgs and primary osds from mon_command output! + **/ + std::vector extractPgs(const std::string& str) override; + + std::map> extractPgAndPrimaryOsd(const std::string& str) override; + + std::vector split(std::string str_to_split, char delimiter) override; + + + std::string convert_to_ceph_index(const std::set &list) override; + std::string convert_to_ceph_index(const std::string &str) override; + + std::set ceph_index_to_set(const std::string &str) override; +}; + +} // namespace librmb + +#endif // SRC_LIBRMB_RADOS_UTIL_IMPL_H_ diff --git a/src/librmb/rbox-io-ctx-impl.h b/src/librmb/rbox-io-ctx-impl.h new file mode 100644 index 00000000..a0ea631b --- /dev/null +++ b/src/librmb/rbox-io-ctx-impl.h @@ -0,0 +1,131 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_RBOX_IO_CTX_IMPL_H_ +#define SRC_LIBRMB_RBOX_IO_CTX_IMPL_H_ + + +#include +#include + +#include "../storage-interface/rbox-io-ctx.h" +namespace librmb{ +class RboxIoCtxImpl:public storage_interface::RboxIoCtx{ + + public: + + virtual ~RboxIoCtxImpl(){} + + RboxIoCtxImpl(): + remove_completion(nullptr), + push_back_completion(nullptr){} + + void set_io_ctx(librados::IoCtx& io_ctx_) override{ + io_ctx=io_ctx_; + } + librados::IoCtx& get_io_ctx() override{ + return io_ctx; + } + librados::IoCtx& get_recovery_io_ctx() override{ + return recovery_io_ctx; + } + int aio_stat(const std::string& oid,librados::AioCompletion *aio_complete,uint64_t *psize,time_t *pmtime) override{ + return get_io_ctx().aio_stat(oid,aio_complete,psize,pmtime); + } + int exec(const std::string& oid, const char *cls, const char *method,ceph::bufferlist& inbl, ceph::bufferlist& outbl) override{ + return get_io_ctx().exec(oid, cls, method, inbl, outbl); + } + int omap_get_vals_by_keys(const std::string& oid,const std::set& keys,std::map *vals)override{ + return get_io_ctx().omap_get_vals_by_keys(oid,keys,vals); + } + int omap_rm_keys(const std::string& oid,const std::set& keys)override{ + return get_io_ctx().omap_rm_keys(oid,keys); + } + int omap_set(const std::map& map,const std::string& oid)override{ + get_io_ctx().omap_set(oid,map); + } + void omap_set(const std::string& oid,const std::map& map)override{ + get_io_ctx().omap_set(oid,map); + } + int setxattr(const std::string& oid,const char *name, librados::bufferlist& bl)override{ + return get_io_ctx().setxattr(oid,name,bl); + } + int getxattrs(const std::string& oid,std::map& attrset)override{ + return get_io_ctx().getxattrs(oid,attrset); + } + const librados::NObjectIterator& nobjects_end() override{ + return get_io_ctx().nobjects_end(); + } + librados::NObjectIterator nobjects_begin() override{ + return get_io_ctx().nobjects_begin(); + } + librados::NObjectIterator nobjects_begin(const librados::bufferlist& filter) override{ + return get_io_ctx().nobjects_begin(filter); + } + void set_namespace(const std::string& nspace) override{ + get_io_ctx().set_namespace(nspace); + } + int stat(const std::string& oid, uint64_t *psize, time_t *pmtime)override{ + return get_io_ctx().stat(oid,psize,pmtime); + } + int aio_operate(const std::string& oid, librados::AioCompletion *aio_completion, + librados::ObjectReadOperation *read_op, librados::bufferlist *pbl)override{ + return get_io_ctx().aio_operate(oid,aio_completion,read_op,pbl); + } + int aio_operate(const std::string& oid, librados::AioCompletion *aio_completion, + librados::ObjectReadOperation *read_op, int flags,librados::bufferlist *pbl)override{ + return get_io_ctx().aio_operate(oid,aio_completion,read_op,flags,pbl); + } + int aio_operate(const std::string& oid, librados::AioCompletion *aio_complete, librados::ObjectWriteOperation *write_op)override{ + return get_io_ctx().aio_operate(oid,aio_complete,write_op); + } + int remove(const std::string& oid){ + return get_io_ctx().remove(oid); + } + int write_full(const std::string& oid, librados::bufferlist& bl) override{ + return get_io_ctx().write_full(oid,bl); + } + int read(const std::string& oid, librados::bufferlist& bl, size_t len, uint64_t off) override{ + return get_io_ctx().read(oid,bl,len,0); + } + int operate(const std::string& oid, librados::ObjectWriteOperation* write_op_xattr) override{ + return get_io_ctx().operate(oid,write_op_xattr); + } + bool append(const std::string& oid, librados::bufferlist& bufferlist, int length) override{ + return get_io_ctx().append(oid,bufferlist, length); + } + int operate( + const std::string& oid,librados::ObjectReadOperation* read_op,librados::bufferlist* buffer) override{ + return get_io_ctx().operate(oid,read_op,buffer); + } + uint64_t get_last_version()override{ + return get_io_ctx().get_last_version(); + } + void set_remove_completion(librados::AioCompletion &aio_commepletion)override{ + this->remove_completion=&aio_commepletion; + } + librados::AioCompletion& get_remove_completion()override{ + return *remove_completion; + } + void set_push_back_completion(librados::AioCompletion &aio_commepletion)override{ + this->remove_completion=&aio_commepletion; + } + librados::AioCompletion& get_push_back_completion()override{ + return *push_back_completion; + } + private: + librados::IoCtx io_ctx; + librados::IoCtx recovery_io_ctx; + librados::AioCompletion *remove_completion; + librados::AioCompletion *push_back_completion; +}; +} +#endif // SRC_LIBRMB_RBOX_IO_CTX_IMPL_H_ \ No newline at end of file diff --git a/src/librmb/tools/rmb/Makefile.am b/src/librmb/tools/rmb/Makefile.am index fb8e7072..29dccad2 100644 --- a/src/librmb/tools/rmb/Makefile.am +++ b/src/librmb/tools/rmb/Makefile.am @@ -15,14 +15,14 @@ shlibs = \ bin_PROGRAMS = rmb rmb_SOURCES = \ - mailbox_tools.cpp \ - mailbox_tools.h \ - ls_cmd_parser.cpp \ - ls_cmd_parser.h \ - rmb-commands.cpp \ - rmb-commands.h \ + mailbox_tools_impl.cpp \ + mailbox_tools_impl.h \ + ls_cmd_parser_impl.cpp \ + ls_cmd_parser_impl.h \ + rmb-commands-impl.cpp \ + rmb-commands-impl.h \ rmb.cpp \ - rados-mail-box.h + rados-mail-box-impl.h rmb_LDADD = $(shlibs) diff --git a/src/librmb/tools/rmb/ls_cmd_parser.cpp b/src/librmb/tools/rmb/ls_cmd_parser_impl.cpp similarity index 63% rename from src/librmb/tools/rmb/ls_cmd_parser.cpp rename to src/librmb/tools/rmb/ls_cmd_parser_impl.cpp index 38569301..64f1bf26 100644 --- a/src/librmb/tools/rmb/ls_cmd_parser.cpp +++ b/src/librmb/tools/rmb/ls_cmd_parser_impl.cpp @@ -9,37 +9,37 @@ * Foundation. See file COPYING. */ -#include "ls_cmd_parser.h" +#include "ls_cmd_parser_impl.h" #include #include namespace librmb { -CmdLineParser::~CmdLineParser() { +CmdLineParserImpl::~CmdLineParserImpl() { for (auto &it : predicates) { delete it.second; } } -Predicate *CmdLineParser::create_predicate(const std::string &_ls_value) { - Predicate *p = new Predicate(); +storage_interface::Predicate *CmdLineParserImpl::create_predicate(const std::string &_ls_value) { + storage_interface::Predicate *p = new librmb::PredicateImpl(); size_t pos = _ls_value.find("="); pos = (pos == std::string::npos) ? _ls_value.find(">") : pos; pos = (pos == std::string::npos) ? _ls_value.find("<") : pos; - p->key = _ls_value.substr(0, pos); - p->op = _ls_value[pos]; - p->value = _ls_value.substr(pos + 1, _ls_value.length()); - p->valid = true; + p->get_key() = _ls_value.substr(0, pos); + p->get_op() = _ls_value[pos]; + p->get_value() = _ls_value.substr(pos + 1, _ls_value.length()); + p->get_valid() = true; - this->keys += p->key + " "; + this->keys += p->get_key() + " "; // std::cout << " predicate: key " << p->key << " op " << p->op << " value " << p->value << std::endl; return p; } -void CmdLineParser::set_output_dir(const std::string& out) { +void CmdLineParserImpl::set_output_dir(const std::string& out) { if (out.length() > 0 && out.at(0) == '~') { // Convert tilde to $HOME path (if exists) char *home = getenv("HOME"); @@ -61,17 +61,17 @@ void CmdLineParser::set_output_dir(const std::string& out) { } } -bool CmdLineParser::parse_ls_string() { +bool CmdLineParserImpl::parse_ls_string() { std::string pred_sep = ";"; size_t pos = ls_value.find(pred_sep); if (pos == std::string::npos) { // single condition. - Predicate *p = create_predicate(ls_value); - if (p->valid) { - predicates[p->key] = p; + storage_interface::Predicate *p = create_predicate(ls_value); + if (p->get_valid()) { + predicates[p->get_key()] = p; } - return p->valid; + return p->get_valid(); } else { int offset = 0; @@ -79,20 +79,20 @@ bool CmdLineParser::parse_ls_string() { while (pos != std::string::npos) { tmp = tmp.substr(0, pos); - Predicate *p = create_predicate(tmp); - if (p->valid) { - predicates[p->key] = p; + storage_interface::Predicate *p = create_predicate(tmp); + if (p->get_valid()) { + predicates[p->get_key()] = p; } tmp = ls_value.substr(offset + pos + 1, ls_value.length()); offset += pos + 1; pos = tmp.find(pred_sep); } - Predicate *p = create_predicate(tmp); - if (p->valid) { - predicates[p->key] = p; + storage_interface::Predicate *p = create_predicate(tmp); + if (p->get_valid()) { + predicates[p->get_key()] = p; } - return p->valid; + return p->get_valid(); } } } // namespace librmb diff --git a/src/librmb/tools/rmb/ls_cmd_parser.h b/src/librmb/tools/rmb/ls_cmd_parser_impl.h similarity index 56% rename from src/librmb/tools/rmb/ls_cmd_parser.h rename to src/librmb/tools/rmb/ls_cmd_parser_impl.h index a1d0d3d5..f4b2b0f0 100644 --- a/src/librmb/tools/rmb/ls_cmd_parser.h +++ b/src/librmb/tools/rmb/ls_cmd_parser_impl.h @@ -9,8 +9,8 @@ * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_H_ -#define SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_H_ +#ifndef SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_IMPL_H_ +#define SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_IMPL_H_ #include #include @@ -18,8 +18,9 @@ #include #include -#include "../../rados-mail.h" -#include "rados-util.h" +#include "../../../storage-interface/rados-mail.h" +#include "../../../storage-interface/tools/rmb/ls_cmd_parser.h" +#include "../../rados-util-impl.h" #ifndef PATH_MAX #define PATH_MAX 256 @@ -27,17 +28,17 @@ namespace librmb { -class Predicate { +class PredicateImpl : public storage_interface::Predicate { public: - std::string key; - std::string op; - std::string value; // value to check against e.g. key > value - bool valid; + std::string& get_key() override{ return key;} + std::string& get_op() override{ return op;} + std::string& get_value() override{ return value;} + bool& get_valid() override{ return valid;} - bool eval(const std::string &_p_value) { - rbox_metadata_key rbox_key = static_cast(*key.c_str()); + bool eval(const std::string &_p_value) override{ + storage_interface::rbox_metadata_key rbox_key = static_cast(*key.c_str()); - if (rbox_key == RBOX_METADATA_RECEIVED_TIME || rbox_key == RBOX_METADATA_OLDV1_SAVE_TIME) { + if (rbox_key == storage_interface::RBOX_METADATA_RECEIVED_TIME || rbox_key == storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME) { // ref value time_t query_date = 0; convert_str_to_time_t(this->value, &query_date); @@ -62,8 +63,8 @@ class Predicate { } // time return true; - } else if (rbox_key == RBOX_METADATA_VIRTUAL_SIZE || rbox_key == RBOX_METADATA_PHYSICAL_SIZE || - rbox_key == RBOX_METADATA_MAIL_UID) { + } else if (rbox_key == storage_interface::RBOX_METADATA_VIRTUAL_SIZE || rbox_key == storage_interface::RBOX_METADATA_PHYSICAL_SIZE || + rbox_key == storage_interface::RBOX_METADATA_MAIL_UID) { uint64_t val = -1; uint64_t val2 = -1; try { @@ -92,39 +93,46 @@ class Predicate { return false; } - bool convert_str_to_time_t(const std::string &date, time_t *val) { - return librmb::RadosUtils::convert_str_to_time_t(date, val); + bool convert_str_to_time_t(const std::string &date, time_t *val) override{ + librmb::RadosUtilsImpl rados_util; + return rados_util.convert_str_to_time_t(date, val); } - int convert_time_t_to_str(const time_t &t, std::string *ret_val) { - return librmb::RadosUtils::convert_time_t_to_str(t, ret_val); + int convert_time_t_to_str(const time_t &t, std::string *ret_val) override{ + librmb::RadosUtilsImpl rados_util; + return rados_util.convert_time_t_to_str(t, ret_val); } + private: + std::string key; + std::string op; + std::string value; // value to check against e.g. key > value + bool valid; }; -class CmdLineParser { +class CmdLineParserImpl : public storage_interface::CmdLineParser{ public: - explicit CmdLineParser(const std::string &_ls_value) { + explicit CmdLineParserImpl(const std::string &_ls_value) { size_t pos = ls_value.find("\""); if (pos != std::string::npos) { this->ls_value = _ls_value.substr(1, _ls_value.length() - 1); } this->ls_value = _ls_value; } - ~CmdLineParser(); - bool parse_ls_string(); - std::map &get_predicates() { return this->predicates; } + ~CmdLineParserImpl(); + bool parse_ls_string() override; + std::map &get_predicates() override{ return this->predicates; } - bool contains_key(const std::string &key) { return keys.find(key) != keys.npos ? true : false; } - Predicate *get_predicate(const std::string &key) { return predicates[key]; } - Predicate *create_predicate(const std::string &ls_value); + bool contains_key(const std::string &key) override{ return keys.find(key) != keys.npos ? true : false; } + storage_interface::Predicate *get_predicate(const std::string &key) override{ return predicates[key]; } + storage_interface::Predicate *create_predicate(const std::string &ls_value) override; - void set_output_dir(const std::string &out); - std::string &get_output_dir() { return this->out_dir; } + void set_output_dir(const std::string &out) override; + std::string &get_output_dir() override{ return this->out_dir; } private: - std::map predicates; + std::map predicates; std::string ls_value; std::string keys; std::string out_dir; }; }; // namespace librmb -#endif // SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_H_ +#endif // SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_IMPL_H_ diff --git a/src/librmb/tools/rmb/mailbox_tools.h b/src/librmb/tools/rmb/mailbox_tools.h deleted file mode 100644 index d8d2cfec..00000000 --- a/src/librmb/tools/rmb/mailbox_tools.h +++ /dev/null @@ -1,42 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_TOOLS_RMB_MAILBOX_TOOLS_H_ -#define SRC_LIBRMB_TOOLS_RMB_MAILBOX_TOOLS_H_ - -#include - -#include "../../rados-mail.h" -#include "rados-mail-box.h" - -namespace librmb { -class MailboxTools { - public: - MailboxTools(librmb::RadosMailBox* mailbox, const std::string& base); - ~MailboxTools() {} - - int init_mailbox_dir(); - int save_mail(librmb::RadosMail* mail_obj); - int delete_mailbox_dir(); - int delete_mail(librmb::RadosMail* mail_obj); - - int build_filename(librmb::RadosMail* mail_obj, std::string* filename); - - std::string& get_mailbox_path() { return this->mailbox_path; } - - private: - librmb::RadosMailBox* mbox; - std::string base_path; - std::string mailbox_path; -}; -}; // namespace librmb - -#endif // SRC_LIBRMB_TOOLS_RMB_MAILBOX_TOOLS_H_ diff --git a/src/librmb/tools/rmb/mailbox_tools.cpp b/src/librmb/tools/rmb/mailbox_tools_impl.cpp similarity index 73% rename from src/librmb/tools/rmb/mailbox_tools.cpp rename to src/librmb/tools/rmb/mailbox_tools_impl.cpp index c2c003a2..040bee0b 100644 --- a/src/librmb/tools/rmb/mailbox_tools.cpp +++ b/src/librmb/tools/rmb/mailbox_tools_impl.cpp @@ -14,12 +14,12 @@ #include #include #include -#include "mailbox_tools.h" -#include "rados-util.h" +#include "mailbox_tools_impl.h" +#include "../../rados-util-impl.h" namespace librmb { -MailboxTools::MailboxTools(librmb::RadosMailBox* mailbox, const std::string& base) : mbox(mailbox), base_path(base) { +MailboxToolsImpl::MailboxToolsImpl(storage_interface::RadosMailBox* mailbox, const std::string& base) : mbox(mailbox), base_path(base) { if (base_path.empty()) { mailbox_path = this->mbox->get_mailbox_guid(); } else if (base_path[base_path.length() - 1] == '/') { @@ -29,7 +29,7 @@ MailboxTools::MailboxTools(librmb::RadosMailBox* mailbox, const std::string& bas } } -int MailboxTools::init_mailbox_dir() { +int MailboxToolsImpl::init_mailbox_dir() { struct stat st = {0}; if (stat(this->base_path.c_str(), &st) == -1) { @@ -47,7 +47,7 @@ int MailboxTools::init_mailbox_dir() { return 0; } -int MailboxTools::delete_mail(librmb::RadosMail* mail_obj) { +int MailboxToolsImpl::delete_mail(storage_interface::RadosMail* mail_obj) { if (mail_obj == nullptr) { return -1; } @@ -59,7 +59,7 @@ int MailboxTools::delete_mail(librmb::RadosMail* mail_obj) { return unlink(file_path.c_str()); } -int MailboxTools::delete_mailbox_dir() { +int MailboxToolsImpl::delete_mailbox_dir() { if (this->mailbox_path.empty() || this->base_path.empty()) { return -1; } @@ -72,7 +72,7 @@ int MailboxTools::delete_mailbox_dir() { return 0; } -int MailboxTools::save_mail(librmb::RadosMail* mail_obj) { +int MailboxToolsImpl::save_mail(storage_interface::RadosMail* mail_obj) { if (mail_obj == nullptr) { return -1; } @@ -86,19 +86,20 @@ int MailboxTools::save_mail(librmb::RadosMail* mail_obj) { if (!myfile.is_open()) { return -1; } - myfile.write(mail_obj->get_mail_buffer()->to_str().c_str(), mail_obj->get_mail_size()); + myfile.write(((librados::bufferlist*)mail_obj->get_mail_buffer())->c_str(), mail_obj->get_mail_size()); myfile.close(); return 0; } -int MailboxTools::build_filename(librmb::RadosMail* mail_obj, std::string* filename) { +int MailboxToolsImpl::build_filename(storage_interface::RadosMail* mail_obj, std::string* filename) { if (mail_obj == nullptr || !filename->empty()) { return -1; } std::stringstream ss; char* m_mail_uid; - RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail_obj->get_metadata(), &m_mail_uid); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, mail_obj->get_metadata(), &m_mail_uid); ss << m_mail_uid << "."; ss << *mail_obj->get_oid(); *filename = ss.str(); diff --git a/src/librmb/tools/rmb/mailbox_tools_impl.h b/src/librmb/tools/rmb/mailbox_tools_impl.h new file mode 100644 index 00000000..4cea04b0 --- /dev/null +++ b/src/librmb/tools/rmb/mailbox_tools_impl.h @@ -0,0 +1,43 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_TOOLS_RMB_MAILBOX_TOOLS_IMPL_H_ +#define SRC_LIBRMB_TOOLS_RMB_MAILBOX_TOOLS_IMPL_H_ + +#include + +#include "../../../storage-interface/rados-mail.h" +#include "../../../storage-interface/tools/rmb/mailbox_tools.h" +#include "../../../storage-interface/tools/rmb/rados-mail-box.h" + +namespace librmb { +class MailboxToolsImpl : public storage_interface::MailboxTools { + public: + MailboxToolsImpl(storage_interface::RadosMailBox* mailbox, const std::string& base); + ~MailboxToolsImpl() {} + + int init_mailbox_dir() override; + int save_mail(storage_interface::RadosMail* mail_obj) override; + int delete_mailbox_dir(); + int delete_mail(storage_interface::RadosMail* mail_obj) override; + + int build_filename(storage_interface::RadosMail* mail_obj, std::string* filename) override; + + std::string& get_mailbox_path() override{ return this->mailbox_path; } + + private: + storage_interface::RadosMailBox* mbox; + std::string base_path; + std::string mailbox_path; +}; +}; // namespace librmb + +#endif // SRC_LIBRMB_TOOLS_RMB_MAILBOX_TOOLS_IMPL_H_ diff --git a/src/librmb/tools/rmb/rados-mail-box-impl.h b/src/librmb/tools/rmb/rados-mail-box-impl.h new file mode 100644 index 00000000..6aefdc0b --- /dev/null +++ b/src/librmb/tools/rmb/rados-mail-box-impl.h @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_TOOLS_RMB_RADOS_MAIL_BOX_IMPL_H_ +#define SRC_LIBRMB_TOOLS_RMB_RADOS_MAIL_BOX_IMPL_H_ +#include +#include +#include +#include + +#include "../../../storage-interface/tools/rmb/ls_cmd_parser.h" +#include "../../rados-util-impl.h" +#include "../../../storage-interface/rados-mail.h" +namespace librmb { + +class RadosMailBoxImpl : public storage_interface::RadosMailBox { + public: + RadosMailBoxImpl(const std::string &_mailbox_guid, int _mail_count, const std::string &_mbox_orig_name) + : mailbox_guid(_mailbox_guid), mail_count(_mail_count), mbox_orig_name(_mbox_orig_name) { + this->mailbox_size = 0; + this->total_mails = 0; + this->parser = nullptr; + } + virtual ~RadosMailBoxImpl() {} + + void add_mail(storage_interface::RadosMail *mail) override{ + total_mails++; + if (!mail->is_valid()) { + mails.push_back(mail); + return; + } + if (parser == nullptr) { + mails.push_back(mail); + return; + } + if (parser->get_predicates().size() == 0) { + mails.push_back(mail); + return; + } + for (std::map::iterator it = parser->get_predicates().begin(); + it != parser->get_predicates().end(); ++it) { + std::map::iterator found_metadata=mail->get_metadata()->find(it->first); + std::map::iterator end_metadata= mail->get_metadata()->end(); + if (found_metadata->first != end_metadata->first || + ((ceph::bufferlist*)found_metadata->second)->to_str()!= ((ceph::bufferlist*)end_metadata->second)->to_str()) { + std::string key = it->first; + char *value; + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(key, mail->get_metadata(), &value); + if (it->second->eval(value)) { + mails.push_back(mail); + } + return; + } else if (it->first.compare("-") == 0) { + mails.push_back(mail); + } + } + } + + inline std::string to_string() { + std::ostringstream ss; + ss << std::endl + << "MAILBOX: " << static_cast(storage_interface::RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << this->mailbox_guid + << std::endl + << " " << static_cast(storage_interface::RBOX_METADATA_ORIG_MAILBOX) << "(mailbox_orig_name)=" << mbox_orig_name + << std::endl + + << " mail_total=" << total_mails << ", mails_displayed=" << mails.size() << std::endl + << " mailbox_size=" << mailbox_size << " bytes " << std::endl; + + std::string padding(" "); + for (std::list::iterator it = mails.begin(); it != mails.end(); ++it) { + ss << (*it)->to_string(padding); + } + return ss.str(); + } + void add_to_mailbox_size(const uint64_t &_mailbox_size) override{ this->mailbox_size += _mailbox_size; } + void set_mails(const std::list &_mails) override{ this->mails = _mails; } + + storage_interface::CmdLineParser *get_xattr_filter() override{ return this->parser; } + void set_xattr_filter(storage_interface::CmdLineParser *_parser) override{ this->parser = _parser; } + std::list &get_mails() override{ return this->mails; } + + std::string &get_mailbox_guid() override{ return this->mailbox_guid; } + void set_mailbox_guid(const std::string &_mailbox_guid) override{ this->mailbox_guid = _mailbox_guid; } + void set_mailbox_orig_name(const std::string &_mbox_orig_name) override{ this->mbox_orig_name = _mbox_orig_name; } + int &get_mail_count() override{ return this->mail_count; } + + private: + storage_interface::CmdLineParser *parser; + + std::string mailbox_guid; + int mail_count; + uint64_t mailbox_size; + std::list mails; + uint64_t total_mails; + std::string mbox_orig_name; +}; +} // namespace librmb + +#endif // SRC_LIBRMB_TOOLS_RMB_RADOS_MAIL_BOX_IMPL_H_ diff --git a/src/librmb/tools/rmb/rados-mail-box.h b/src/librmb/tools/rmb/rados-mail-box.h deleted file mode 100644 index 22e8516d..00000000 --- a/src/librmb/tools/rmb/rados-mail-box.h +++ /dev/null @@ -1,106 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_TOOLS_RMB_RMB_H_ -#define SRC_LIBRMB_TOOLS_RMB_RMB_H_ - -#include -#include -#include -#include - -#include "../../rados-mail.h" -#include "ls_cmd_parser.h" -#include "rados-util.h" -namespace librmb { - -class RadosMailBox { - public: - RadosMailBox(const std::string &_mailbox_guid, int _mail_count, const std::string &_mbox_orig_name) - : mailbox_guid(_mailbox_guid), mail_count(_mail_count), mbox_orig_name(_mbox_orig_name) { - this->mailbox_size = 0; - this->total_mails = 0; - this->parser = nullptr; - } - virtual ~RadosMailBox() {} - - void add_mail(RadosMail *mail) { - total_mails++; - if (!mail->is_valid()) { - mails.push_back(mail); - return; - } - if (parser == nullptr) { - mails.push_back(mail); - return; - } - if (parser->get_predicates().size() == 0) { - mails.push_back(mail); - return; - } - for (std::map::iterator it = parser->get_predicates().begin(); - it != parser->get_predicates().end(); ++it) { - if (mail->get_metadata()->find(it->first) != mail->get_metadata()->end()) { - std::string key = it->first; - char *value; - RadosUtils::get_metadata(key, mail->get_metadata(), &value); - if (it->second->eval(value)) { - mails.push_back(mail); - } - return; - } else if (it->first.compare("-") == 0) { - mails.push_back(mail); - } - } - } - - inline std::string to_string() { - std::ostringstream ss; - ss << std::endl - << "MAILBOX: " << static_cast(RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << this->mailbox_guid - << std::endl - << " " << static_cast(RBOX_METADATA_ORIG_MAILBOX) << "(mailbox_orig_name)=" << mbox_orig_name - << std::endl - - << " mail_total=" << total_mails << ", mails_displayed=" << mails.size() << std::endl - << " mailbox_size=" << mailbox_size << " bytes " << std::endl; - - std::string padding(" "); - for (std::list::iterator it = mails.begin(); it != mails.end(); ++it) { - ss << (*it)->to_string(padding); - } - return ss.str(); - } - inline void add_to_mailbox_size(const uint64_t &_mailbox_size) { this->mailbox_size += _mailbox_size; } - void set_mails(const std::list &_mails) { this->mails = _mails; } - - CmdLineParser *get_xattr_filter() { return this->parser; } - void set_xattr_filter(CmdLineParser *_parser) { this->parser = _parser; } - std::list &get_mails() { return this->mails; } - - std::string &get_mailbox_guid() { return this->mailbox_guid; } - void set_mailbox_guid(const std::string &_mailbox_guid) { this->mailbox_guid = _mailbox_guid; } - void set_mailbox_orig_name(const std::string &_mbox_orig_name) { this->mbox_orig_name = _mbox_orig_name; } - int &get_mail_count() { return this->mail_count; } - - private: - CmdLineParser *parser; - - std::string mailbox_guid; - int mail_count; - uint64_t mailbox_size; - std::list mails; - uint64_t total_mails; - std::string mbox_orig_name; -}; -} // namespace librmb - -#endif // SRC_LIBRMB_TOOLS_RMB_RMB_H_ diff --git a/src/librmb/tools/rmb/rmb-commands.cpp b/src/librmb/tools/rmb/rmb-commands-impl.cpp similarity index 64% rename from src/librmb/tools/rmb/rmb-commands.cpp rename to src/librmb/tools/rmb/rmb-commands-impl.cpp index 58c1dd15..8b28b243 100644 --- a/src/librmb/tools/rmb/rmb-commands.cpp +++ b/src/librmb/tools/rmb/rmb-commands-impl.cpp @@ -8,25 +8,29 @@ * License version 2.1, as published by the Free Software * Foundation. See file COPYING. */ - -#include "rmb-commands.h" +#include "rmb-commands-impl.h" #include #include // std::sort #include #include "../../rados-cluster-impl.h" #include "../../rados-storage-impl.h" -#include "rados-util.h" -#include "rados-dovecot-config.h" -#include "rados-dovecot-ceph-cfg-impl.h" -#include "rados-namespace-manager.h" -#include "rados-metadata-storage-ima.h" -#include "rados-metadata-storage-default.h" -#include "ls_cmd_parser.h" +#include "../../rados-util-impl.h" +#include "../../rados-dovecot-config.h" +#include "../../rados-dovecot-ceph-cfg-impl.h" +#include "../../rados-namespace-manager-impl.h" +#include "../../rados-metadata-storage-ima.h" +#include "../../rados-metadata-storage-default.h" +#include "../../rados-metadata-impl.h" +#include "../../../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../../rados-save-log-impl.h" +#include "ls_cmd_parser_impl.h" +#include "rados-mail-box-impl.h" +#include "mailbox_tools_impl.h" namespace librmb { - -RmbCommands::RmbCommands(librmb::RadosStorage *storage_, librmb::RadosCluster *cluster_, + +RmbCommandsImpl::RmbCommandsImpl(storage_interface::RadosStorage *storage_, storage_interface::RadosCluster *cluster_, std::map *opts_) { this->storage = storage_; this->cluster = cluster_; @@ -35,18 +39,19 @@ RmbCommands::RmbCommands(librmb::RadosStorage *storage_, librmb::RadosCluster *c is_debug = ((*opts).find("debug") != (*opts).end()) ? true : false; } } -RmbCommands::~RmbCommands() {} +RmbCommandsImpl::~RmbCommandsImpl() {} -void RmbCommands::print_debug(const std::string &msg) { +void RmbCommandsImpl::print_debug(const std::string &msg) { if (this->is_debug) { std::cout << msg << std::endl; } } -int RmbCommands::delete_with_save_log(const std::string &save_log, const std::string &rados_cluster, +int RmbCommandsImpl::delete_with_save_log(const std::string &save_log, const std::string &rados_cluster, const std::string &rados_user, - std::map> *moved_items) { + std::map> *moved_items) { librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); + int count = 0; if (moved_items == nullptr) { @@ -56,14 +61,14 @@ int RmbCommands::delete_with_save_log(const std::string &save_log, const std::st /** check content **/ std::ifstream read(save_log); if (!read.is_open()) { - std::cerr << " path to log file not valid " << std::endl; + std::cout << " path to log file not valid " << std::endl; return -1; } int line_count = 0; while (true) { line_count++; - librmb::RadosSaveLogEntry entry; - read >> entry; + librmb::RadosSaveLogEntryImpl *entry = new librmb::RadosSaveLogEntryImpl() ; + read >> (*entry); if (read.eof()) { break; } @@ -72,38 +77,38 @@ int RmbCommands::delete_with_save_log(const std::string &save_log, const std::st break; } - if (storage.get_pool_name().compare(entry.pool) != 0) { + if (storage.get_pool_name().compare(entry->get_pool()) != 0) { // close connection before open a new one. // TODO(jrse): worst case are alternating pool entries e.g. mail_storage , // mail_storage_alt.... maybe we should group the entries by pool... storage.close_connection(); - int open_connection = storage.open_connection(entry.pool, rados_cluster, rados_user); + int open_connection = storage.open_connection(entry->get_pool(), rados_cluster, rados_user); if (open_connection < 0) { - std::cerr << " error opening rados connection. Errorcode: " << open_connection << std::endl; + std::cout << " error opening rados connection. Errorcode: " << open_connection << std::endl; cluster.deinit(); return -1; } } - storage.set_namespace(entry.ns); - if (entry.op.compare("save") == 0 || entry.op.compare("cpy") == 0) { - int ret_delete = storage.delete_mail(entry.oid); + storage.set_namespace(entry->get_ns()); + if (entry->get_op().compare("save") == 0 || entry->get_op().compare("cpy") == 0) { + int ret_delete = storage.delete_mail(entry->get_oid()); if (ret_delete < 0) { - std::cout << "Object " << entry.oid << " not deleted: errorcode: " << ret_delete << std::endl; + std::cout << "Object " << entry->get_oid()<< " not deleted: errorcode: " << ret_delete << std::endl; } else { - std::cout << "Object " << entry.oid << " successfully deleted" << std::endl; + std::cout << "Object " << entry->get_oid()<< " successfully deleted" << std::endl; count++; } } else { - int ret = storage.move(entry.oid, entry.ns.c_str(), entry.src_oid, entry.src_ns.c_str(), entry.metadata, true); + int ret = storage.move(entry->get_oid(), entry->get_ns().c_str(), entry->get_src_oid(), entry->get_src_ns().c_str(), entry->get_metadata(), true); if (ret < 0) { - std::cerr << "moving : " << entry.oid << " to " << entry.src_oid << " failed ! ret code: " << ret << std::endl; + std::cerr << "moving : " << entry->get_oid() << " to " << entry->get_src_oid() << " failed ! ret code: " << ret << "count is::"<find(entry.src_user) == moved_items->end()) { - std::list entries; - entries.push_back(entry); - (*moved_items)[entry.src_user] = entries; + if (moved_items->find(entry->get_src_user()) == moved_items->end()) { + std::list entries; + entries.push_back((storage_interface::RadosSaveLogEntry*)entry); + (*moved_items)[entry->get_src_user()] = entries; } else { - (*moved_items)[entry.src_user].push_back(entry); + (*moved_items)[entry->get_src_user()].push_back((storage_interface::RadosSaveLogEntry*)entry); } } count++; @@ -115,7 +120,7 @@ int RmbCommands::delete_with_save_log(const std::string &save_log, const std::st return count; } -int RmbCommands::lspools() { +int RmbCommandsImpl::lspools() { librmb::RadosClusterImpl cluster; cluster.init(); @@ -134,13 +139,13 @@ int RmbCommands::lspools() { return 0; } -int RmbCommands::delete_namespace(librmb::RadosStorageMetadataModule *ms, std::list &mail_objects, - librmb::RadosCephConfig *cfg, bool confirmed) { +int RmbCommandsImpl::delete_namespace(storage_interface::RadosStorageMetadataModule *ms, std::list &mail_objects, + storage_interface::RadosCephConfig *cfg, bool confirmed) { if (ms == nullptr || cfg == nullptr) { return -1; } - librmb::CmdLineParser parser("-"); + librmb::CmdLineParserImpl parser("-"); if (parser.parse_ls_string()) { std::string sort_type = "uid"; int ret = load_objects(ms, mail_objects, sort_type); @@ -148,7 +153,7 @@ int RmbCommands::delete_namespace(librmb::RadosStorageMetadataModule *ms, std::l return ret; } - for (std::list::iterator it_mail = mail_objects.begin(); it_mail != mail_objects.end(); + for (std::list::iterator it_mail = mail_objects.begin(); it_mail != mail_objects.end(); ++it_mail) { (*opts)["to_delete"] = *(*it_mail)->get_oid(); delete_mail(confirmed); @@ -164,7 +169,7 @@ int RmbCommands::delete_namespace(librmb::RadosStorageMetadataModule *ms, std::l } return 0; } -int RmbCommands::delete_mail(bool confirmed) { +int RmbCommandsImpl::delete_mail(bool confirmed) { int ret = -1; print_debug("entry: delete_mail"); if (!confirmed) { @@ -185,7 +190,7 @@ int RmbCommands::delete_mail(bool confirmed) { return ret; } -int RmbCommands::rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const std::string &uid) { +int RmbCommandsImpl::rename_user(storage_interface::RadosCephConfig *cfg, bool confirmed, const std::string &uid) { print_debug("entry: rename_user"); if (cfg == nullptr) { return -1; @@ -211,7 +216,7 @@ int RmbCommands::rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const print_debug("end: rename_user"); return -1; } - std::list list; + std::list list; std::cout << " copy namespace configuration src " << src_ << " to dest " << dest_ << " in namespace " << cfg->get_user_ns() << std::endl; storage->set_namespace(cfg->get_user_ns()); @@ -246,7 +251,7 @@ int RmbCommands::rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const return ret; } -int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg) { +int RmbCommandsImpl::configuration(bool confirmed, storage_interface::RadosCephConfig *ceph_cfg) { print_debug("entry: configuration"); bool has_update = (*opts).find("update") != (*opts).end(); bool has_ls = (*opts).find("print_cfg") != (*opts).end(); @@ -257,7 +262,7 @@ int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg } if (has_ls) { - std::cout << ceph_cfg.get_config()->to_string() << std::endl; + std::cout << ceph_cfg->get_config()->to_string() << std::endl; print_debug("end: configuration"); return 0; } @@ -288,7 +293,7 @@ int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg std::string key = (*opts)["update"].substr(0, key_val_separator_idx); std::string key_val = (*opts)["update"].substr(key_val_separator_idx + 1, (*opts)["update"].length() - 1); - bool failed = ceph_cfg.update_valid_key_value(key, key_val) ? false : true; + bool failed = ceph_cfg->update_valid_key_value(key, key_val) ? false : true; if (failed) { std::cout << "Error: key : " << key << " value: " << key_val << " is not valid !" << std::endl; if (key_val.compare("TRUE") == 0 || key_val.compare("FALSE") == 0) { @@ -299,7 +304,7 @@ int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg } std::cout << "cfg: key " << key << " cfg_val: " << key_val << std::endl; - if (ceph_cfg.save_cfg() < 0) { + if (ceph_cfg->save_cfg() < 0) { std::cout << " saving cfg failed" << std::endl; print_debug("end: configuration"); @@ -311,72 +316,75 @@ int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg return 0; } -bool RmbCommands::sort_uid(librmb::RadosMail *i, librmb::RadosMail *j) { +bool RmbCommandsImpl::sort_uid(storage_interface::RadosMail *i, storage_interface::RadosMail *j) { std::string::size_type sz; // alias of size_t char *t; if (i == nullptr || j == nullptr) { return false; } - RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, i->get_metadata(), &t); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, i->get_metadata(), &t); try { uint64_t i_uid = std::stol(t, &sz); char *m_mail_uid; - RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, i->get_metadata(), &m_mail_uid); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, i->get_metadata(), &m_mail_uid); uint64_t j_uid = std::stol(m_mail_uid, &sz); return i_uid < j_uid; } catch (std::exception &e) { char *uid; - RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, i->get_metadata(), &uid); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, i->get_metadata(), &uid); std::cerr << " sort_uid: " << t << "(" << *i->get_oid() << ") or " << uid << " (" << j->get_oid() << ") is not a number" << std::endl; return false; } } -bool RmbCommands::sort_recv_date(librmb::RadosMail *i, librmb::RadosMail *j) { +bool RmbCommandsImpl::sort_recv_date(storage_interface::RadosMail *i, storage_interface::RadosMail *j) { std::string::size_type sz; // alias of size_t char *t; if (i == nullptr || j == nullptr) { return false; } - RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, i->get_metadata(), &t); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, i->get_metadata(), &t); try { int64_t i_uid = std::stol(t, &sz); char *m_time; - RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, i->get_metadata(), &m_time); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, i->get_metadata(), &m_time); int64_t j_uid = std::stol(m_time, &sz); return i_uid < j_uid; } catch (std::exception &e) { char *m_recv_time; - RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, i->get_metadata(), &m_recv_time); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, i->get_metadata(), &m_recv_time); std::cerr << " sort_recv_date: " << t << " or " << m_recv_time << " is not a number" << std::endl; return false; } } -bool RmbCommands::sort_phy_size(librmb::RadosMail *i, librmb::RadosMail *j) { +bool RmbCommandsImpl::sort_phy_size(storage_interface::RadosMail *i, storage_interface::RadosMail *j) { std::string::size_type sz; // alias of size_t char *t; if (i == nullptr || j == nullptr) { return false; } - RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, i->get_metadata(), &t); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, i->get_metadata(), &t); try { uint64_t i_uid = std::stol(t, &sz); char *m_phy_size; - RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, i->get_metadata(), &m_phy_size); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, i->get_metadata(), &m_phy_size); uint64_t j_uid = std::stol(m_phy_size, &sz); return i_uid < j_uid; } catch (std::exception &e) { char *m_phy_size; - RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, i->get_metadata(), &m_phy_size); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, i->get_metadata(), &m_phy_size); std::cerr << " sort_physical_size: " << t << " or " << m_phy_size << " is not a number" << std::endl; return false; } } -bool RmbCommands::sort_save_date(librmb::RadosMail *i, librmb::RadosMail *j) { +bool RmbCommandsImpl::sort_save_date(storage_interface::RadosMail *i, storage_interface::RadosMail *j) { if (i == nullptr || j == nullptr) { return false; } @@ -384,16 +392,17 @@ bool RmbCommands::sort_save_date(librmb::RadosMail *i, librmb::RadosMail *j) { } struct AioStat { - librmb::RadosMail *mail; - std::list *mail_objects; + storage_interface::RadosMail *mail; + std::list *mail_objects; uint64_t object_size = 0; time_t save_date_rados; - librmb::RadosStorageMetadataModule *ms; + storage_interface::RadosStorageMetadataModule *ms; bool load_metadata; librados::AioCompletion *completion; }; static void aio_cb(rados_completion_t cb, void *arg) { + librmb::RadosUtilsImpl rados_utils; if (arg == nullptr) { return; } @@ -413,7 +422,7 @@ static void aio_cb(rados_completion_t cb, void *arg) { if (stat->mail->get_metadata()->empty()) { stat->mail->set_valid(false); } - if (!librmb::RadosUtils::validate_metadata(stat->mail->get_metadata())) { + if (!rados_utils.validate_metadata(stat->mail->get_metadata())) { stat->mail->set_valid(false); } } @@ -424,35 +433,36 @@ static void aio_cb(rados_completion_t cb, void *arg) { delete stat; } -int RmbCommands::overwrite_ceph_object_index(std::set &mail_oids){ +int RmbCommandsImpl::overwrite_ceph_object_index(std::set &mail_oids){ return storage->ceph_index_overwrite(mail_oids); } -std::set RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms){ +std::set RmbCommandsImpl::load_objects(storage_interface::RadosStorageMetadataModule *ms){ + librmb::RadosUtilsImpl rados_utils; std::set mail_list; - librados::NObjectIterator iter_guid = storage->find_mails(nullptr); - while (iter_guid != librados::NObjectIterator::__EndObjectIterator) { - librmb::RadosMail mail; - mail.set_oid((*iter_guid).get_oid()); + mail_list = storage->find_mails(nullptr); + std::set::iterator it; + for(it=mail_list.begin();it!=mail_list.end();++it){ + storage_interface::RadosMail *mail=storage->alloc_rados_mail(); + mail->set_oid(*it); - int load_metadata_ret = ms->load_metadata(&mail); - if (load_metadata_ret < 0 || !librmb::RadosUtils::validate_metadata(mail.get_metadata())) { - std::cerr << "metadata for object : " << mail.get_oid()->c_str() << " is not valid, skipping object " << std::endl; - iter_guid++; + int load_metadata_ret = ms->load_metadata(mail); + if (load_metadata_ret < 0 || !rados_utils.validate_metadata(mail->get_metadata())) { + std::cerr << "metadata for object : " << mail->get_oid()->c_str() << " is not valid, skipping object " << std::endl; continue; } - mail_list.insert((*iter_guid).get_oid()); - iter_guid++; + delete mail; + mail=nullptr; } return mail_list; } -int RmbCommands::remove_ceph_object_index(){ +int RmbCommandsImpl::remove_ceph_object_index(){ return storage->ceph_index_delete(); } -int RmbCommands::append_ceph_object_index(const std::set &mail_oids){ +int RmbCommandsImpl::append_ceph_object_index(const std::set &mail_oids){ return storage->ceph_index_append(mail_oids); } -int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms, std::list &mail_objects, +int RmbCommandsImpl::load_objects(storage_interface::RadosStorageMetadataModule *ms, std::list &mail_objects, std::string &sort_string, bool load_metadata) { time_t begin = time(NULL); @@ -464,28 +474,26 @@ int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms, std::list< // TODO(jrse): Fix completions..... std::list completions; // load all objects metadata into memory - librados::NObjectIterator iter(storage->find_mails(nullptr)); - while (iter != librados::NObjectIterator::__EndObjectIterator) { - librmb::RadosMail *mail = new librmb::RadosMail(); + std::set mail_list=storage->find_mails(nullptr); + std::set::iterator mail_iter; + for (mail_iter=mail_list.begin();mail_iter!=mail_list.end();mail_iter++) { + storage_interface::RadosMail *mail = storage->alloc_rados_mail(); AioStat *stat = new AioStat(); stat->mail = mail; stat->mail_objects = &mail_objects; stat->load_metadata = load_metadata; stat->ms = ms; - std::string oid = iter->get_oid(); + std::string oid = *mail_iter; stat->completion = librados::Rados::aio_create_completion(static_cast(stat), aio_cb, NULL); - int ret = storage->get_io_ctx().aio_stat(oid, stat->completion, &stat->object_size, &stat->save_date_rados); + int ret = storage->get_io_ctx_wrapper()->aio_stat(oid, stat->completion, &stat->object_size, &stat->save_date_rados); if (ret != 0) { std::cout << " object '" << oid << "' is not a valid mail object, size = 0, ret code: " << ret << std::endl; - ++iter; delete mail; delete stat; continue; } mail->set_oid(oid); completions.push_back(stat->completion); - - ++iter; if (is_debug) { std::cout << "added: mail " << *mail->get_oid() << std::endl; } @@ -495,7 +503,8 @@ int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms, std::list< (*it)->wait_for_complete_and_cb(); (*it)->release(); } - + + if (load_metadata) { if (sort_string.compare("uid") == 0) { mail_objects.sort(sort_uid); @@ -518,28 +527,29 @@ int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms, std::list< return 0; } -int RmbCommands::print_mail(std::map *mailbox, std::string &output_dir, +int RmbCommandsImpl::print_mail(std::map *mailbox, std::string &output_dir, bool download) { print_debug("entry:: print_mail"); - for (std::map::iterator it = mailbox->begin(); it != mailbox->end(); ++it) { + for (std::map::iterator it = mailbox->begin(); it != mailbox->end(); ++it) { if (it->second->get_mail_count() == 0) { continue; } - std::cout << it->second->to_string() << std::endl; + librmb::RadosMailBoxImpl it_impl=*(dynamic_cast(it->second)); + std::cout << it_impl.to_string() << std::endl; if (!download) { continue; } - librmb::MailboxTools tools(it->second, output_dir); + + librmb::MailboxToolsImpl tools(it->second, output_dir); if (tools.init_mailbox_dir() < 0) { std::cout << " error initializing output dir : " << output_dir << std::endl; break; } - for (std::list::iterator it_mail = it->second->get_mails().begin(); + for (std::list::iterator it_mail = it->second->get_mails().begin(); it_mail != it->second->get_mails().end(); ++it_mail) { const std::string oid = *(*it_mail)->get_oid(); - librados::bufferlist bl; - (*it_mail)->set_mail_buffer(&bl); - if (storage->read_mail(oid, (*it_mail)->get_mail_buffer()) > 0) { + (*it_mail)->set_mail_buffer(storage->alloc_mail_buffer()); + if (storage->read_mail(oid,*it_mail,0) > 0) { if (tools.save_mail((*it_mail)) < 0) { std::cout << " error saving mail : " << oid << " to " << tools.get_mailbox_path() << std::endl; } @@ -550,18 +560,18 @@ int RmbCommands::print_mail(std::map *mailb return 0; } -int RmbCommands::query_mail_storage(std::list *mail_objects, librmb::CmdLineParser *parser, +int RmbCommandsImpl::query_mail_storage(std::list *mail_objects, storage_interface::CmdLineParser *parser, bool download, bool silent) { print_debug("entry: query_mail_storage"); - - std::map mailbox; - for (std::list::iterator it = mail_objects->begin(); it != mail_objects->end(); ++it) { - std::string mailbox_key = std::string(1, static_cast(librmb::RBOX_METADATA_MAILBOX_GUID)); + librmb::RadosUtilsImpl rados_utils; + std::map mailbox; + for (std::list::iterator it = mail_objects->begin(); it != mail_objects->end(); ++it) { + std::string mailbox_key = std::string(1, static_cast(storage_interface::RBOX_METADATA_MAILBOX_GUID)); char *mailbox_guid = NULL; - RadosUtils::get_metadata(mailbox_key, (*it)->get_metadata(), &mailbox_guid); - std::string mailbox_orig_name_key = std::string(1, static_cast(librmb::RBOX_METADATA_ORIG_MAILBOX)); + rados_utils.get_metadata(mailbox_key, (*it)->get_metadata(), &mailbox_guid); + std::string mailbox_orig_name_key = std::string(1, static_cast(storage_interface::RBOX_METADATA_ORIG_MAILBOX)); char *mailbox_orig_name = NULL; - RadosUtils::get_metadata(mailbox_orig_name_key, (*it)->get_metadata(), &mailbox_orig_name); + rados_utils.get_metadata(mailbox_orig_name_key, (*it)->get_metadata(), &mailbox_orig_name); if (mailbox_guid == NULL || mailbox_orig_name == NULL) { std::cout << " mail " << *(*it)->get_oid() << " with empty mailbox guid is not valid: " << std::endl; @@ -569,7 +579,7 @@ int RmbCommands::query_mail_storage(std::list *mail_objects } if (parser->contains_key(mailbox_key)) { - librmb::Predicate *p = parser->get_predicate(mailbox_key); + storage_interface::Predicate *p = parser->get_predicate(mailbox_key); if (!p->eval(mailbox_guid)) { continue; } @@ -578,7 +588,7 @@ int RmbCommands::query_mail_storage(std::list *mail_objects mailbox[mailbox_guid]->add_mail((*it)); mailbox[mailbox_guid]->add_to_mailbox_size((*it)->get_mail_size()); } else { - mailbox[mailbox_guid] = new librmb::RadosMailBox(mailbox_guid, 1, mailbox_orig_name); + mailbox[mailbox_guid] = new librmb::RadosMailBoxImpl(mailbox_guid, 1, mailbox_orig_name); mailbox[mailbox_guid]->set_xattr_filter(parser); mailbox[mailbox_guid]->add_mail((*it)); mailbox[mailbox_guid]->add_to_mailbox_size((*it)->get_mail_size()); @@ -596,15 +606,15 @@ int RmbCommands::query_mail_storage(std::list *mail_objects return ret; } -RadosStorageMetadataModule *RmbCommands::init_metadata_storage_module(librmb::RadosCephConfig &ceph_cfg, +storage_interface::RadosStorageMetadataModule *RmbCommandsImpl::init_metadata_storage_module(storage_interface::RadosCephConfig *ceph_cfg, std::string *uid) { print_debug("entry: init_metadata_storage_module"); librmb::RadosConfig dovecot_cfg; - RadosStorageMetadataModule *ms = nullptr; + storage_interface::RadosStorageMetadataModule *ms = nullptr; dovecot_cfg.set_config_valid(true); - ceph_cfg.set_config_valid(true); - librmb::RadosDovecotCephCfgImpl cfg(dovecot_cfg, ceph_cfg); - librmb::RadosNamespaceManager mgr(&cfg); + ceph_cfg->set_config_valid(true); + storage_interface::RadosDovecotCephCfg *cfg=new librmb::RadosDovecotCephCfgImpl(dovecot_cfg, ceph_cfg); + librmb::RadosNamespaceManagerImpl mgr(cfg); if (uid == nullptr) { std::cerr << "please set valid uid ptr" << std::endl; @@ -613,14 +623,14 @@ RadosStorageMetadataModule *RmbCommands::init_metadata_storage_module(librmb::Ra } // decide metadata storage! - std::string storage_module_name = ceph_cfg.get_metadata_storage_module(); + std::string storage_module_name = ceph_cfg->get_metadata_storage_module(); if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) { - ms = new librmb::RadosMetadataStorageIma(&storage->get_io_ctx(), &cfg); + ms = new librmb::RadosMetadataStorageIma(storage->get_io_ctx_wrapper(), cfg); } else { - ms = new librmb::RadosMetadataStorageDefault(&storage->get_io_ctx()); + ms = new librmb::RadosMetadataStorageDefault(storage->get_io_ctx_wrapper()); } if (!(*opts)["namespace"].empty()) { - *uid = (*opts)["namespace"] + cfg.get_user_suffix(); + *uid = (*opts)["namespace"] + cfg->get_user_suffix(); } std::string ns; if (mgr.lookup_key(*uid, &ns)) { @@ -638,28 +648,30 @@ RadosStorageMetadataModule *RmbCommands::init_metadata_storage_module(librmb::Ra print_debug("end: init_metadata_storage_module"); return ms; } -int RmbCommands::update_attributes(librmb::RadosStorageMetadataModule *ms, +int RmbCommandsImpl::update_attributes(storage_interface::RadosStorageMetadataModule *ms, std::map *metadata) { std::string oid = (*opts)["set"]; + librmb::RadosUtilsImpl rados_utils; if (!oid.empty() && metadata->size() > 0) { for (std::map::iterator it = metadata->begin(); it != metadata->end(); ++it) { std::cout << oid << "=> " << it->first << " = " << it->second << '\n'; - librmb::rbox_metadata_key ke = static_cast(it->first[0]); + storage_interface::rbox_metadata_key ke = static_cast(it->first[0]); std::string value = it->second; - if (librmb::RadosUtils::is_date_attribute(ke)) { - if (!librmb::RadosUtils::is_numeric(value.c_str())) { + if (rados_utils.is_date_attribute(ke)) { + if (!rados_utils.is_numeric(value.c_str())) { std::string date; - if (librmb::RadosUtils::convert_string_to_date(value, &date)) { + if (rados_utils.convert_string_to_date(value, &date)) { value = date; } } } - librmb::RadosMail obj; - obj.set_oid(oid); - ms->load_metadata(&obj); - librmb::RadosMetadata attr(ke, value); - ms->set_metadata(&obj, attr); + storage_interface::RadosMail *obj=storage->alloc_rados_mail(); + obj->set_oid(oid); + ms->load_metadata(obj); + ms->set_metadata(obj); std::cout << " saving object ..." << std::endl; + delete obj; + obj=nullptr; } } else { std::cerr << " invalid number of arguments, check usage " << std::endl; @@ -667,7 +679,7 @@ int RmbCommands::update_attributes(librmb::RadosStorageMetadataModule *ms, } return 0; } -void RmbCommands::set_output_path(librmb::CmdLineParser *parser) { +void RmbCommandsImpl::set_output_path(storage_interface::CmdLineParser *parser) { if ((*opts).find("out") != (*opts).end()) { parser->set_output_dir((*opts)["out"]); } else { diff --git a/src/librmb/tools/rmb/rmb-commands-impl.h b/src/librmb/tools/rmb/rmb-commands-impl.h new file mode 100644 index 00000000..1a509bd0 --- /dev/null +++ b/src/librmb/tools/rmb/rmb-commands-impl.h @@ -0,0 +1,85 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_LIBRMB_TOOLS_RMB_RMB_COMMANDS_IMPL_H_ +#define SRC_LIBRMB_TOOLS_RMB_RMB_COMMANDS_IMPL_H_ +#include + +#include +#include +#include +#include +#include +#include + +#include "../../../storage-interface/rados-storage.h" +#include "../../../storage-interface/rados-cluster.h" +#include "../../../storage-interface/rados-metadata-storage.h" +#include "../../../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../../../storage-interface/rados-ceph-config.h" +#include "../../../storage-interface/tools/rmb/ls_cmd_parser.h" +#include "../../../storage-interface/tools/rmb/mailbox_tools.h" +#include "../../../storage-interface/rados-metadata-storage-module.h" +#include "../../../storage-interface/tools/rmb/rmb-commands.h" +#include "../../../storage-interface/rados-save-log.h" +using storage_interface::RmbCommands; +namespace librmb { + +class RmbCommandsImpl : public storage_interface::RmbCommands { + public: + RmbCommandsImpl() {} + RmbCommandsImpl(storage_interface::RadosStorage *storage_, storage_interface::RadosCluster *cluster_, + std::map *opts_); + virtual ~RmbCommandsImpl(); + + int delete_with_save_log(const std::string &save_log, const std::string &rados_cluster, + const std::string &rados_user, + std::map> *moved_items) override; + void print_debug(const std::string &msg) override; + int lspools() override; + int delete_mail(bool confirmed) override; + int delete_namespace(storage_interface::RadosStorageMetadataModule *ms, std::list &mail_objects, + storage_interface::RadosCephConfig *cfg, bool confirmed) override; + + int rename_user(storage_interface::RadosCephConfig *cfg, bool confirmed, const std::string &uid) override; + int configuration(bool confirmed, storage_interface::RadosCephConfig *ceph_cfg) override; + int load_objects(storage_interface::RadosStorageMetadataModule *ms, std::list &mail_objects, + std::string &sort_string, bool load_metadata = true)override ; + + int update_attributes(storage_interface::RadosStorageMetadataModule *ms, std::map *metadata) override; + int print_mail(std::map *mailbox, std::string &output_dir, bool download) override; + int query_mail_storage(std::list *mail_objects, storage_interface::CmdLineParser *parser, bool download, + bool silent) override; + + storage_interface::RadosStorageMetadataModule *init_metadata_storage_module(storage_interface::RadosCephConfig *ceph_cfg, std::string *uid) override; + + static bool sort_uid(storage_interface::RadosMail *i, storage_interface::RadosMail *j); + static bool sort_recv_date(storage_interface::RadosMail *i, storage_interface::RadosMail *j); + static bool sort_phy_size(storage_interface::RadosMail *i, storage_interface::RadosMail *j); + static bool sort_save_date(storage_interface::RadosMail *i, storage_interface::RadosMail *j); + + void set_output_path(storage_interface::CmdLineParser *parser) override; + + int overwrite_ceph_object_index(std::set &mail_oids) override; + std::set load_objects(storage_interface::RadosStorageMetadataModule *ms) override; + int remove_ceph_object_index() override; + int append_ceph_object_index(const std::set &mail_oids) override; + + private: + std::map *opts; + storage_interface::RadosStorage *storage; + storage_interface::RadosCluster *cluster; + bool is_debug; +}; + +} /* namespace librmb */ + +#endif /* SRC_LIBRMB_TOOLS_RMB_RMB_COMMANDS_IMPL_H_ */ diff --git a/src/librmb/tools/rmb/rmb-commands.h b/src/librmb/tools/rmb/rmb-commands.h deleted file mode 100644 index d33c9b2f..00000000 --- a/src/librmb/tools/rmb/rmb-commands.h +++ /dev/null @@ -1,81 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Copyright (c) 2017-2018 Tallence AG and the authors - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - */ - -#ifndef SRC_LIBRMB_TOOLS_RMB_RMB_COMMANDS_H_ -#define SRC_LIBRMB_TOOLS_RMB_RMB_COMMANDS_H_ -#include - -#include -#include -#include -#include -#include -#include - -#include "rados-storage.h" -#include "rados-cluster.h" -#include "rados-metadata-storage.h" -#include "rados-dovecot-ceph-cfg.h" -#include "rados-ceph-config.h" -#include "ls_cmd_parser.h" -#include "mailbox_tools.h" -#include "rados-metadata-storage-module.h" -#include "rados-save-log.h" - -namespace librmb { - -class RmbCommands { - public: - RmbCommands(librmb::RadosStorage *storage_, librmb::RadosCluster *cluster_, - std::map *opts_); - virtual ~RmbCommands(); - - static int delete_with_save_log(const std::string &save_log, const std::string &rados_cluster, - const std::string &rados_user, - std::map> *moved_items); - void print_debug(const std::string &msg); - static int lspools(); - int delete_mail(bool confirmed); - int delete_namespace(librmb::RadosStorageMetadataModule *ms, std::list &mail_objects, - librmb::RadosCephConfig *cfg, bool confirmed); - - int rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const std::string &uid); - - int configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg); - - int load_objects(librmb::RadosStorageMetadataModule *ms, std::list &mail_objects, - std::string &sort_string, bool load_metadata = true); - int update_attributes(librmb::RadosStorageMetadataModule *ms, std::map *metadata); - int print_mail(std::map *mailbox, std::string &output_dir, bool download); - int query_mail_storage(std::list *mail_objects, librmb::CmdLineParser *parser, bool download, - bool silent); - librmb::RadosStorageMetadataModule *init_metadata_storage_module(librmb::RadosCephConfig &ceph_cfg, std::string *uid); - static bool sort_uid(librmb::RadosMail *i, librmb::RadosMail *j); - static bool sort_recv_date(librmb::RadosMail *i, librmb::RadosMail *j); - static bool sort_phy_size(librmb::RadosMail *i, librmb::RadosMail *j); - static bool sort_save_date(librmb::RadosMail *i, librmb::RadosMail *j); - - void set_output_path(librmb::CmdLineParser *parser); - - int overwrite_ceph_object_index(std::set &mail_oids); - std::set load_objects(librmb::RadosStorageMetadataModule *ms); - int remove_ceph_object_index(); - int append_ceph_object_index(const std::set &mail_oids); - private: - std::map *opts; - librmb::RadosStorage *storage; - librmb::RadosCluster *cluster; - bool is_debug; -}; - -} /* namespace librmb */ - -#endif /* SRC_LIBRMB_TOOLS_RMB_RMB_COMMANDS_H_ */ diff --git a/src/librmb/tools/rmb/rmb.cpp b/src/librmb/tools/rmb/rmb.cpp index be99368b..7a931eb0 100644 --- a/src/librmb/tools/rmb/rmb.cpp +++ b/src/librmb/tools/rmb/rmb.cpp @@ -21,21 +21,27 @@ #include -#include "../../rados-cluster.h" +#include "../../../storage-interface/rados-cluster.h" #include "../../rados-cluster-impl.h" -#include "../../rados-mail.h" -#include "../../rados-storage.h" +#include "../../../storage-interface/rados-mail.h" +#include "../../../storage-interface/rados-storage.h" #include "../../rados-storage-impl.h" #include "../../rados-metadata-storage-ima.h" -#include "../../rados-metadata-storage-module.h" -#include "ls_cmd_parser.h" -#include "mailbox_tools.h" -#include "rados-util.h" -#include "rados-namespace-manager.h" -#include "rados-dovecot-ceph-cfg.h" -#include "rados-dovecot-ceph-cfg-impl.h" -#include "rados-metadata-storage-default.h" -#include "rmb-commands.h" +#include "../../../storage-interface/rados-metadata-storage-module.h" +#include "../../../storage-interface/tools/rmb/ls_cmd_parser.h" +#include "../../tools/rmb/ls_cmd_parser_impl.h" +#include "../../../storage-interface/tools/rmb/mailbox_tools.h" +#include "../../../storage-interface/rados-util.h" +#include "../../rados-namespace-manager-impl.h" +#include "../../../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../../rados-dovecot-ceph-cfg-impl.h" +#include "../../rados-metadata-storage-default.h" +#include "../../../storage-interface/tools/rmb/rmb-commands.h" +#include "../../tools/rmb/rmb-commands-impl.h" +#include "../../../storage-interface/rados-save-log.h" +#include "../../../storage-interface/rados-ceph-config.h" +#include "../../rados-ceph-config-impl.h" + #undef PACKAGE_BUGREPORT #undef PACKAGE_NAME #undef PACKAGE_STRING @@ -201,7 +207,7 @@ __attribute__((noreturn)) static void usage_exit() { exit(1); } -static void release_exit(std::list *mail_objects, librmb::RadosCluster *cluster, bool show_usage) { +static void release_exit(std::list *mail_objects, storage_interface::RadosCluster *cluster, bool show_usage) { if (mail_objects != nullptr) { for (auto mo : *mail_objects) { delete mo; @@ -282,13 +288,14 @@ static void parse_cmd_line_args(std::map *opts, bool & } int main(int argc, const char **argv) { - std::list mail_objects; + std::list mail_objects; std::vector args; std::map opts; std::map metadata; std::string sort_type; - librmb::RmbCommands *rmb_commands = nullptr; + storage_interface::RmbCommands *rmb_commands = nullptr; + storage_interface::RmbCommands *temp_rmb_commands= new librmb::RmbCommandsImpl(); bool is_config_option = false; bool create_config = false; @@ -323,8 +330,14 @@ int main(int argc, const char **argv) { if (!remove_save_log.empty()) { if (confirmed) { - std::map> moved_items; - return librmb::RmbCommands::delete_with_save_log(remove_save_log, rados_cluster, rados_user, &moved_items); + std::map> moved_items; + int ret= temp_rmb_commands->delete_with_save_log(remove_save_log, rados_cluster, rados_user, &moved_items); + std::map>::iterator it=moved_items.begin(); + for(std::list::iterator iter = it->second.begin();iter != it->second.end(); ++iter){ + delete *iter; + *iter=nullptr; + } + return ret; } else { std::cout << "WARNING:" << std::endl; std::cout << "Performing this command, will delete all mail objects from ceph object store which are " @@ -338,9 +351,10 @@ int main(int argc, const char **argv) { std::string pool_name(opts.find("pool") == opts.end() ? "mail_storage" : opts["pool"]); if (is_lspools_cmd) { - return librmb::RmbCommands::lspools(); + return temp_rmb_commands->lspools(); } - + delete temp_rmb_commands; + temp_rmb_commands=nullptr; librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); int open_connection = storage.open_connection(pool_name, rados_cluster, rados_user); @@ -351,15 +365,15 @@ int main(int argc, const char **argv) { } // initialize configuration - librmb::RadosCephConfig ceph_cfg(&storage.get_io_ctx()); + storage_interface::RadosCephConfig *ceph_cfg=new librmb::RadosCephConfigImpl(storage.get_io_ctx_wrapper()); // set config object - config_obj = opts.find("cfg_obj") != opts.end() ? opts["cfg_obj"] : ceph_cfg.get_cfg_object_name(); - ceph_cfg.set_cfg_object_name(config_obj); + config_obj = opts.find("cfg_obj") != opts.end() ? opts["cfg_obj"] : ceph_cfg->get_cfg_object_name(); + ceph_cfg->set_cfg_object_name(config_obj); - int ret_load_cfg = ceph_cfg.load_cfg(); + int ret_load_cfg = ceph_cfg->load_cfg(); if (ret_load_cfg < 0) { if (create_config) { - if (ceph_cfg.save_cfg() < 0) { + if (ceph_cfg->save_cfg() < 0) { std::cerr << "loading config object failed " << std::endl; } else { std::cout << "config created" << std::endl; @@ -375,12 +389,13 @@ int main(int argc, const char **argv) { } // connection to rados is established! - rmb_commands = new librmb::RmbCommands(&storage, &cluster, &opts); + rmb_commands =new librmb::RmbCommandsImpl(&storage, &cluster, &opts); if (is_config_option) { if (rmb_commands->configuration(confirmed, ceph_cfg) < 0) { std::cerr << "error processing config option" << std::endl; } delete rmb_commands; + delete ceph_cfg; // tear down. release_exit(nullptr, &cluster, false); exit(0); @@ -393,18 +408,19 @@ int main(int argc, const char **argv) { std::string uid; // load metadata configuration - librmb::RadosStorageMetadataModule *ms = rmb_commands->init_metadata_storage_module(ceph_cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_commands->init_metadata_storage_module(ceph_cfg, &uid); if (ms == nullptr) { /// error exit! std::cerr << " Error initializing metadata module " << std::endl; delete rmb_commands; + delete ceph_cfg; release_exit(&mail_objects, &cluster, false); exit(0); } if (delete_mail_option) { if (opts["to_delete"].size() == 1 && opts["to_delete"].compare("-") == 0) { - if (rmb_commands->delete_namespace(ms, mail_objects, &ceph_cfg, confirmed) < 0) { + if (rmb_commands->delete_namespace(ms, mail_objects, ceph_cfg, confirmed) < 0) { std::cerr << "error deleting namespace " << std::endl; release_exit(&mail_objects, &cluster, false); } @@ -416,14 +432,15 @@ int main(int argc, const char **argv) { release_exit(&mail_objects, &cluster, false); delete rmb_commands; delete ms; + delete ceph_cfg; exit(0); } else if (rename_user_option) { - if (rmb_commands->rename_user(&ceph_cfg, confirmed, uid) < 0) { + if (rmb_commands->rename_user(ceph_cfg, confirmed, uid) < 0) { std::cerr << "error renaming user" << std::endl; } } else if (opts.find("ls") != opts.end()) { - librmb::CmdLineParser parser(opts["ls"]); + librmb::CmdLineParserImpl parser(opts["ls"]); if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { rmb_commands->load_objects(ms, mail_objects, sort_type); rmb_commands->query_mail_storage(&mail_objects, &parser, false, false); @@ -432,7 +449,7 @@ int main(int argc, const char **argv) { << std::endl; } } else if (opts.find("get") != opts.end()) { - librmb::CmdLineParser parser(opts["get"]); + librmb::CmdLineParserImpl parser(opts["get"]); rmb_commands->set_output_path(&parser); @@ -447,6 +464,7 @@ int main(int argc, const char **argv) { delete rmb_commands; delete ms; + delete ceph_cfg; // tear down. release_exit(&mail_objects, &cluster, false); diff --git a/src/storage-engine/storage-backend-factory.h b/src/storage-engine/storage-backend-factory.h new file mode 100644 index 00000000..4fc7cd85 --- /dev/null +++ b/src/storage-engine/storage-backend-factory.h @@ -0,0 +1,276 @@ +#ifndef SRC_STORAGE_ENGINE_STORAGE_BACKEND_FACTORY_H_ +#define SRC_STORAGE_ENGINE_STORAGE_BACKEND_FACTORY_H_ +#include +#include + +#include "../storage-interface/rados-cluster.h" +#include "../librmb/rados-cluster-impl.h" +#include "../storage-interface/rados-storage.h" +#include "../librmb/rados-storage-impl.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../librmb/rados-dovecot-ceph-cfg-impl.h" +#include "../storage-interface/rados-namespace-manager.h" +#include "../librmb/rados-namespace-manager-impl.h" +#include "../storage-interface/rados-metadata-storage.h" +#include "../librmb/rados-metadata-storage-impl.h" +#include "../storage-interface/rados-mail.h" +#include "../librmb/rados-mail-impl.h" +#include "../storage-interface/rados-ceph-config.h" +#include "../librmb/rados-ceph-config-impl.h" +#include "../storage-interface/rados-metadata.h" +#include "../librmb/rados-metadata-impl.h" +#include "../storage-interface/rados-types.h" +#include "../librmb/rados-dovecot-config.h" +#include "../storage-interface/tools/rmb/rmb-commands.h" +#include "../librmb/tools/rmb/rmb-commands-impl.h" +#include "../storage-interface/rados-save-log.h" +#include "../librmb/rados-save-log-impl.h" +#include "../storage-interface/tools/rmb/ls_cmd_parser.h" +#include "../librmb/tools/rmb/ls_cmd_parser_impl.h" +#include "../storage-interface/rados-util.h" +#include "../librmb/rados-util-impl.h" +#include "../storage-interface/rados-plugin-logger.h" + +namespace storage_engine { +enum StorageType { CEPH, S3 }; +class StorageBackendFactory { + public: + + static void init_logger(storage_interface::RadosDovecotCephCfg *dovecot_ceph_cfg) { + librmb::RadosPluginLogger::getInstance().init(dovecot_ceph_cfg->get_log_file_path(), + dovecot_ceph_cfg->get_log_level()); + } + + static storage_interface::RadosCluster *create_cluster(StorageType storage_type) { + storage_interface::RadosCluster *cluster = nullptr; + if (storage_type == CEPH) { + cluster = new librmb::RadosClusterImpl(); + } + return cluster; + } + + static storage_interface::RadosStorage *create_storage(StorageType storage_type, storage_interface::RadosCluster *cluster) { + storage_interface::RadosStorage *storage = nullptr; + if (storage_type == CEPH) { + storage = new librmb::RadosStorageImpl(cluster); + } + return storage; + } + + static storage_interface::RadosDovecotCephCfg *create_dovecot_ceph_cfg_io(StorageType storage_type,storage_interface::RboxIoCtx *io_ctx_) { + storage_interface::RadosDovecotCephCfg *dovecot_ceph_cfg = nullptr; + if (storage_type == CEPH) { + dovecot_ceph_cfg = new librmb::RadosDovecotCephCfgImpl(io_ctx_); + } + return dovecot_ceph_cfg; + } + + static storage_interface::RadosDovecotCephCfg *create_dovecot_ceph_cfg( + StorageType storage_type, librmb::RadosConfig &dovecot_cfg_, storage_interface::RadosCephConfig *rados_cfg_){ + storage_interface::RadosDovecotCephCfg *dovecot_ceph_cfg = nullptr; + if(storage_type == CEPH){ + dovecot_ceph_cfg=new librmb::RadosDovecotCephCfgImpl(dovecot_cfg_, rados_cfg_); + } + return dovecot_ceph_cfg; + } + + static storage_interface::RadosNamespaceManager *create_namespace_manager(StorageType storage_type, + storage_interface::RadosDovecotCephCfg *dovecot_ceph_cfg) { + storage_interface::RadosNamespaceManager *name_space_manager = nullptr; + if (storage_type == CEPH) { + name_space_manager = new librmb::RadosNamespaceManagerImpl(dovecot_ceph_cfg); + } + return name_space_manager; + } + + static storage_interface::RadosMetadataStorage *create_metadata_storage( + StorageType storage_type,storage_interface::RboxIoCtx *io_ctx,storage_interface::RadosDovecotCephCfg * config) { + storage_interface::RadosMetadataStorage *metadata_storage = nullptr; + if (storage_type == CEPH) { + metadata_storage = new librmb::RadosMetadataStorageImpl(io_ctx,config); + } + return metadata_storage; + } + + static storage_interface::RadosMail *create_mail(StorageType storage_type){ + storage_interface::RadosMail *mail = nullptr; + if (storage_type == CEPH){ + mail=new librmb::RadosMailImpl(); + } + return mail; + } + + static storage_interface::RadosCephConfig *create_ceph_config(StorageType storage_type){ + storage_interface::RadosCephConfig *ceph_config = nullptr; + if(storage_type==CEPH){ + ceph_config=new librmb::RadosCephConfigImpl(); + } + return ceph_config; + } + + static storage_interface::RadosCephConfig *create_ceph_config_io(StorageType storage_type,storage_interface::RboxIoCtx *io_ctx_){ + storage_interface::RadosCephConfig *ceph_config = nullptr; + if(storage_type==CEPH){ + ceph_config=new librmb::RadosCephConfigImpl(io_ctx_); + } + return ceph_config; + } + + static storage_interface::RadosMetadata *create_metadata_default(StorageType storage_type){ + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_str_key_val( + StorageType storage_type,std::string& key_, std::string& val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(key_,val); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_string( + StorageType storage_type,storage_interface::rbox_metadata_key _key,const std::string& val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(_key,val); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_time( + StorageType storage_type,storage_interface::rbox_metadata_key _key,const time_t& val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(_key,val); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_char( + StorageType storage_type,storage_interface::rbox_metadata_key _key,const char* val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(_key,val); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_uint( + StorageType storage_type,storage_interface::rbox_metadata_key _key,const uint& val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(_key,val); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_size_t( + StorageType storage_type,storage_interface::rbox_metadata_key _key,const size_t& val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(_key,val); + } + return metadata; + } + + static storage_interface::RadosMetadata *create_metadata_int( + StorageType storage_type,storage_interface::rbox_metadata_key _key,const int val){ + + storage_interface::RadosMetadata *metadata = nullptr; + if(storage_type==CEPH){ + metadata=new librmb::RadosMetadataImpl(_key,val); + } + return metadata; + } + static storage_interface::RmbCommands *create_rmb_commands( + StorageType storage_type, storage_interface::RadosStorage *storage_, storage_interface::RadosCluster *cluster_, + std::map *opts_){ + + storage_interface::RmbCommands *rmb_commands = nullptr; + if(storage_type==CEPH){ + rmb_commands=new librmb::RmbCommandsImpl(storage_, cluster_, opts_); + } + return rmb_commands; + } + static storage_interface::RmbCommands *create_rmb_commands_default(StorageType storage_type){ + storage_interface::RmbCommands *rmb_commands = nullptr; + if(storage_type==CEPH){ + rmb_commands=new librmb::RmbCommandsImpl(); + } + return rmb_commands; + } + + static storage_interface::RadosSaveLogEntry *create_save_log_entry( + StorageType storage_type,const std::string &oid_, const std::string &ns_, const std::string &pool_, const std::string &op_){ + + storage_interface::RadosSaveLogEntry *save_log_entry = nullptr; + if(storage_type==CEPH){ + save_log_entry=new librmb::RadosSaveLogEntryImpl(oid_, ns_, pool_, op_); + } + return save_log_entry; + } + + static storage_interface::RadosSaveLogEntry *create_save_log_entry_default(StorageType storage_type){ + storage_interface::RadosSaveLogEntry *save_log_entry = nullptr; + if(storage_type==CEPH){ + save_log_entry=new librmb::RadosSaveLogEntryImpl(); + } + return save_log_entry; + } + + static storage_interface::RadosSaveLog *create_save_log(StorageType storage_type,const std::string& log_file){ + storage_interface::RadosSaveLog *save_log = nullptr; + if(storage_type==CEPH){ + save_log=new librmb::RadosSaveLogImpl(log_file); + } + return save_log; + } + static storage_interface::RadosSaveLog *create_save_log_default(StorageType storage_type){ + storage_interface::RadosSaveLog *save_log = nullptr; + if(storage_type==CEPH){ + save_log=new librmb::RadosSaveLogImpl(); + } + return save_log; + } + + static storage_interface::CmdLineParser *create_cmd_line_parser(StorageType storage_type,const std::string &_ls_value){ + storage_interface::CmdLineParser *cmd_line_parser = nullptr; + if(storage_type==CEPH){ + cmd_line_parser=new librmb::CmdLineParserImpl(_ls_value); + } + return cmd_line_parser; + } + + static storage_interface::RadosUtils *create_rados_utils(StorageType storage_type){ + storage_interface::RadosUtils *rados_utils = nullptr; + if(storage_type==CEPH){ + rados_utils=new librmb::RadosUtilsImpl(); + } + return rados_utils; + } +}; +class StorageObjectFactory{ + public: + template + static BufferType* get_buffer(StorageType storage_type, void* storage_address){ + BufferType* storage_buffer = nullptr; + if(storage_type==CEPH){ + storage_buffer=(ceph::bufferlist*)storage_address; + } + return storage_buffer; + } + +}; +} // namespace interface_engine +#endif // SRC_STORAGE_ENGINE_STORAGE_BACKEND_FACTORY_H_ \ No newline at end of file diff --git a/src/storage-interface/Makefile.am b/src/storage-interface/Makefile.am new file mode 100644 index 00000000..3c1041db --- /dev/null +++ b/src/storage-interface/Makefile.am @@ -0,0 +1,37 @@ +# +# Copyright (c) 2017-2018 Tallence AG and the authors +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. + +SUBDIRS = . tools + +LIBINTERFACE = libinterface.la + +libinterface_ladir = $(moduledir) + +libinterface_la_LTLIBRARIES = \ + $(LIBINTERFACE) + +libinterface_la_SOURCES = \ + rados-ceph-config.h \ + rados-ceph-json-config.h \ + rados-cluster.h \ + rados-dictionary.h \ + rados-dovecot-ceph-cfg.h \ + rados-guid-generator.h \ + rados-mail.h \ + rados-metadata-storage-module.h \ + rados-metadata-storage.h \ + rados-metadata.h \ + rados-namespace-manager.h \ + rados-save-log.h \ + rados-storage.h \ + rados-types.h \ + rados-util.h \ + rbox-io-ctx.h \ + rados-plugin-logger.h + + diff --git a/src/storage-interface/rados-ceph-config.h b/src/storage-interface/rados-ceph-config.h new file mode 100644 index 00000000..2d8c7f57 --- /dev/null +++ b/src/storage-interface/rados-ceph-config.h @@ -0,0 +1,72 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ +#ifndef SRC_STORAGE_INTERFACES__RADOS_CEPH_CONFIG_H_ +#define SRC_STORAGE_INTERFACES__RADOS_CEPH_CONFIG_H_ + +#include +#include "rados-ceph-json-config.h" +#include "rados-types.h" +#include "rados-storage.h" + +namespace storage_interface { +/** + * RadosCephConfig + * + * read Plugin configuration from rados storage + */ +class RadosCephConfig { + public: + virtual ~RadosCephConfig() {} + + // load settings from rados cfg_object + virtual int load_cfg() = 0; + virtual int save_cfg() = 0; + + virtual void set_io_ctx(storage_interface::RboxIoCtx *io_ctx_) = 0; + virtual bool is_config_valid() = 0; + virtual void set_config_valid(bool valid_) = 0; + virtual bool is_user_mapping() = 0; + virtual void set_user_mapping(bool value_) = 0; + virtual void set_user_ns(const std::string &ns_) = 0; + virtual std::string &get_user_ns() = 0; + virtual void set_user_suffix(const std::string &ns_suffix_) = 0; + virtual std::string &get_user_suffix() = 0; + virtual const std::string &get_public_namespace() const = 0; + virtual void set_public_namespace(const std::string &public_namespace_) = 0; + virtual void set_cfg_object_name(const std::string &cfg_object_name_) = 0; + virtual std::string get_cfg_object_name() = 0; + virtual storage_interface::RadosCephJsonConfig *get_config() = 0; + + virtual bool is_valid_key_value(const std::string &key, const std::string &value) = 0; + virtual bool update_valid_key_value(const std::string &key, const std::string &value) = 0; + // virtual bool is_ceph_posix_bugfix_enabled() = 0; + virtual bool is_mail_attribute(enum rbox_metadata_key key) = 0; + virtual bool is_updateable_attribute(enum rbox_metadata_key key) = 0; + virtual bool is_update_attributes() = 0; + virtual void set_update_attributes(const std::string &update_attributes_) = 0; + + virtual void update_mail_attribute(const char *value) = 0; + virtual void update_updateable_attribute(const char *value) = 0; + + virtual const std::string &get_metadata_storage_module() = 0; + virtual const std::string &get_metadata_storage_attribute() = 0; + + virtual const std::string &get_mail_attribute_key() = 0; + virtual const std::string &get_updateable_attribute_key() = 0; + virtual const std::string &get_update_attributes_key() = 0; + virtual int save_object(const std::string &oid, void* buffer) = 0; + virtual int read_object(const std::string &oid, void* buffer) = 0; + virtual void set_io_ctx_namespace(const std::string &namespace_) = 0; +}; + +} /* namespace storage_interface */ + +#endif /*SRC_STORAGE_INTERFACES__RADOS_CEPH_CONFIG_H_*/ diff --git a/src/storage-interface/rados-ceph-json-config.h b/src/storage-interface/rados-ceph-json-config.h new file mode 100644 index 00000000..92a2af27 --- /dev/null +++ b/src/storage-interface/rados-ceph-json-config.h @@ -0,0 +1,90 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_RADOS_CEPH_JSON_CONFIG_H_ +#define SRC_STORAGE_INTERFACES_RADOS_CEPH_JSON_CONFIG_H_ + +#include +#include "rados-types.h" + +namespace storage_interface { +/** + * Rados CephJsonConfig + * + * Dovecot-Ceph-plugin configuration + * Ceph Configuration is json object, class + * provides access to json object / Serialize / Deserialize + * + */ +class RadosCephJsonConfig { + public: + virtual ~RadosCephJsonConfig() {} + + virtual bool from_json(void* buffer) = 0; + virtual bool to_json(void* buffer) = 0; + virtual std::string to_string() = 0; + + virtual const std::string& get_cfg_object_name() const = 0; + + virtual void set_cfg_object_name(const std::string& cfgObjectName) = 0; + + virtual const std::string& get_user_mapping() const = 0; + + virtual void set_user_mapping(const std::string& user_mapping_) = 0; + + virtual bool is_valid() const = 0; + + virtual void set_valid(bool isValid) = 0; + + virtual std::string& get_user_ns() = 0; + + virtual void set_user_ns(const std::string& user_ns_) = 0; + + virtual std::string& get_user_suffix() = 0; + + virtual void set_user_suffix(const std::string& user_suffix_) = 0; + + virtual const std::string& get_public_namespace() const = 0; + + virtual void set_public_namespace(const std::string& public_namespace_) = 0; + + virtual void set_mail_attributes(const std::string& mail_attributes_) = 0; + virtual void set_update_attributes(const std::string& update_attributes_) = 0; + virtual void set_updateable_attributes(const std::string& updateable_attributes_) = 0; + + virtual bool is_mail_attribute(enum rbox_metadata_key key) = 0; + virtual bool is_updateable_attribute(enum rbox_metadata_key key) = 0; + virtual bool is_update_attributes() = 0; + + virtual void set_metadata_storage_module(const std::string& metadata_storage_module_) = 0; + virtual const std::string& get_metadata_storage_module() = 0; + virtual void set_metadata_storage_attribute(const std::string& metadata_storage_attribute_) = 0; + virtual const std::string& get_metadata_storage_attribute() = 0; + + virtual void update_mail_attribute(const char* value) = 0; + virtual void update_updateable_attribute(const char* value) = 0; + + virtual const std::string& get_key_user_mapping() const = 0; + virtual const std::string& get_key_ns_cfg() const = 0; + virtual const std::string& get_key_ns_suffix() const = 0; + virtual const std::string& get_key_public_namespace() const = 0; + + virtual const std::string& get_mail_attribute_key() = 0; + virtual const std::string& get_updateable_attribute_key() = 0; + virtual const std::string& get_update_attributes_key() = 0; + + virtual const std::string& get_metadata_storage_module_key() = 0; + virtual const std::string& get_metadata_storage_attribute_key() = 0; +}; + +} /* namespace storage_interface */ + +#endif // SRC_STORAGE_INTERFACES_RADOS_CEPH_JSON_CONFIG_H_ diff --git a/src/librmb/rados-cluster.h b/src/storage-interface/rados-cluster.h similarity index 83% rename from src/librmb/rados-cluster.h rename to src/storage-interface/rados-cluster.h index b57a10b4..1177c419 100644 --- a/src/librmb/rados-cluster.h +++ b/src/storage-interface/rados-cluster.h @@ -9,14 +9,14 @@ * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_INTERFACES_RADOS_CLUSTER_INTERFACE_H_ -#define SRC_LIBRMB_INTERFACES_RADOS_CLUSTER_INTERFACE_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_CLUSTER_INTERFACE_H_ +#define SRC_STORAGE_INTERFACES_RADOS_CLUSTER_INTERFACE_H_ #include - -#include - -namespace librmb { +#include +#include +#include"rbox-io-ctx.h" +namespace storage_interface { /** class RadosDictionary * brief an abstract Rados Cluster * details This abstract class provides the api @@ -52,7 +52,7 @@ class RadosCluster { * @praam[in] valid io_ctx. * @return linux errror code or 0 if successful * */ - virtual int io_ctx_create(const std::string &pool, librados::IoCtx *io_ctx) = 0; + virtual int io_ctx_create(const std::string &pool, storage_interface::RboxIoCtx *io_ctx_wrapper) = 0; /*! * read ceph configuration * @param[in] option option name as described in the ceph documentation @@ -64,7 +64,7 @@ class RadosCluster { /** * creates or returns the recovery index io ctx */ - virtual int recovery_index_io_ctx(const std::string &pool,librados::IoCtx *io_ctx) = 0; + virtual int recovery_index_io_ctx(const std::string &pool,storage_interface::RboxIoCtx *io_ctx_wrapper) = 0; virtual int get_config_option(const char *option, std::string *value) = 0; @@ -91,6 +91,6 @@ class RadosCluster { }; -} // namespace librmb +} // namespace storage_interface -#endif // SRC_LIBRMB_INTERFACES_RADOS_CLUSTER_INTERFACE_H_ +#endif // SRC_STORAGE_INTERFACES_RADOS_CLUSTER_INTERFACE_H_ diff --git a/src/librmb/rados-dictionary.h b/src/storage-interface/rados-dictionary.h similarity index 53% rename from src/librmb/rados-dictionary.h rename to src/storage-interface/rados-dictionary.h index 9ab0a85e..fdb9cb11 100644 --- a/src/librmb/rados-dictionary.h +++ b/src/storage-interface/rados-dictionary.h @@ -9,14 +9,13 @@ * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_INTERFACES_RADOS_DICTIONARY_INTERFACE_H_ -#define SRC_LIBRMB_INTERFACES_RADOS_DICTIONARY_INTERFACE_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_DICTIONARY_INTERFACE_H_ +#define SRC_STORAGE_INTERFACES_RADOS_DICTIONARY_INTERFACE_H_ #include +#include "rbox-io-ctx.h" -#include - -namespace librmb { +namespace storage_interface { /** * Rados Dictionary @@ -35,16 +34,19 @@ class RadosDictionary { virtual const std::string& get_username() = 0; virtual const std::string& get_poolname() = 0; - virtual librados::IoCtx& get_io_ctx(const std::string& key) = 0; - virtual librados::IoCtx& get_shared_io_ctx() = 0; - virtual librados::IoCtx& get_private_io_ctx() = 0; + virtual storage_interface::RboxIoCtx* get_io_ctx_wrapper(const std::string& key) = 0; + virtual storage_interface::RboxIoCtx* get_shared_io_ctx_wrapper() = 0; + virtual storage_interface::RboxIoCtx* get_private_io_ctx_wrapper() = 0; - virtual void remove_completion(librados::AioCompletion* c) = 0; - virtual void push_back_completion(librados::AioCompletion* c) = 0; + virtual void remove_completion(storage_interface::RboxIoCtx* remove_completion_wrapper) = 0; + virtual void push_back_completion(storage_interface::RboxIoCtx* push_back_completion_wrapper_) = 0; virtual void wait_for_completions() = 0; virtual int get(const std::string& key, std::string* value_r) = 0; + + storage_interface::RboxIoCtx* remove_completion_wrapper; + storage_interface::RboxIoCtx* push_back_completion_wrapper; }; -} // namespace librmb +} // namespace storage_interface -#endif // SRC_LIBRMB_INTERFACES_RADOS_DICTIONARY_INTERFACE_H_ +#endif // SRC_STORAGE_INTERFACES_RADOS_DICTIONARY_INTERFACE_H_ diff --git a/src/librmb/rados-dovecot-ceph-cfg.h b/src/storage-interface/rados-dovecot-ceph-cfg.h similarity index 81% rename from src/librmb/rados-dovecot-ceph-cfg.h rename to src/storage-interface/rados-dovecot-ceph-cfg.h index 4d6e0b7b..e1403ec9 100644 --- a/src/librmb/rados-dovecot-ceph-cfg.h +++ b/src/storage-interface/rados-dovecot-ceph-cfg.h @@ -8,14 +8,18 @@ * License version 2.1, as published by the Free Software * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_RADOS_DOVECOT_CEPH_CFG_H_ -#define SRC_LIBRMB_RADOS_DOVECOT_CEPH_CFG_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_DOVECOT_CEPH_CFG_H_ +#define SRC_STORAGE_INTERFACES_RADOS_DOVECOT_CEPH_CFG_H_ #include "rados-types.h" #include #include +#include #include "rados-storage.h" -namespace librmb { +#include "rados-ceph-config.h" +#include "rbox-io-ctx.h" +#include "rados-types.h" +namespace storage_interface { /** * class RadosDovecotCephCfg @@ -44,6 +48,9 @@ class RadosDovecotCephCfg { virtual int get_chunk_size() = 0; virtual int get_write_method() = 0; + virtual const std::string &get_log_file_path() = 0; + virtual const std::string &get_log_level() = 0; + virtual int get_object_search_method() = 0; virtual int get_object_search_threads() = 0; @@ -69,7 +76,7 @@ class RadosDovecotCephCfg { virtual std::string &get_key_prefix_keywords() = 0; - virtual void set_io_ctx(librados::IoCtx *io_ctx) = 0; + virtual void set_io_ctx_wrapper(storage_interface::RboxIoCtx *io_ctx_wrapper) = 0; virtual int load_rados_config() = 0; virtual int save_default_rados_config() = 0; virtual void set_user_mapping(bool value_) = 0; @@ -86,22 +93,23 @@ class RadosDovecotCephCfg { * @param[in] buffer configuration object * * @return linux error codes or 0 if successful */ - virtual int save_object(const std::string &oid, librados::bufferlist &buffer) = 0; + virtual int save_object(const std::string &oid, std::istream &stream_buffer) = 0; /*! * read configuration from object * @param[in] unique ident * @param[out] valid pointer to buffer. * @return linux error codes or 0 if successful */ - virtual int read_object(const std::string &oid, librados::bufferlist *buffer) = 0; + virtual int read_object(const std::string &oid, std::ostream &stream_buffer) = 0; /*! * set rados configuration namespace */ virtual void set_io_ctx_namespace(const std::string &namespace_) = 0; virtual bool is_rbox_check_empty_mailboxes() = 0; + virtual storage_interface::RadosCephConfig *get_rados_ceph_cfg() = 0; }; -} /* namespace librmb */ +} /* namespace storage_interface */ -#endif /* SRC_LIBRMB_RADOS_DOVECOT_CEPH_CFG_H_ */ +#endif /* SRC_STORAGE_INTERFACES_RADOS_DOVECOT_CEPH_CFG_H_ */ diff --git a/src/librmb/rados-guid-generator.h b/src/storage-interface/rados-guid-generator.h similarity index 72% rename from src/librmb/rados-guid-generator.h rename to src/storage-interface/rados-guid-generator.h index 7c0507b0..fb1052ae 100644 --- a/src/librmb/rados-guid-generator.h +++ b/src/storage-interface/rados-guid-generator.h @@ -9,10 +9,10 @@ * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_RADOS_GUID_GENERATOR_H_ -#define SRC_LIBRMB_RADOS_GUID_GENERATOR_H_ - -namespace librmb { +#ifndef SRC_STORAGE_INTERFACES_RADOS_GUID_GENERATOR_H_ +#define SRC_STORAGE_INTERFACES_RADOS_GUID_GENERATOR_H_ +#include +namespace storage_interface { /** * Rados Guid Generator * @@ -29,6 +29,6 @@ class RadosGuidGenerator { virtual void generate_guid(std::string *guid) = 0; }; -} /* namespace librmb */ +} /* namespace storage_interface */ -#endif /* SRC_LIBRMB_RADOS_GUID_GENERATOR_H_ */ +#endif /* SRC_STORAGE_INTERFACES_RADOS_GUID_GENERATOR_H_ */ diff --git a/src/storage-interface/rados-mail.h b/src/storage-interface/rados-mail.h new file mode 100644 index 00000000..834c3bd3 --- /dev/null +++ b/src/storage-interface/rados-mail.h @@ -0,0 +1,74 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_RADOS_MAIL_INTERFACE_H_ +#define SRC_STORAGE_INTERFACES_RADOS_MAIL_INTERFACE_H_ + +#include +#include +#include +#include +#include "rados-metadata.h" + +namespace storage_interface { + +using std::map; +using std::string; + +class RadosMail{ + public: + virtual ~RadosMail(){} + virtual void set_oid(const char* _oid) = 0; + virtual void set_oid(const string& _oid) = 0; + virtual void set_mail_size(const int _size) = 0; + virtual void set_rados_save_date(const time_t& _save_date) = 0; + virtual string* get_oid() = 0; + virtual int get_mail_size() = 0; + virtual time_t get_rados_save_date() = 0; + virtual uint8_t get_guid_ref() = 0; + + /*! + * @return ptr to internal buffer . + */ + virtual void set_mail_buffer(void* buffer) = 0; + virtual void* get_mail_buffer() = 0; + virtual map* get_metadata() = 0; + virtual bool is_index_ref() = 0; + virtual void set_index_ref(bool ref) = 0; + virtual bool is_valid() = 0; + virtual void set_valid(bool valid_) = 0; + + virtual bool is_restored() = 0; + virtual void set_restored(bool restored_) = 0; + + virtual bool is_lost_object() = 0; + virtual void set_lost_object(bool is_lost_object) = 0; + virtual string to_string(const string& padding) = 0; + virtual void add_metadata(const storage_interface::RadosMetadata* metadata) = 0; + virtual bool is_deprecated_uid() = 0; + virtual void set_deprecated_uid(bool deprecated_uid_) = 0; + /*! + * Some metadata isn't saved as xattribute (default). To access those, get_extended_metadata can + * be used. + */ + virtual map* get_extended_metadata() = 0; + /*! + * Save metadata to extended metadata store currently omap + * @param[in] metadata valid radosMetadata. + */ + virtual void add_extended_metadata(const storage_interface::RadosMetadata* metadata) = 0; + + virtual const string get_extended_metadata(const string& key) = 0; +}; + +} // namespace storage_interface + +#endif // SRC_STORAGE_INTERFACES_RADOS_MAIL_INTERFACE_H_ diff --git a/src/storage-interface/rados-metadata-storage-module.h b/src/storage-interface/rados-metadata-storage-module.h new file mode 100644 index 00000000..17135789 --- /dev/null +++ b/src/storage-interface/rados-metadata-storage-module.h @@ -0,0 +1,41 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_RADOS_METADATA_STORAGE_MODULE_H_ +#define SRC_STORAGE_INTERFACES_RADOS_METADATA_STORAGE_MODULE_H_ + +#include +#include +#include +#include "rados-mail.h" +#include "rbox-io-ctx.h" +#include "rados-metadata.h" + +namespace storage_interface { +class RadosStorageMetadataModule { + public: + virtual ~RadosStorageMetadataModule(){}; + /* update io_ctx */ + virtual void set_io_ctx(storage_interface::RboxIoCtx *io_ctx_wrapper){}; + /* load the metadta into RadosMail */ + virtual int load_metadata(storage_interface::RadosMail *mail) = 0; + /* set a new metadata attribute to a mail object */ + virtual int set_metadata(storage_interface::RadosMail *mail) = 0; + /* update the given metadata attributes */ + virtual bool update_metadata(const std::string &oid, std::list &to_update) = 0; + /* manage keywords */ + virtual int update_keyword_metadata(const std::string &oid, storage_interface::RadosMetadata *metadata) = 0; + virtual int remove_keyword_metadata(const std::string &oid, std::string &key) = 0; +}; + +} // namespace storage_interface + +#endif /* SRC_STORAGE_INTERFACES_RADOS_METADATA_STORAGE_MODULE_H_ */ diff --git a/src/librmb/rados-metadata-storage.h b/src/storage-interface/rados-metadata-storage.h similarity index 69% rename from src/librmb/rados-metadata-storage.h rename to src/storage-interface/rados-metadata-storage.h index ef50f683..00de964d 100644 --- a/src/librmb/rados-metadata-storage.h +++ b/src/storage-interface/rados-metadata-storage.h @@ -8,13 +8,13 @@ * License version 2.1, as published by the Free Software * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_RADOS_METADATA_STORAGE_PRODUCER_H_ -#define SRC_LIBRMB_RADOS_METADATA_STORAGE_PRODUCER_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_METADATA_STORAGE_PRODUCER_H_ +#define SRC_STORAGE_INTERFACES_RADOS_METADATA_STORAGE_PRODUCER_H_ #include "rados-dovecot-ceph-cfg.h" - #include "rados-metadata-storage-module.h" +#include "rbox-io-ctx.h" -namespace librmb { +namespace storage_interface { /** * Rados MetadataStorage * @@ -25,10 +25,9 @@ class RadosMetadataStorage { public: virtual ~RadosMetadataStorage() {} /* create the medata data class based on configuration */ - virtual RadosStorageMetadataModule *create_metadata_storage(librados::IoCtx *io_ctx_, RadosDovecotCephCfg *cfg_) = 0; virtual RadosStorageMetadataModule *get_storage() = 0; }; -} // namespace librmb +} // namespace storage_interface -#endif /* SRC_LIBRMB_RADOS_METADATA_STORAGE_PRODUCER_H_ */ +#endif /* SRC_STORAGE_INTERFACES_RADOS_METADATA_STORAGE_PRODUCER_H_ */ diff --git a/src/storage-interface/rados-metadata.h b/src/storage-interface/rados-metadata.h new file mode 100644 index 00000000..17760180 --- /dev/null +++ b/src/storage-interface/rados-metadata.h @@ -0,0 +1,41 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) = 0;2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_RADOS_METADATA_H_ +#define SRC_STORAGE_INTERFACES_RADOS_METADATA_H_ + + +#include +#include +#include +#include +#include "rados-types.h" +namespace storage_interface { + +class RadosMetadata { + public: + virtual ~RadosMetadata(){} + + virtual void convert(const char* value, time_t* t) = 0; + virtual bool from_string(const std::string& str) = 0; + virtual std::string to_string() = 0; + virtual void convert(enum rbox_metadata_key _key, const std::string& val) = 0; + virtual void convert(enum rbox_metadata_key _key, const time_t& time) = 0; + virtual void convert(enum rbox_metadata_key _key, const char* value) = 0; + virtual void convert(enum rbox_metadata_key _key, const uint& value) = 0; + virtual void convert(enum rbox_metadata_key _key, const size_t& value) = 0; + virtual void convert(enum rbox_metadata_key _key, const int value) = 0; + virtual void* get_buffer() = 0; + virtual std::string& get_key() = 0; + virtual void set_key(std::string& key_) = 0; +}; +} // end namespace +#endif /* SRC_STORAGE_INTERFACES_RADOS_METADATA_H_ */ diff --git a/src/storage-interface/rados-namespace-manager.h b/src/storage-interface/rados-namespace-manager.h new file mode 100644 index 00000000..22faed6a --- /dev/null +++ b/src/storage-interface/rados-namespace-manager.h @@ -0,0 +1,22 @@ +#ifndef SRC_STORAGE_INTERFACES_RADOS_NAMESPACE_MANAGER_H_ +#define SRC_STORAGE_INTERFACES_RADOS_NAMESPACE_MANAGER_H_ +#include +#include +#include "rados-storage.h" +#include "rados-dovecot-ceph-cfg.h" +#include "rados-guid-generator.h" +namespace storage_interface +{ +class RadosNamespaceManager { + public: + virtual ~RadosNamespaceManager(){} + virtual void set_config(storage_interface::RadosDovecotCephCfg *config_) = 0; + virtual storage_interface::RadosDovecotCephCfg *get_config() = 0; + + virtual void set_namespace_oid(std::string &namespace_oid_) = 0; + virtual bool lookup_key(const std::string &uid, std::string *value) = 0; + virtual bool add_namespace_entry(const std::string &uid, std::string *value, storage_interface::RadosGuidGenerator *guid_generator_) = 0; +}; + +} // namespace storage_interface +#endif SRC_STORAGE_INTERFACES_RADOS_NAMESPACE_MANAGER_H_ \ No newline at end of file diff --git a/src/storage-interface/rados-plugin-logger.h b/src/storage-interface/rados-plugin-logger.h new file mode 100644 index 00000000..ffd63d89 --- /dev/null +++ b/src/storage-interface/rados-plugin-logger.h @@ -0,0 +1,107 @@ +#ifndef SRC_LIBRMB_RADOS_PLUGIN_LOGGER_H_ +#define SRC_LIBRMB_RADOS_PLUGIN_LOGGER_H_ + +#ifdef SPD_LOGGING_FRAMEWORK + #include + #include + #include +#endif + +#include +#include +#include +#include + +namespace librmb { + +enum LOG_LEVEL { + DEBUG_LEVEL, + WARN, + INFO, + ERROR +}; + + +class RadosPluginLogger { + public: + // Public static member function to access the instance + static RadosPluginLogger& getInstance() { + static RadosPluginLogger instance; // Guaranteed to be destroyed at the end of the program + return instance; + } + + template + void log(librmb::LOG_LEVEL level, const char* format, Args&&... args) { + #ifdef SPD_LOGGING_FRAMEWORK + + if(!logger) { + // you need to first call init(...) before using the logger + return; + } + std::string message = fmt::format(format, std::forward(args)...); + logger->log(convert(level), message); + //TODO: may decrease performance + logger->flush(); + #endif + } + void init(const std::string &logFilePath, const std::string &loglevel) { + #ifdef SPD_LOGGING_FRAMEWORK + if(logger){ + return; + } + bool truncate = false; + auto file_sink = std::make_shared(logFilePath, truncate); + logger = spdlog::stdout_color_mt("dovecot_ceph_plugin"); + + logger->sinks().emplace_back(file_sink); + if(loglevel == "debug"){ + logger->set_level(spdlog::level::debug); + } + else if(loglevel == "info"){ + logger->set_level(spdlog::level::info); + } + else if(loglevel == "warn"){ + logger->set_level(spdlog::level::warn); + } + else if(loglevel == "error"){ + logger->set_level(spdlog::level::err); + } + else { + logger->set_level(spdlog::level::debug); + } + #endif + } + private: + + RadosPluginLogger() { + // const std::string logFilePath = "/tmp/dovecot_ceph_plugin.log"; + // init(logFilePath, "debug"); + } + #ifdef SPD_LOGGING_FRAMEWORK + + spdlog::level::level_enum convert(librmb::LOG_LEVEL level) { + switch(level) { + case librmb::LOG_LEVEL::DEBUG_LEVEL: + return spdlog::level::debug; + case librmb::LOG_LEVEL::WARN: + return spdlog::level::warn; + case librmb::LOG_LEVEL::INFO: + return spdlog::level::info; + case librmb::LOG_LEVEL::ERROR: + return spdlog::level::err; + default: + return spdlog::level::debug; + } + } + #endif + + RadosPluginLogger(const RadosPluginLogger&) = delete; + RadosPluginLogger& operator=(const RadosPluginLogger&) = delete; + #ifdef SPD_LOGGING_FRAMEWORK + std::shared_ptr logger; + #endif +}; + +} // namespace librmb + +#endif // SRC_LIBRMB_RADOS_PLUGIN_LOGGER_H_ diff --git a/src/storage-interface/rados-save-log.h b/src/storage-interface/rados-save-log.h new file mode 100644 index 00000000..7fe45ce4 --- /dev/null +++ b/src/storage-interface/rados-save-log.h @@ -0,0 +1,62 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_RADOS_SAVE_LOG_H_ +#define SRC_STORAGE_INTERFACES_RADOS_SAVE_LOG_H_ + +#include +#include +#include +#include +#include +#include + +#include "rados-metadata.h" + +namespace storage_interface { +/** + * RadosSaveLogEntry + * + * Class provides access to the savelog.format. + * + */ +class RadosSaveLogEntry { + public: + ~RadosSaveLogEntry(){}; + virtual bool parse_mv_op() = 0; + virtual std::string op_save() = 0; + virtual std::string op_cpy() = 0; + virtual std::string op_mv(const std::string &src_ns, const std::string &src_oid, const std::string &src_user, + std::list &metadata) = 0; + virtual std::string& get_oid() = 0; + virtual std::string& get_ns() = 0; + virtual std::string& get_pool() = 0; + virtual std::string& get_op() = 0; + virtual std::string& get_src_oid() = 0; + virtual std::string& get_src_ns() = 0; + virtual std::string& get_src_user() = 0; + virtual std::list& get_metadata() = 0; +}; + +class RadosSaveLog { + public: + virtual ~RadosSaveLog(){}; + virtual void set_save_log_file(const std::string &logfile_) = 0; + + virtual bool open() = 0; + virtual void append(const RadosSaveLogEntry *entry) = 0; + virtual bool close() = 0; + virtual bool is_open() = 0; +}; + +} /* namespace storage_interface */ + +#endif /*SRC_STORAGE_INTERFACES_RADOS_SAVE_LOG_H_ */ diff --git a/src/librmb/rados-storage.h b/src/storage-interface/rados-storage.h similarity index 64% rename from src/librmb/rados-storage.h rename to src/storage-interface/rados-storage.h index fcc0690d..a2565ecd 100644 --- a/src/librmb/rados-storage.h +++ b/src/storage-interface/rados-storage.h @@ -9,19 +9,19 @@ * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_INTERFACES_RADOS_STORAGE_INTERFACE_H_ -#define SRC_LIBRMB_INTERFACES_RADOS_STORAGE_INTERFACE_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_STORAGE_INTERFACE_H_ +#define SRC_STORAGE_INTERFACES_RADOS_STORAGE_INTERFACE_H_ #include #include #include - -#include +#include #include "rados-cluster.h" #include "rados-mail.h" #include "rados-types.h" +#include "rbox-io-ctx.h" -namespace librmb { +namespace storage_interface { /** class RadosStorage * brief an abstract Rados Storage * details This abstract class provides the api @@ -34,14 +34,7 @@ class RadosStorage { /*! * if connected, return the valid ioCtx */ - virtual librados::IoCtx &get_io_ctx() = 0; - - /*! - * if connected, return the valid ioCtx for recovery index - */ - virtual librados::IoCtx &get_recovery_io_ctx() = 0; - - + virtual storage_interface::RboxIoCtx* get_io_ctx_wrapper() = 0; /*! get the object size and object save date * @param[in] oid unique ident for the object * @param[out] psize size of the object @@ -56,25 +49,14 @@ class RadosStorage { * */ virtual std::string get_namespace() = 0; /*! get the pool name - * @return copy of the current p - -TEST_F(StorageTest, scanForPg) { - -librmbtest::RadosClusterMock mock_test = librmbtest::RadosClusterMock(); -mock. -librmb::RadosStorageImpl underTest = librmbtest::RadosStorageImpl(mock_test); - -underTest.ceph_index_append() -underTest.ceph_index_add("dkfkjdf") - -} - ool name + * @return copy of the current + pool name * */ virtual std::string get_pool_name() = 0; /* set the wait method for async operations */ - virtual void set_ceph_wait_method(enum rbox_ceph_aio_wait_method wait_method) = 0; - + virtual void set_ceph_wait_method(storage_interface::rbox_ceph_aio_wait_method wait_method) = 0; + /*! get the max operation size in mb * @return the maximal number of mb to write in a single write operation*/ virtual int get_max_write_size() = 0; @@ -85,47 +67,19 @@ underTest.ceph_index_add("dkfkjdf") /*! get the max ceph object size */ virtual int get_max_object_size() = 0; - - /*! In case the current object size exceeds the max_write (bytes), object should be split into - * max smaller operations and executed separately. - * - * @param[in] current_object ptr to a valid mailobject. - * @param[in] write_op_xattr pointer to a write operation / if null, function will create new one. - * @param[in] max_write max number of bytes to write in one operation - * - * @return <0 in case of failure - * */ - virtual int split_buffer_and_exec_op(RadosMail *current_object, librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write) = 0; - - /*! deletes a mail object from rados - * @param[in] mail pointer to valid mail object - * - * @return <0 in case of failure. */ - virtual int delete_mail(RadosMail *mail) = 0; /*! delete object with given oid * @param[in] object identifier. * * @return <0 in case of failure */ virtual int delete_mail(const std::string &oid) = 0; - /*! asynchron execution of a write operation - * - * @param[in] io_ctx valid io context - * @param[in] oid object identifier - * @param[in] c valid pointer to a completion. - * @param[in] op the prepared write operation - * */ - virtual int aio_operate(librados::IoCtx *io_ctx_, const std::string &oid, librados::AioCompletion *c, - librados::ObjectWriteOperation *op) = 0; /*! search for mails based on given Filter * @param[in] attr a list of filter attributes * * @return object iterator or librados::NObjectIterator::__EndObjectIterator */ - virtual librados::NObjectIterator find_mails(const RadosMetadata *attr) = 0; - - - virtual std::set find_mails_async(const RadosMetadata *attr, + virtual std::set find_mails(const storage_interface::RadosMetadata *attr) = 0; + + virtual std::set find_mails_async(const storage_interface::RadosMetadata *attr, std::string &pool_name, int num_threads, void (*ptr)(std::string&)) = 0; @@ -168,31 +122,6 @@ underTest.ceph_index_add("dkfkjdf") * close the connection. (clean up structures to allow reconnect) */ virtual void close_connection() = 0; - - /*! wait for all write operations to complete - * @param[in] completion_op_map map of write operations with matching completion objects. - * @return false if successful !!!! - * */ - virtual bool wait_for_write_operations_complete(librados::AioCompletion *completion, - librados::ObjectWriteOperation *write_operation) = 0; - /*! - * wait for all rados operations - * - * @param[in] object_list list of outstanding rados objects - * - * @return true if successful - */ - virtual bool wait_for_rados_operations(const std::list &object_list) = 0; - - /*! save the mail object - * - * @param[in] oid unique object identifier - * @param[in] buffer the objects data - * - * @return linux errorcode or 0 if successful - * */ - virtual int save_mail(const std::string &oid, librados::bufferlist &buffer) = 0; - /** * append oid to index object */ @@ -231,7 +160,9 @@ underTest.ceph_index_add("dkfkjdf") * @param[out] buffer valid ptr to bufferlist. * @return linux errorcode or 0 if successful * */ - virtual int read_mail(const std::string &oid, librados::bufferlist *buffer) = 0; + // virtual int read_mail(const std::string &oid, librados::bufferlist *buffer) = 0; + virtual int read_mail(const std::string &oid, storage_interface::RadosMail* mail,int try_counter)=0; + /*! move a object from the given namespace to the other, updates the metadata given in to_update list * * @param[in] src_oid unique identifier of source object @@ -243,7 +174,7 @@ underTest.ceph_index_add("dkfkjdf") * @return linux errorcode or 0 if successful * */ virtual int move(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update, bool delete_source) = 0; + std::list &to_update, bool delete_source) = 0; /*! copy a object from the given namespace to the other, updates the metadata given in to_update list * @param[in] src_oid unique identifier of source object @@ -254,33 +185,30 @@ underTest.ceph_index_add("dkfkjdf") * @return linux errorcode or 0 if successful */ virtual int copy(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update) = 0; + std::list &to_update) = 0; /*! save the mail * @param[in] mail valid rados mail. * @return false in case of error * */ - virtual bool save_mail(RadosMail *mail) = 0; - /*! - * save the mail - * @param[in] write_op_xattr write operation to use - * @param[in] mail valid mail object - * @return false in case of error. - * - */ - virtual bool save_mail(librados::ObjectWriteOperation *write_op_xattr, RadosMail *mail) = 0; + virtual bool save_mail(storage_interface::RadosMail *mail) = 0; + /*! create a new RadosMail * create new rados Mail Object. * return pointer to mail object or nullptr * */ - virtual librmb::RadosMail *alloc_rados_mail() = 0; + virtual storage_interface::RadosMail *alloc_rados_mail() = 0; /*! free the Rados Mail Object * @param[in] mail ptr to valid mail object * */ - virtual void free_rados_mail(librmb::RadosMail *mail) = 0; + virtual void free_rados_mail(storage_interface::RadosMail *mail) = 0; + virtual void* alloc_mail_buffer() = 0; + virtual const char* get_mail_buffer(void *buffer,int *mail_buff_size) = 0; + virtual void free_mail_buffer(void *mail_buffer) = 0; + virtual void append_to_buffer(void *buff,const unsigned char *chunk, size_t size) = 0; }; -} // namespace librmb +} // namespace storage_interface -#endif // SRC_LIBRMB_INTERFACES_RADOS_STORAGE_INTERFACE_H_ +#endif // SRC_STORAGE_INTERFACES_RADOS_STORAGE_INTERFACE_H_ diff --git a/src/librmb/rados-types.h b/src/storage-interface/rados-types.h similarity index 95% rename from src/librmb/rados-types.h rename to src/storage-interface/rados-types.h index 32572193..978712f2 100644 --- a/src/librmb/rados-types.h +++ b/src/storage-interface/rados-types.h @@ -9,10 +9,10 @@ * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_RADOS_TYPES_H_ -#define SRC_LIBRMB_RADOS_TYPES_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_TYPES_H_ +#define SRC_STORAGE_INTERFACES_RADOS_TYPES_H_ -namespace librmb { +namespace storage_interface { #define GUID_128_SIZE 16 /** * The available metadata keys used as rados @@ -123,4 +123,4 @@ static const char *rbox_metadata_key_to_char(rbox_metadata_key type) { } enum rbox_ceph_aio_wait_method { WAIT_FOR_COMPLETE_AND_CB, WAIT_FOR_SAFE_AND_CB }; } // namespace -#endif /* SRC_LIBRMB_RADOS_TYPES_H_ */ +#endif /* SRC_STORAGE_INTERFACES_RADOS_TYPES_H_ */ diff --git a/src/librmb/rados-util.h b/src/storage-interface/rados-util.h similarity index 56% rename from src/librmb/rados-util.h rename to src/storage-interface/rados-util.h index b3ae010f..f6b0b05e 100644 --- a/src/librmb/rados-util.h +++ b/src/storage-interface/rados-util.h @@ -1,16 +1,16 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// -*- mode:C++ ; tab-width:8 ; c-basic-offset:2 ; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab /* * Copyright (c) 2017-2018 Tallence AG and the authors * - * This is free software; you can redistribute it and/or + * This is free software ; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License version 2.1, as published by the Free Software * Foundation. See file COPYING. */ -#ifndef SRC_LIBRMB_RADOS_UTIL_H_ -#define SRC_LIBRMB_RADOS_UTIL_H_ +#ifndef SRC_STORAGE_INTERFACES_RADOS_UTIL_H_ +#define SRC_STORAGE_INTERFACES_RADOS_UTIL_H_ #include #include @@ -18,17 +18,16 @@ #include #include -#include #include #include #include -#include #include "rados-storage.h" #include "rados-metadata-storage.h" #include "rados-types.h" +#include "rbox-io-ctx.h" -namespace librmb { +namespace storage_interface { /** * Rados Utils @@ -38,8 +37,7 @@ namespace librmb { */ class RadosUtils { public: - RadosUtils(); - virtual ~RadosUtils(); + virtual ~RadosUtils(){} /*! * convert given date string to time_t * @@ -47,43 +45,43 @@ class RadosUtils { * @param[out] val ptr to time_t. * @return true if sucessfull. */ - static bool convert_str_to_time_t(const std::string &date, time_t *val); + virtual bool convert_str_to_time_t(const std::string &date, time_t *val) = 0; /*! * check if given string is a numeric value. * @param[in] s string if s is empty => false * @return true if given string is numeric. */ - static bool is_numeric(const char *s); + virtual bool is_numeric(const char *s) = 0; /*! * check if given string is a numeric value. * @param[in] text string, if string is empty => true * @return true if given string is numeric. */ - static bool is_numeric_optional(const char *text); + virtual bool is_numeric_optional(const char *text) = 0; /*! * checks if key is a data attribute */ - static bool is_date_attribute(const rbox_metadata_key &key); + virtual bool is_date_attribute(const storage_interface::rbox_metadata_key &key) = 0; /*! * converts given data_string to numeric string * @param[in] date_string Date format: %Y-%m-%d %H:%M:%S * @param[out] date : unix timestamp. */ - static bool convert_string_to_date(const std::string &date_string, std::string *date); + virtual bool convert_string_to_date(const std::string &date_string, std::string *date) = 0; /*! * converts given time_to to string %Y-%m-%d %H:%M:%S * @param[in] t time_t * @param[out] ret_val : ptr to valid string buffer. * @return <0 error */ - static int convert_time_t_to_str(const time_t &t, std::string *ret_val); + virtual int convert_time_t_to_str(const time_t &t, std::string *ret_val) = 0; /*! * converts flags to hex string * @param[in] flags flags * @param[out] ptr to string buffer: * @return false if not sucessful */ - static bool flags_to_string(const uint8_t &flags, std::string *flags_str); + virtual bool flags_to_string(const uint8_t &flags, std::string *flags_str) = 0; /*! * converts hex string to uint8_t @@ -91,7 +89,7 @@ class RadosUtils { * @param[out] flags to uint8_t * @return false if not sucessful */ - static bool string_to_flags(const std::string &flags_str, uint8_t *flags); + virtual bool string_to_flags(const std::string &flags_str, uint8_t *flags) = 0; /*! * replace string in text. @@ -99,7 +97,7 @@ class RadosUtils { * @param[in] find : text to find. * @param[in] replace: text to replace. */ - static void find_and_replace(std::string *source, std::string const &find, std::string const &replace); + virtual void find_and_replace(std::string *source, std::string const &find, std::string const &replace) = 0; /*! * get a list of key value pairs @@ -107,14 +105,14 @@ class RadosUtils { * @param[in] oid: unique identifier * @param[out] kv_map valid ptr to key value map. */ - static int get_all_keys_and_values(librados::IoCtx *io_ctx, const std::string &oid, - std::map *kv_map); + virtual int get_all_keys_and_values(storage_interface::RboxIoCtx *io_ctx, const std::string &oid, + storage_interface::RadosMail* mail) = 0; /*! * get the text representation of uint flags. * @param[in] flags * @param[out] flat : string representation */ - static void resolve_flags(const uint8_t &flags, std::string *flat); + virtual void resolve_flags(const uint8_t &flags, std::string *flat) = 0; /*! * copy object to alternative storage * @param[in] src_oid @@ -125,8 +123,8 @@ class RadosUtils { * @param[in] bool inverse if true, copy from alt to primary. * @return linux error code or 0 if sucessful */ - static int copy_to_alt(std::string &src_oid, std::string &dest_oid, RadosStorage *primary, RadosStorage *alt_storage, - RadosMetadataStorage *metadata, bool inverse); + virtual int copy_to_alt(std::string &src_oid, std::string &dest_oid, storage_interface::RadosStorage *primary, storage_interface::RadosStorage *alt_storage, + storage_interface::RadosMetadataStorage *metadata, bool inverse) = 0; /*! * move object to alternative storage * @param[in] src_oid @@ -137,8 +135,8 @@ class RadosUtils { * @param[in] bool inverse if true, move from alt to primary. * @return linux error code or 0 if sucessful */ - static int move_to_alt(std::string &oid, RadosStorage *primary, RadosStorage *alt_storage, - RadosMetadataStorage *metadata, bool inverse); + virtual int move_to_alt(std::string &oid, storage_interface::RadosStorage *primary, storage_interface::RadosStorage *alt_storage, + storage_interface::RadosMetadataStorage *metadata, bool inverse) = 0; /*! * increment (add) value directly on osd * @param[in] ioctx @@ -148,7 +146,7 @@ class RadosUtils { * * @return linux error code or 0 if sucessful */ - static int osd_add(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, long long value_to_add); + virtual int osd_add(storage_interface::RboxIoCtx *ioctx, const std::string &oid, const std::string &key, long long value_to_add) = 0; /*! * decrement (sub) value directly on osd * @param[in] ioctx @@ -158,8 +156,8 @@ class RadosUtils { * * @return linux error code or 0 if sucessful */ - static int osd_sub(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, - long long value_to_subtract); + virtual int osd_sub(storage_interface::RboxIoCtx *ioctx, const std::string &oid, const std::string &key, + long long value_to_subtract) = 0; /*! * check all given metadata key is valid @@ -167,7 +165,7 @@ class RadosUtils { * @param[in] metadata * @return true if all keys and value are correct. (type, name, value) */ - static bool validate_metadata(std::map *metadata); + virtual bool validate_metadata(std::map *metadata) = 0; /*! * get metadata * @@ -175,7 +173,7 @@ class RadosUtils { * @param[int] valid pointer to metadata map * @return the metadata value */ - static void get_metadata(const std::string &key, std::map *metadata, char **value); + virtual void get_metadata(const std::string &key, std::map *metadata, char **value) = 0; /*! * get metadata @@ -184,25 +182,25 @@ class RadosUtils { * @param[int] valid pointer to metadata map * @return the metadata value */ - static void get_metadata(rbox_metadata_key key, std::map *metadata, char **value); + virtual void get_metadata(storage_interface::rbox_metadata_key key, std::map *metadata, char **value) = 0; /** * POC Implemnentation to extract pgs and primary osds from mon_command output! **/ - static std::vector extractPgs(const std::string& str); + virtual std::vector extractPgs(const std::string& str) = 0; - static std::map> extractPgAndPrimaryOsd(const std::string& str); + virtual std::map> extractPgAndPrimaryOsd(const std::string& str) = 0; - static std::vector split(std::string str_to_split, char delimiter); + virtual std::vector split(std::string str_to_split, char delimiter) = 0; - static std::string convert_to_ceph_index(const std::set &list); - static std::string convert_to_ceph_index(const std::string &str); + virtual std::string convert_to_ceph_index(const std::set &list) = 0; + virtual std::string convert_to_ceph_index(const std::string &str) = 0; - static std::set ceph_index_to_set(const std::string &str); + virtual std::set ceph_index_to_set(const std::string &str) = 0; }; -} // namespace librmb +} // namespace storage_interface -#endif // SRC_LIBRMB_RADOS_UTIL_H_ +#endif // SRC_STORAGE_INTERFACES_RADOS_UTIL_H_ diff --git a/src/storage-interface/rbox-io-ctx.h b/src/storage-interface/rbox-io-ctx.h new file mode 100644 index 00000000..d85fadf4 --- /dev/null +++ b/src/storage-interface/rbox-io-ctx.h @@ -0,0 +1,49 @@ + + +#ifndef SRC_STORAGE_INTERFACES_RBOX_IO_CTX_H_ +#define SRC_STORAGE_INTERFACES_RBOX_IO_CTX_H_ + +#include +#include +using librados::AioCompletion; + +namespace storage_interface{ +class RboxIoCtx{ + public: + virtual ~RboxIoCtx() {} + virtual void set_io_ctx(librados::IoCtx& io_ctx_)=0; + virtual librados::IoCtx& get_io_ctx()=0; + virtual librados::IoCtx& get_recovery_io_ctx()=0; + virtual int aio_stat(const std::string& oid,librados::AioCompletion *aio_complete,uint64_t *psize,time_t *pmtime)=0; + virtual int exec(const std::string& oid, const char *cls, const char *method,ceph::bufferlist& inbl, ceph::bufferlist& outbl)=0; + virtual int omap_get_vals_by_keys(const std::string& oid,const std::set& keys, + std::map *vals)=0; + virtual int omap_rm_keys(const std::string& oid,const std::set& keys)=0; + virtual int omap_set(const std::map& map,const std::string& oid)=0; + virtual void omap_set(const std::string& oid,const std::map& map)=0; + virtual int getxattrs(const std::string& oid,std::map& attrset)=0; + virtual int setxattr(const std::string& oid,const char *name, librados::bufferlist& bl)=0; + virtual const librados::NObjectIterator& nobjects_end()=0; + virtual librados::NObjectIterator nobjects_begin()=0; + virtual librados::NObjectIterator nobjects_begin(const librados::bufferlist& filter)=0; + virtual void set_namespace(const std::string& nspace)=0; + virtual int stat(const std::string& oid, uint64_t *psize, time_t *pmtime)=0; + virtual int aio_operate(const std::string& oid, librados::AioCompletion *aio_completion, + librados::ObjectReadOperation *read_op, librados::bufferlist *pbl)=0; + virtual int aio_operate(const std::string& oid, librados::AioCompletion *aio_completion, + librados::ObjectReadOperation *read_op, int flags,librados::bufferlist *pbl)=0; + virtual int aio_operate(const std::string& oid, librados::AioCompletion *aio_complete, librados::ObjectWriteOperation *write_op)=0; + virtual int remove(const std::string& oid)=0; + virtual int write_full(const std::string& oid, librados::bufferlist& bl)=0; + virtual int read(const std::string& oid, librados::bufferlist& bl, size_t len, uint64_t off)=0; + virtual int operate(const std::string& oid, librados::ObjectWriteOperation* write_op_xattr)=0; + virtual bool append(const std::string& oid, librados::bufferlist& bufferlist, int length)=0; + virtual int operate(const std::string& oid,librados::ObjectReadOperation* read_op,librados::bufferlist* buffer)=0; + virtual uint64_t get_last_version()=0; + virtual void set_remove_completion(librados::AioCompletion &aio_commepletion)=0; + virtual librados::AioCompletion& get_remove_completion()=0; + virtual void set_push_back_completion(librados::AioCompletion &aio_commepletion)=0; + virtual librados::AioCompletion& get_push_back_completion()=0; + }; +} //namespace storage_interface +#endif // SRC_STORAGE_INTERFACES_RBOX_IO_CTX_H_ \ No newline at end of file diff --git a/src/storage-interface/tools/Makefile.am b/src/storage-interface/tools/Makefile.am new file mode 100644 index 00000000..2b0585d8 --- /dev/null +++ b/src/storage-interface/tools/Makefile.am @@ -0,0 +1,9 @@ +# +# Copyright (c) 2017-2018 Tallence AG and the authors +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. + +SUBDIRS = rmb \ No newline at end of file diff --git a/src/storage-interface/tools/rmb/Makefile.am b/src/storage-interface/tools/rmb/Makefile.am new file mode 100644 index 00000000..d5d5cf66 --- /dev/null +++ b/src/storage-interface/tools/rmb/Makefile.am @@ -0,0 +1,29 @@ +# +# Copyright (c) 2017-2018 Tallence AG and the authors +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/storage-interface + +shlibs = \ + $(top_builddir)/src/storage-interface/libinterface.la + +bin_PROGRAMS = rmb + +rmb_SOURCES = \ + mailbox_tools.h \ + ls_cmd_parser.h \ + rmb-commands.h \ + rados-mail-box.h + +rmb_LDADD = $(shlibs) + +#man1_MANS = rmb.man + +#EXTRA_DIST = \ +# $(man1_MANS) + diff --git a/src/storage-interface/tools/rmb/ls_cmd_parser.h b/src/storage-interface/tools/rmb/ls_cmd_parser.h new file mode 100644 index 00000000..49777150 --- /dev/null +++ b/src/storage-interface/tools/rmb/ls_cmd_parser.h @@ -0,0 +1,57 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_INTERFACES_TOOLS_RMB_LS_CMD_PARSER_H_ +#define SRC_INTERFACES_TOOLS_RMB_LS_CMD_PARSER_H_ + +#include +#include +#include +#include +#include + +#include "../../rados-mail.h" +#include "../../rados-util.h" + +#ifndef PATH_MAX +#define PATH_MAX 256 +#endif + +namespace storage_interface { + +class Predicate{ + public: + virtual ~Predicate(){} + virtual std::string& get_key() = 0; + virtual std::string& get_op() = 0; + virtual std::string& get_value() = 0; + virtual bool& get_valid() = 0; + + virtual bool eval(const std::string &_p_value) = 0; + virtual bool convert_str_to_time_t(const std::string &date, time_t *val) = 0; + virtual int convert_time_t_to_str(const time_t &t, std::string *ret_val) = 0; +}; + +class CmdLineParser { + public: + virtual ~CmdLineParser(){} + virtual bool parse_ls_string() = 0; + virtual std::map &get_predicates() = 0; + + virtual bool contains_key(const std::string &key) = 0; + virtual Predicate *get_predicate(const std::string &key) = 0; + virtual Predicate *create_predicate(const std::string &ls_value) = 0; + + virtual void set_output_dir(const std::string &out) = 0; + virtual std::string &get_output_dir() = 0; +}; +}; // namespace storage_interface +#endif // SRC_INTERFACES_TOOLS_RMB_LS_CMD_PARSER_H_ diff --git a/src/storage-interface/tools/rmb/mailbox_tools.h b/src/storage-interface/tools/rmb/mailbox_tools.h new file mode 100644 index 00000000..93be6475 --- /dev/null +++ b/src/storage-interface/tools/rmb/mailbox_tools.h @@ -0,0 +1,36 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_TOOLS_RMB_MAILBOX_TOOLS_H_ +#define SRC_STORAGE_INTERFACES_TOOLS_RMB_MAILBOX_TOOLS_H_ + +#include + +#include "../../rados-mail.h" +#include "rados-mail-box.h" + +namespace storage_interface { +class MailboxTools { + public: + virtual ~MailboxTools() {} + + virtual int init_mailbox_dir() = 0; + virtual int save_mail(storage_interface::RadosMail* mail_obj) = 0; + virtual int delete_mailbox_dir() = 0; + virtual int delete_mail(storage_interface::RadosMail* mail_obj) = 0; + + virtual int build_filename(storage_interface::RadosMail* mail_obj, std::string* filename) = 0; + + virtual std::string& get_mailbox_path() = 0; +}; +}; // namespace storage_interface + +#endif // SRC_STORAGE_INTERFACES_TOOLS_RMB_MAILBOX_TOOLS_H_ diff --git a/src/storage-interface/tools/rmb/rados-mail-box.h b/src/storage-interface/tools/rmb/rados-mail-box.h new file mode 100644 index 00000000..45905362 --- /dev/null +++ b/src/storage-interface/tools/rmb/rados-mail-box.h @@ -0,0 +1,43 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_TOOLS_RMB_RADOS_MAIL_BOX_H_ +#define SRC_STORAGE_INTERFACES_TOOLS_RMB_RADOS_MAIL_BOX_H_ + +#include +#include +#include +#include + +#include "../../rados-mail.h" +#include "ls_cmd_parser.h" +#include "../../rados-util.h" +namespace storage_interface { + +class RadosMailBox { + public: + virtual ~RadosMailBox() {} + + virtual void add_mail(storage_interface::RadosMail *mail) = 0; + virtual void add_to_mailbox_size(const uint64_t &_mailbox_size) = 0; + virtual void set_mails(const std::list &_mails) = 0; + + virtual storage_interface::CmdLineParser *get_xattr_filter() = 0; + virtual void set_xattr_filter(storage_interface::CmdLineParser *_parser) = 0; + virtual std::list &get_mails() = 0; + virtual std::string &get_mailbox_guid() = 0; + virtual void set_mailbox_guid(const std::string &_mailbox_guid) = 0; + virtual void set_mailbox_orig_name(const std::string &_mbox_orig_name) = 0; + virtual int &get_mail_count() = 0; +}; +} // namespace storage_interface + +#endif // SRC_STORAGE_INTERFACES_TOOLS_RMB_RADOS_MAIL_BOX_H_ diff --git a/src/storage-interface/tools/rmb/rmb-commands.h b/src/storage-interface/tools/rmb/rmb-commands.h new file mode 100644 index 00000000..aa92a39e --- /dev/null +++ b/src/storage-interface/tools/rmb/rmb-commands.h @@ -0,0 +1,68 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Copyright (c) 2017-2018 Tallence AG and the authors + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef SRC_STORAGE_INTERFACES_TOOLS_RMB_RMB_COMMANDS_H_ +#define SRC_STORAGE_INTERFACES_TOOLS_RMB_RMB_COMMANDS_H_ +#include + +#include +#include +#include +#include +#include +#include + +#include "../../rados-storage.h" +#include "../../rados-cluster.h" +#include "../../rados-metadata-storage.h" +#include "../../rados-dovecot-ceph-cfg.h" +#include "../../rados-ceph-config.h" +#include "ls_cmd_parser.h" +#include "mailbox_tools.h" +#include "../../rados-metadata-storage-module.h" +#include "../../rados-save-log.h" + +namespace storage_interface { + +class RmbCommands { + public: + virtual ~RmbCommands(){} + + virtual int delete_with_save_log(const std::string &save_log, const std::string &rados_cluster, + const std::string &rados_user, + std::map> *moved_items) = 0; + virtual void print_debug(const std::string &msg) = 0; + virtual int lspools() = 0; + virtual int delete_mail(bool confirmed) = 0; + virtual int delete_namespace(storage_interface::RadosStorageMetadataModule *ms, std::list &mail_objects, + storage_interface::RadosCephConfig *cfg, bool confirmed) = 0; + + virtual int rename_user(storage_interface::RadosCephConfig *cfg, bool confirmed, const std::string &uid) = 0; + + virtual int configuration(bool confirmed, storage_interface::RadosCephConfig *ceph_cfg) = 0; + + virtual int load_objects(storage_interface::RadosStorageMetadataModule *ms, std::list &mail_objects, + std::string &sort_string, bool load_metadata = true) = 0; + virtual int update_attributes(storage_interface::RadosStorageMetadataModule *ms, std::map *metadata) = 0; + virtual int print_mail(std::map *mailbox, std::string &output_dir, bool download) = 0; + virtual int query_mail_storage(std::list *mail_objects, storage_interface::CmdLineParser *parser, bool download, + bool silent) = 0; + virtual storage_interface::RadosStorageMetadataModule *init_metadata_storage_module(storage_interface::RadosCephConfig *ceph_cfg, std::string *uid) = 0; + virtual void set_output_path(storage_interface::CmdLineParser *parser) = 0; + virtual int overwrite_ceph_object_index(std::set &mail_oids) = 0; + virtual std::set load_objects(storage_interface::RadosStorageMetadataModule *ms) = 0; + virtual int remove_ceph_object_index() = 0; + virtual int append_ceph_object_index(const std::set &mail_oids) = 0; +}; + +} /* namespace storage_interface */ + +#endif /* SRC_STORAGE_INTERFACES_TOOLS_RMB_RMB_COMMANDS_H_ */ diff --git a/src/storage-rbox/Makefile.am b/src/storage-rbox/Makefile.am index 1a770d3c..8746cbe8 100644 --- a/src/storage-rbox/Makefile.am +++ b/src/storage-rbox/Makefile.am @@ -13,9 +13,9 @@ shlibs = \ $(top_builddir)/src/librmb/librmb.la rmbtoollibs = \ - $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o \ - $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o \ - $(top_builddir)/src/librmb/tools/rmb/rmb-commands.o + $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser_impl.o \ + $(top_builddir)/src/librmb/tools/rmb/mailbox_tools_impl.o \ + $(top_builddir)/src/librmb/tools/rmb/rmb-commands-impl.o libstorage_rbox_plugin_la_CPPFLAGS = \ $(LIBDOVECOT_INCLUDE) \ diff --git a/src/storage-rbox/doveadm-rbox-plugin.cpp b/src/storage-rbox/doveadm-rbox-plugin.cpp index e9d585c7..686b5961 100644 --- a/src/storage-rbox/doveadm-rbox-plugin.cpp +++ b/src/storage-rbox/doveadm-rbox-plugin.cpp @@ -47,40 +47,44 @@ extern "C" { #include "config-local.h" } -#include "tools/rmb/rmb-commands.h" -#include "rados-cluster.h" -#include "rados-cluster-impl.h" -#include "rados-storage.h" -#include "rados-storage-impl.h" -#include "rados-dovecot-ceph-cfg.h" -#include "rados-dovecot-ceph-cfg-impl.h" -#include "rados-namespace-manager.h" +#include "../storage-interface/tools/rmb/rmb-commands.h" +#include "../storage-interface/rados-cluster.h" +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../storage-interface/rados-namespace-manager.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" #include "rbox-storage.h" #include "rbox-save.h" #include "rbox-storage.hpp" +#include "../storage-engine/storage-backend-factory.h" +#include "../storage-interface/rados-save-log.h" -int check_namespace_mailboxes(const struct mail_namespace *ns, const std::list &mail_objects); +int check_namespace_mailboxes(const struct mail_namespace *ns, const std::list &mail_objects); static int iterate_list_objects(struct mail_namespace* ns, const struct mailbox_info *info, std::set &object_list); class RboxDoveadmPlugin { public: RboxDoveadmPlugin() { - this->cluster = new librmb::RadosClusterImpl(); - this->storage = new librmb::RadosStorageImpl(cluster); - this->config = new librmb::RadosDovecotCephCfgImpl(&storage->get_io_ctx()); + this->cluster =storage_engine::StorageBackendFactory::create_cluster(storage_engine::CEPH); + this->storage = storage_engine::StorageBackendFactory::create_storage(storage_engine::CEPH,this->cluster); + this->config = storage_engine::StorageBackendFactory::create_dovecot_ceph_cfg_io( + storage_engine::CEPH,this->storage->get_io_ctx_wrapper()); } ~RboxDoveadmPlugin() { if (config != nullptr) { delete config; + config=nullptr; } if (storage != nullptr) { storage->close_connection(); delete storage; + storage=nullptr; } if (cluster != nullptr) { cluster->deinit(); delete cluster; + cluster=nullptr; } } @@ -122,9 +126,9 @@ class RboxDoveadmPlugin { } public: - librmb::RadosCluster *cluster; - librmb::RadosStorage *storage; - librmb::RadosDovecotCephCfg *config; + storage_interface::RadosCluster *cluster=nullptr; + storage_interface::RadosStorage *storage=nullptr; + storage_interface::RadosDovecotCephCfg *config=nullptr; }; static int open_connection_load_config(RboxDoveadmPlugin *plugin) { @@ -133,7 +137,7 @@ static int open_connection_load_config(RboxDoveadmPlugin *plugin) { if (open < 0) { return open; } - librmb::RadosCephConfig *cfg = (static_cast(plugin->config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); int ret = cfg->load_cfg(); if (ret < 0) { i_error("Error accessing configuration. Errorcode: %d", ret); @@ -142,49 +146,63 @@ static int open_connection_load_config(RboxDoveadmPlugin *plugin) { return ret; } static int cmd_rmb_config(std::map &opts) { - RboxDoveadmPlugin plugin; - plugin.read_doveadm_plugin_configuration(); - int open = open_connection_load_config(&plugin); + RboxDoveadmPlugin *plugin=new RboxDoveadmPlugin(); + plugin->read_doveadm_plugin_configuration(); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); + delete plugin; return open; } - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); - int ret = rmb_cmds.configuration(true, *cfg); + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH, plugin->storage, plugin->cluster, &opts); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); + int ret = rmb_cmds->configuration(true, cfg); if (ret < 0) { i_error("Error processing ceph configuration. Errorcode: %d", ret); + delete rmb_cmds; + delete plugin; return -1; } + delete rmb_cmds; + delete plugin; return 0; } static int cmd_rmb_search_run(std::map &opts, struct mail_user *user, bool download, - librmb::CmdLineParser &parser, std::list &mail_objects, bool silent, + storage_interface::CmdLineParser *parser, std::list &mail_objects, bool silent, bool load_metadata = true) { - RboxDoveadmPlugin plugin; - int open = open_connection_load_config(&plugin); + RboxDoveadmPlugin *plugin=new RboxDoveadmPlugin(); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); + delete plugin; return open; } opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH, plugin->storage, plugin->cluster, &opts); std::string uid; - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); - librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_cmds->init_metadata_storage_module(cfg, &uid); if (ms == nullptr) { i_error(" Error initializing metadata module"); delete ms; + delete rmb_cmds; + delete plugin; return -1; } - int ret = rmb_cmds.load_objects(ms, mail_objects, opts["sort"], load_metadata); + int ret = rmb_cmds->load_objects(ms, mail_objects, opts["sort"], load_metadata); if (ret < 0) { i_error("Error loading ceph objects. Errorcode: %d", ret); + delete rmb_cmds; delete ms; + delete plugin; return ret; } if (user->namespaces != NULL) { @@ -194,16 +212,18 @@ static int cmd_rmb_search_run(std::map &opts, struct m } } if (download) { - rmb_cmds.set_output_path(&parser); + rmb_cmds->set_output_path(parser); } if (load_metadata) { - ret = rmb_cmds.query_mail_storage(&mail_objects, &parser, download, silent); + ret = rmb_cmds->query_mail_storage(&mail_objects, parser, download, silent); if (ret < 0) { i_error("Error query mail storage. Errorcode: %d", ret); } } + delete rmb_cmds; delete ms; - + delete parser; + delete plugin; return ret; } @@ -220,15 +240,16 @@ static int cmd_rmb_ls_run(struct doveadm_mail_cmd_context *ctx, struct mail_user std::map opts; opts["ls"] = search_query; opts["sort"] = sort != NULL ? sort : "uid"; - librmb::CmdLineParser parser(opts["ls"]); - - if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { - std::list mail_objects; + storage_interface::CmdLineParser *parser=storage_engine::StorageBackendFactory::create_cmd_line_parser( + storage_engine::CEPH,opts["ls"]); + + if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser->parse_ls_string()) { + std::list mail_objects; ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(), - [](librmb::RadosMail *n) -> bool { return n->is_index_ref() == false; }); + [](storage_interface::RadosMail *n) -> bool { return n->is_index_ref() == false; }); if (it_mail != mail_objects.end()) { std::cout << "There are unreferenced objects " << std::endl; @@ -247,9 +268,11 @@ static int cmd_rmb_ls_mb_run(struct doveadm_mail_cmd_context *ctx, struct mail_u std::map opts; opts["ls"] = "mb"; opts["sort"] = "uid"; - librmb::CmdLineParser parser(opts["ls"]); - if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { - std::list mail_objects; + storage_interface::CmdLineParser *parser=storage_engine::StorageBackendFactory::create_cmd_line_parser( + storage_engine::CEPH,opts["ls"]); + + if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser->parse_ls_string()) { + std::list mail_objects; ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); } else { i_error("invalid ls search query"); @@ -274,10 +297,11 @@ static int cmd_rmb_get_run(struct doveadm_mail_cmd_context *ctx, struct mail_use opts["out"] = output_path; } opts["sort"] = "uid"; - - librmb::CmdLineParser parser(opts["get"]); - if (opts["get"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { - std::list mail_objects; + storage_interface::CmdLineParser *parser=storage_engine::StorageBackendFactory::create_cmd_line_parser( + storage_engine::CEPH,opts["get"]); + + if (opts["get"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser->parse_ls_string()) { + std::list mail_objects; ctx->exit_code = cmd_rmb_search_run(opts, user, true, parser, mail_objects, false); for (auto mo : mail_objects) { delete mo; @@ -305,11 +329,12 @@ static int cmd_rmb_set_run(struct doveadm_mail_cmd_context *ctx, struct mail_use return 0; } - RboxDoveadmPlugin plugin; - int open = open_connection_load_config(&plugin); + RboxDoveadmPlugin *plugin= new RboxDoveadmPlugin(); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("error opening rados connection, check config: %d", open); ctx->exit_code = open; + delete plugin; return 0; } @@ -317,15 +342,19 @@ static int cmd_rmb_set_run(struct doveadm_mail_cmd_context *ctx, struct mail_use opts["set"] = oid; opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH, plugin->storage, plugin->cluster, &opts); - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); std::string uid; - librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_cmds->init_metadata_storage_module(cfg, &uid); if (ms == nullptr) { i_error(" Error initializing metadata module "); delete ms; + delete rmb_cmds; + delete plugin; ctx->exit_code = -1; return 0; } @@ -342,7 +371,7 @@ static int cmd_rmb_set_run(struct doveadm_mail_cmd_context *ctx, struct mail_use std::cerr << "check key " << tokens[0] << " is not a valid attribute" << std::endl; } else { metadata[tokens[0]] = tokens[1]; - ctx->exit_code = rmb_cmds.update_attributes(ms, &metadata); + ctx->exit_code = rmb_cmds->update_attributes(ms, &metadata); std::cerr << " token " << tokens[0] << " : " << tokens[1] << std::endl; } } else { @@ -350,6 +379,8 @@ static int cmd_rmb_set_run(struct doveadm_mail_cmd_context *ctx, struct mail_use ctx->exit_code = -1; } delete ms; + delete rmb_cmds; + delete plugin; return 0; } @@ -367,11 +398,12 @@ static int cmd_rmb_delete_run(struct doveadm_mail_cmd_context *ctx, struct mail_ return 0; } - RboxDoveadmPlugin plugin; - int open = open_connection_load_config(&plugin); + RboxDoveadmPlugin *plugin=new RboxDoveadmPlugin(); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); ctx->exit_code = open; + delete plugin; return 0; } @@ -379,22 +411,28 @@ static int cmd_rmb_delete_run(struct doveadm_mail_cmd_context *ctx, struct mail_ opts["to_delete"] = oid; opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH, plugin->storage, plugin->cluster, &opts); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); std::string uid; - librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_cmds->init_metadata_storage_module(cfg, &uid); if (ms == nullptr) { i_error(" Error initializing metadata module "); delete ms; + delete rmb_cmds; + delete plugin; ctx->exit_code = -1; return 0; } - ctx->exit_code = rmb_cmds.delete_mail(true); + ctx->exit_code = rmb_cmds->delete_mail(true); if (ctx->exit_code < 0) { i_error("Error deleting mail. Errorcode: %d", ctx->exit_code); } delete ms; + delete rmb_cmds; + delete plugin; return 0; } static int cmd_rmb_rename_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) { @@ -411,11 +449,12 @@ static int cmd_rmb_rename_run(struct doveadm_mail_cmd_context *ctx, struct mail_ return 0; } - RboxDoveadmPlugin plugin; - int open = open_connection_load_config(&plugin); + RboxDoveadmPlugin *plugin=new RboxDoveadmPlugin(); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); ctx->exit_code = open; + delete plugin; return 0; } @@ -423,15 +462,18 @@ static int cmd_rmb_rename_run(struct doveadm_mail_cmd_context *ctx, struct mail_ opts["to_rename"] = new_user_name; opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH, plugin->storage, plugin->cluster, &opts); std::string uid; - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); - ctx->exit_code = rmb_cmds.rename_user(cfg, true, user->username); + ctx->exit_code = rmb_cmds->rename_user(cfg, true, user->username); if (ctx->exit_code < 0) { i_error("Error renaming user. Errorcode: %d", ctx->exit_code); } - + delete rmb_cmds; + delete plugin; return 0; } static int restore_index_entry(struct mail_user *user, const char *mailbox_name, const std::string &str_mail_guid, @@ -501,7 +543,7 @@ static int doveadm_rmb_mail_next_user(struct doveadm_mail_cmd_context *ctx, const struct mail_storage_service_input *input, struct mail_storage_service_user *cur_service_user, struct mail_user **cur_mail_user, const char **error_r, - std::list *entries) { + std::list *entries) { const char *error, *ip; int ret; @@ -541,20 +583,25 @@ static int doveadm_rmb_mail_next_user(struct doveadm_mail_cmd_context *ctx, #endif return ret; } - for (std::list::iterator it = entries->begin(); it != entries->end(); ++it) { + for (std::list::iterator it = entries->begin(); it != entries->end(); ++it) { // do something here! - std::string key_guid(1, static_cast(librmb::RBOX_METADATA_GUID)); - std::string key_mbox_name(1, static_cast(librmb::RBOX_METADATA_ORIG_MAILBOX)); + std::string key_guid(1, static_cast(storage_interface::RBOX_METADATA_GUID)); + std::string key_mbox_name(1, static_cast(storage_interface::RBOX_METADATA_ORIG_MAILBOX)); - std::list::iterator it_guid = - std::find_if(it->metadata.begin(), it->metadata.end(), - [key_guid](librmb::RadosMetadata const &m) { return m.key == key_guid; }); - std::list::iterator it_mb = - std::find_if(it->metadata.begin(), it->metadata.end(), - [key_mbox_name](librmb::RadosMetadata const &m) { return m.key == key_mbox_name; }); + std::list::iterator it_guid = + std::find_if((*it)->get_metadata().begin(), (*it)->get_metadata().end(), + [key_guid](storage_interface::RadosMetadata* const m) { return m->get_key() == key_guid; }); + std::list::iterator it_mb = + std::find_if((*it)->get_metadata().begin(), (*it)->get_metadata().end(), + [key_mbox_name](storage_interface::RadosMetadata* const m) { return m->get_key() == key_mbox_name; }); - restore_index_entry(*cur_mail_user, (*it_mb).bl.to_str().c_str(), (*it_guid).bl.to_str(), it->src_oid); + restore_index_entry(*cur_mail_user, + (storage_engine::StorageObjectFactory::get_buffer(storage_engine::CEPH,(*it_mb)->get_buffer()))->c_str(), + (storage_engine::StorageObjectFactory::get_buffer(storage_engine::CEPH,(*it_guid)->get_buffer()))->to_str(), + (*it)->get_src_oid()); + delete (*it); + (*it)=nullptr; } mail_user_unref(cur_mail_user); @@ -581,21 +628,23 @@ static int cmd_rmb_revert_log_run(struct doveadm_mail_cmd_context *ctx, struct m return 0; } - RboxDoveadmPlugin plugin; - plugin.read_plugin_configuration(user); - int open = plugin.open_connection(); + RboxDoveadmPlugin *plugin = new RboxDoveadmPlugin(); + plugin->read_plugin_configuration(user); + int open = plugin->open_connection(); if (open < 0) { i_error("error opening rados connection, check config: %d", open); ctx->exit_code = open; return 0; } - std::map> moved_items; - ctx->exit_code = librmb::RmbCommands::delete_with_save_log(log_file, plugin.config->get_rados_cluster_name(), - plugin.config->get_rados_username(), &moved_items) >= 0 + std::map> moved_items; + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands_default(storage_engine::CEPH); + ctx->exit_code = rmb_cmds->delete_with_save_log(log_file, plugin->config->get_rados_cluster_name(), + plugin->config->get_rados_username(), &moved_items) >= 0 ? 0 : -1; - for (std::map>::iterator iter = moved_items.begin(); + for (std::map>::iterator iter = moved_items.begin(); iter != moved_items.end(); ++iter) { const char *error; @@ -606,11 +655,13 @@ static int cmd_rmb_revert_log_run(struct doveadm_mail_cmd_context *ctx, struct m doveadm_rmb_mail_next_user(ctx, &ctx->storage_service_input, cur_service_user, &cur_mail_user, &error, &iter->second); } + delete rmb_cmds; + delete plugin; return 0; } static int iterate_mailbox(const struct mail_namespace *ns, const struct mailbox_info *info, - const std::list &mail_objects) { + const std::list &mail_objects) { int ret = 0; struct mailbox_transaction_context *mailbox_transaction; struct mail_search_context *search_ctx; @@ -665,7 +716,7 @@ static int iterate_mailbox(const struct mail_namespace *ns, const struct mailbox std::string oid = guid_128_to_string(obox_rec->oid); auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(), - [oid](librmb::RadosMail *m) { return m->get_oid()->compare(oid) == 0; }); + [oid](storage_interface::RadosMail *m) { return m->get_oid()->compare(oid) == 0; }); if (it_mail == mail_objects.end()) { std::cout << " missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid @@ -693,7 +744,7 @@ static int iterate_mailbox(const struct mail_namespace *ns, const struct mailbox return ret; } -int check_namespace_mailboxes(const struct mail_namespace *ns, const std::list &mail_objects) { +int check_namespace_mailboxes(const struct mail_namespace *ns, const std::list &mail_objects) { struct mailbox_list_iterate_context *iter; const struct mailbox_info *info; int ret = 0; @@ -719,16 +770,18 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc std::map opts; opts["ls"] = "-"; // search all objects opts["sort"] = "uid"; - librmb::CmdLineParser parser(opts["ls"]); - parser.parse_ls_string(); - std::list mail_objects; + storage_interface::CmdLineParser *parser=storage_engine::StorageBackendFactory::create_cmd_line_parser( + storage_engine::CEPH,opts["ls"]); + + parser->parse_ls_string(); + std::list mail_objects; ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, true, false); if (ctx->exit_code < 0) { return 0; } auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(), - [](librmb::RadosMail *m) { return m->is_index_ref() == false; }); + [](storage_interface::RadosMail *m) { return m->is_index_ref() == false; }); if (it_mail != mail_objects.end()) { std::cout << std::endl << "There are mail objects without a index reference: " << std::endl; @@ -739,24 +792,29 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc ctx->exit_code = 1; } - RboxDoveadmPlugin plugin; + RboxDoveadmPlugin *plugin =new RboxDoveadmPlugin(); - int open = open_connection_load_config(&plugin); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("Error open connection to cluster %d", open); ctx->exit_code = open; + delete plugin; return 0; } opts["namespace"] = user->username; - - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH,plugin->storage, plugin->cluster, &opts); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); std::string uid; - librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_cmds->init_metadata_storage_module(cfg, &uid); if (ms == nullptr) { i_error(" Error initializing metadata module "); + delete rmb_cmds; delete ms; + delete plugin; ctx->exit_code = -1; return 0; } @@ -765,13 +823,14 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc std::cout << mo->to_string(" ") << std::endl; if (open >= 0 && ctx_->delete_not_referenced_objects && !mo->is_index_ref()) { std::cout << "mail object: " << mo->get_oid()->c_str() - << " deleted: " << (plugin.storage->delete_mail(mo) < 0 ? " FALSE " : " TRUE") << std::endl; + << " deleted: " << (plugin->storage->delete_mail(*mo->get_oid()) < 0 ? " FALSE " : " TRUE") << std::endl; ctx->exit_code = 2; } delete mo; } delete ms; - + delete rmb_cmds; + delete plugin; return 0; } @@ -780,31 +839,37 @@ static int cmd_rmb_create_ceph_index_run(struct doveadm_mail_cmd_context *_ctx, struct create_ceph_index_cmd_context *ctx = (struct create_ceph_index_cmd_context *)_ctx; std::set mail_objects; - RboxDoveadmPlugin plugin; - plugin.read_plugin_configuration(user); + RboxDoveadmPlugin *plugin=new RboxDoveadmPlugin(); + plugin->read_plugin_configuration(user); - int open = open_connection_load_config(&plugin); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); _ctx->exit_code = open; + delete plugin; return open; } i_info("connection to rados open"); std::map opts; opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); + + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands( + storage_engine::CEPH, plugin->storage, plugin->cluster, &opts); std::string uid; - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); - librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_cmds->init_metadata_storage_module(cfg, &uid); if (ms == nullptr) { i_error(" Error initializing metadata module"); delete ms; + delete rmb_cmds; + delete plugin; _ctx->exit_code = -1; return -1; } - if(rmb_cmds.remove_ceph_object_index() < 0){ + if(rmb_cmds->remove_ceph_object_index() < 0){ i_debug(" Error overwriting ceph object index"); } @@ -830,9 +895,11 @@ static int cmd_rmb_create_ceph_index_run(struct doveadm_mail_cmd_context *_ctx, //append to index. i_info("found %s mails in namespace %d",info->vname, mail_objects.size()); - if(rmb_cmds.append_ceph_object_index(mail_objects) < 0){ + if(rmb_cmds->append_ceph_object_index(mail_objects) < 0){ i_error(" Error overwriting ceph object index"); delete ms; + delete rmb_cmds; + delete plugin; _ctx->exit_code = -1; return -1; } @@ -842,10 +909,12 @@ static int cmd_rmb_create_ceph_index_run(struct doveadm_mail_cmd_context *_ctx, } // end of for }else{ - mail_objects = rmb_cmds.load_objects(ms); - if(rmb_cmds.overwrite_ceph_object_index(mail_objects) < 0){ + mail_objects = rmb_cmds->load_objects(ms); + if(rmb_cmds->overwrite_ceph_object_index(mail_objects) < 0){ i_error(" Error overwriting ceph object index"); delete ms; + delete rmb_cmds; + delete plugin; return -1; } i_info("found %d mails in namespace",mail_objects.size()); @@ -854,7 +923,8 @@ static int cmd_rmb_create_ceph_index_run(struct doveadm_mail_cmd_context *_ctx, } i_info("index created"); _ctx->exit_code = ret; - + delete rmb_cmds; + delete plugin; return ret; } static int iterate_list_objects(struct mail_namespace* ns, const struct mailbox_info *info, std::set &object_list){ @@ -1002,29 +1072,35 @@ static int cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx, struct static int cmd_rmb_mailbox_delete_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) { int ret = cmd_mailbox_delete_run(ctx, user); if (ret == 0 && ctx->exit_code == 0) { - RboxDoveadmPlugin plugin; + RboxDoveadmPlugin *plugin = new RboxDoveadmPlugin(); i_debug("cleaning up rbox specific files and objects :ret=%d", ret); - plugin.read_plugin_configuration(user); - int open = open_connection_load_config(&plugin); + plugin->read_plugin_configuration(user); + int open = open_connection_load_config(plugin); if (open < 0) { i_error("error opening rados connection, check config: %d", open); ctx->exit_code = open; + delete plugin; return 0; } std::map opts; opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); std::string uid; - librmb::RadosCephConfig *cfg = - (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = + plugin->config->get_rados_ceph_cfg(); if (cfg->is_user_mapping()) { // we need to delete the namespace object. // iterate over all mailboxes, if we have no more mails, delete the user namespace object // for the current user. - librmb::RadosNamespaceManager mgr(plugin.config); - check_users_mailbox_delete_ns_object(user, plugin.config, &mgr, plugin.storage); + storage_interface::RadosNamespaceManager *mgr= + storage_engine::StorageBackendFactory::create_namespace_manager( + storage_engine::CEPH,plugin->config); + check_users_mailbox_delete_ns_object(user, plugin->config, mgr, plugin->storage); + delete mgr; + mgr=nullptr; + delete plugin; + plugin=nullptr; } } return 0; @@ -1236,26 +1312,30 @@ int cmd_rmb_config_show(int argc, char *argv[]) { return cmd_rmb_config(opts); } int cmd_rmb_config_create(int argc, char *argv[]) { - RboxDoveadmPlugin plugin; - plugin.read_doveadm_plugin_configuration(); - int open = plugin.open_connection(); + RboxDoveadmPlugin *plugin= new RboxDoveadmPlugin(); + plugin->read_doveadm_plugin_configuration(); + int open = plugin->open_connection(); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); + delete plugin; return -1; } - librmb::RadosCephConfig *cfg = (static_cast(plugin.config))->get_rados_ceph_cfg(); + storage_interface::RadosCephConfig *cfg = plugin->config->get_rados_ceph_cfg(); int ret = cfg->load_cfg(); if (ret < 0) { ret = cfg->save_cfg(); if (ret < 0) { i_error("error creating configuration %d", ret); + delete plugin; return -1; } std::cout << "config created" << std::endl; } else { std::cout << "config already exist" << std::endl; + delete plugin; return 1; } + delete plugin; return 0; } @@ -1275,7 +1355,14 @@ int cmd_rmb_config_update(int argc, char *argv[]) { return ret; } -int cmd_rmb_lspools(int argc, char *argv[]) { return librmb::RmbCommands::RmbCommands::lspools(); } +int cmd_rmb_lspools(int argc, char *argv[]) { + storage_interface::RmbCommands *rmb_cmds= + storage_engine::StorageBackendFactory::create_rmb_commands_default( + storage_engine::CEPH); + int ret=rmb_cmds->lspools(); + delete rmb_cmds; + return ret; +} int cmd_rmb_version(int argc, char *argv[]) { std::cout << "Plugin version:: " << PACKAGE_VERSION << std::endl; return 0; diff --git a/src/storage-rbox/istream-bufferlist.cpp b/src/storage-rbox/istream-bufferlist.cpp index a6c143f4..a80fc0b1 100644 --- a/src/storage-rbox/istream-bufferlist.cpp +++ b/src/storage-rbox/istream-bufferlist.cpp @@ -17,10 +17,10 @@ extern "C" { #include "istream-bufferlist.h" #include - +#include struct bufferlist_istream { struct istream_private istream; - librados::bufferlist *bl; + void *bl; }; static ssize_t i_stream_data_read(struct istream_private *stream) { stream->istream.eof = TRUE; // all in! @@ -38,12 +38,12 @@ static void rbox_istream_destroy(struct iostream_private *stream) { struct bufferlist_istream *bstream = (struct bufferlist_istream *)stream; delete bstream->bl; } -struct istream *i_stream_create_from_bufferlist(librados::bufferlist *data, const size_t &size) { +struct istream *i_stream_create_from_bufferlist(void *data,const char *mail_buff, const size_t &size) { struct bufferlist_istream *bstream; bstream = i_new(struct bufferlist_istream, 1); // use unsigned char* for binary data! - bstream->istream.buffer = reinterpret_cast(data->c_str()); + bstream->istream.buffer = reinterpret_cast(mail_buff); bstream->istream.pos = size; bstream->istream.max_buffer_size = (size_t)-1; diff --git a/src/storage-rbox/istream-bufferlist.h b/src/storage-rbox/istream-bufferlist.h index 85e5920a..89e2d8ba 100644 --- a/src/storage-rbox/istream-bufferlist.h +++ b/src/storage-rbox/istream-bufferlist.h @@ -19,6 +19,6 @@ * @param[in] data valid pointer to bufferlist which is avail while istream is avail. * @param[in] size size of initial buffer. */ -struct istream *i_stream_create_from_bufferlist(librados::bufferlist *data, const size_t &size); +struct istream *i_stream_create_from_bufferlist(void *data,const char *mail_buff, const size_t &size); #endif /* SRC_STORAGE_RBOX_ISTREAM_BUFFERLIST_H_ */ diff --git a/src/storage-rbox/ostream-bufferlist.cpp b/src/storage-rbox/ostream-bufferlist.cpp index 83f4880e..63196f0f 100644 --- a/src/storage-rbox/ostream-bufferlist.cpp +++ b/src/storage-rbox/ostream-bufferlist.cpp @@ -20,10 +20,10 @@ extern "C" { struct bufferlist_ostream { struct ostream_private ostream; - librados::bufferlist *buf; + void *buf; bool seeked; - librmb::RadosStorage *rados_storage; - librmb::RadosMail *rados_mail; + storage_interface::RadosStorage *rados_storage; + storage_interface::RadosMail *rados_mail; bool execute_write_ops; }; @@ -59,26 +59,17 @@ static ssize_t o_stream_buffer_sendv(struct ostream_private *stream, const struc unsigned int i; uint64_t val = stream->ostream.offset; if (bstream->execute_write_ops) { - bstream->buf->clear(); + bstream->rados_storage->free_mail_buffer(bstream->buf); } for (i = 0; i < iov_count; i++) { - // use unsigned char* for binary data! - bstream->buf->append(reinterpret_cast(iov[i].iov_base), iov[i].iov_len); + bstream->rados_storage->append_to_buffer(bstream->buf,reinterpret_cast(iov[i].iov_base), iov[i].iov_len); stream->ostream.offset += iov[i].iov_len; ret += iov[i].iov_len; } - - if (bstream->execute_write_ops) { - librados::ObjectWriteOperation write_op; - write_op.write(val, *bstream->buf); - - bstream->rados_storage->aio_operate(&bstream->rados_storage->get_io_ctx(), *bstream->rados_mail->get_oid(), - bstream->rados_mail->get_completion(), &write_op); - } return ret; } -struct ostream *o_stream_create_bufferlist(librmb::RadosMail *rados_mail, librmb::RadosStorage *rados_storage, +struct ostream *o_stream_create_bufferlist(storage_interface::RadosMail *rados_mail, storage_interface::RadosStorage *rados_storage, bool execute_write_ops) { struct bufferlist_ostream *bstream; struct ostream *output; @@ -98,10 +89,6 @@ struct ostream *o_stream_create_bufferlist(librmb::RadosMail *rados_mail, librmb bstream->rados_storage = rados_storage; bstream->rados_mail = rados_mail; bstream->execute_write_ops = execute_write_ops; - if (execute_write_ops) { - rados_mail->set_completion(librados::Rados::aio_create_completion()); - rados_mail->set_active_op(1); - } output = o_stream_create(&bstream->ostream, NULL, -1); o_stream_set_name(output, "(buffer)"); return output; diff --git a/src/storage-rbox/ostream-bufferlist.h b/src/storage-rbox/ostream-bufferlist.h index a46b3596..f8b2f58a 100644 --- a/src/storage-rbox/ostream-bufferlist.h +++ b/src/storage-rbox/ostream-bufferlist.h @@ -17,10 +17,10 @@ extern "C" { #include "ostream-private.h" } #include -#include "rados-storage.h" -#include "rados-mail.h" +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-mail.h" -struct ostream *o_stream_create_bufferlist(librmb::RadosMail *rados_mail, librmb::RadosStorage *rados_storage, +struct ostream *o_stream_create_bufferlist(storage_interface::RadosMail *rados_mail, storage_interface::RadosStorage *rados_storage, bool execute_write_ops); int o_stream_buffer_write_at(struct ostream_private *stream, const void *data, size_t size, uoff_t offset); #endif /* SRC_STORAGE_RBOX_OSTREAM_BUFFERLIST_H_ */ diff --git a/src/storage-rbox/rbox-copy.cpp b/src/storage-rbox/rbox-copy.cpp index 4cb8e976..3f25f750 100644 --- a/src/storage-rbox/rbox-copy.cpp +++ b/src/storage-rbox/rbox-copy.cpp @@ -23,18 +23,21 @@ extern "C" { #include "ostream-private.h" #include "debug-helper.h" } -#include "../librmb/rados-mail.h" +#include "../storage-interface/rados-mail.h" #include "rbox-storage.hpp" #include "rbox-mail.h" #include "rbox-save.h" #include "rbox-sync.h" #include "rbox-copy.h" -#include "rados-util.h" +#include "../storage-interface/rados-util.h" +#include "../storage-engine/storage-backend-factory.h" +#include "../storage-interface/rados-metadata.h" const char *SETTINGS_RBOX_UPDATE_IMMUTABLE = "rbox_update_immutable"; const char *SETTINGS_DEF_UPDATE_IMMUTABLE = "false"; -using librmb::rbox_metadata_key; +using storage_interface::rbox_metadata_key; +using storage_engine::StorageBackendFactory; int rbox_mail_storage_copy(struct mail_save_context *ctx, struct mail *mail); @@ -105,7 +108,7 @@ static int rbox_mail_save_copy_default_metadata(struct mail_save_context *ctx, s return 0; } -static void set_mailbox_metadata(struct mail_save_context *ctx, std::list *metadata_update) { +static void set_mailbox_metadata(struct mail_save_context *ctx, std::list *metadata_update) { { FUNC_START(); struct mailbox *dest_mbox = ctx->transaction->box; @@ -115,17 +118,22 @@ static void set_mailbox_metadata(struct mail_save_context *ctx, std::listmailbox_guid)); + storage_interface::RadosMetadata *metadata_mailbox_guid= + StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, guid_128_to_string(dest_rbox->mailbox_guid)); metadata_update->push_back(metadata_mailbox_guid); - librmb::RadosMetadata metadata_mbn(rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, ctx->data.received_date); + storage_interface::RadosMetadata *metadata_mbn= + StorageBackendFactory::create_metadata_time( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, ctx->data.received_date); metadata_update->push_back(metadata_mbn); if (r_storage->config->is_update_attributes()) { if (r_storage->config->is_updateable_attribute(rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX)) { // updates the plain text mailbox name - librmb::RadosMetadata metadata_mbn(rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, dest_rbox->box.name); + storage_interface::RadosMetadata *metadata_mbn= + StorageBackendFactory::create_metadata_char( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, dest_rbox->box.name); metadata_update->push_back(metadata_mbn); } } @@ -177,12 +185,12 @@ static void mail_copy_set_failed(struct mail_save_context *ctx, struct mail *mai errstr = mail_storage_get_last_error(mail->box->storage, &error); mail_storage_set_error(ctx->transaction->box->storage, error, t_strdup_printf("%s (%s)", errstr, func)); } -static int copy_mail(struct mail_save_context *ctx, librmb::RadosStorage *rados_storage, struct rbox_mail *rmail, +static int copy_mail(struct mail_save_context *ctx, storage_interface::RadosStorage *rados_storage, struct rbox_mail *rmail, const std::string *ns_src, const std::string *ns_dest) { struct rbox_save_context *r_ctx = (struct rbox_save_context *)ctx; struct rbox_storage *r_storage = (struct rbox_storage *)&r_ctx->mbox->storage->storage; - std::list metadata_update; + std::list metadata_update; std::string src_oid = *rmail->rados_mail->get_oid(); setup_mail_object(ctx); @@ -217,8 +225,15 @@ static int copy_mail(struct mail_save_context *ctx, librmb::RadosStorage *rados_ rbox_add_to_index(ctx); if (r_storage->save_log->is_open()) { - r_storage->save_log->append(librmb::RadosSaveLogEntry(dest_oid, *ns_dest, rados_storage->get_pool_name(), - librmb::RadosSaveLogEntry::op_cpy())); + storage_interface::RadosSaveLogEntry *save_log_entry= + storage_engine::StorageBackendFactory::create_save_log_entry_default(storage_engine::CEPH); + + r_storage->save_log->append(storage_engine::StorageBackendFactory::create_save_log_entry( + storage_engine::CEPH, + dest_oid, *ns_dest, rados_storage->get_pool_name(), + save_log_entry->op_cpy())); + delete save_log_entry; + save_log_entry=nullptr; } #ifdef DEBUG i_debug("copy successfully finished: from src %s to oid = %s", src_oid.c_str(), dest_oid.c_str()); @@ -261,7 +276,7 @@ static int rbox_mail_storage_try_copy(struct mail_save_context **_ctx, struct ma return -1; } - librmb::RadosStorage *rados_storage = !from_alt_storage ? r_storage->s : r_storage->alt; + storage_interface::RadosStorage *rados_storage = !from_alt_storage ? r_storage->s : r_storage->alt; if (copy_mail(ctx, rados_storage, rmail, &ns_src, &ns_dest) < 0) { FUNC_END_RET("ret == -1, copy mail failed"); diff --git a/src/storage-rbox/rbox-mail.cpp b/src/storage-rbox/rbox-mail.cpp index ed04d4b3..89565874 100644 --- a/src/storage-rbox/rbox-mail.cpp +++ b/src/storage-rbox/rbox-mail.cpp @@ -37,15 +37,17 @@ extern "C" { #endif } -#include "../librmb/rados-mail.h" +#include "../storage-interface/rados-mail.h" #include "rbox-storage.hpp" -#include "../librmb/rados-storage-impl.h" #include "istream-bufferlist.h" #include "rbox-mail.h" -#include "rados-util.h" +#include "../storage-interface/rados-util.h" +#include "../storage-engine/storage-backend-factory.h" +#include "../storage-interface/rados-metadata.h" +#include "../storage-interface/rados-storage.h" -using librmb::RadosMail; -using librmb::rbox_metadata_key; +using storage_interface::RadosMail; +using storage_interface::rbox_metadata_key; void rbox_mail_set_expunged(struct rbox_mail *mail) { FUNC_START(); @@ -132,9 +134,9 @@ static int rbox_mail_metadata_get(struct rbox_mail *rmail, enum rbox_metadata_ke // update metadata storage io_ctx and load metadata if (alt_storage) { - r_storage->ms->get_storage()->set_io_ctx(&r_storage->alt->get_io_ctx()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->alt->get_io_ctx_wrapper()); } else { - r_storage->ms->get_storage()->set_io_ctx(&r_storage->s->get_io_ctx()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->s->get_io_ctx_wrapper()); } /*#283: virtual mailbox needs this (different initialisation path)*/ @@ -149,12 +151,11 @@ static int rbox_mail_metadata_get(struct rbox_mail *rmail, enum rbox_metadata_ke } } if(rmail->rados_mail->get_oid()->length() == 0){ - i_info("mail uid: %d , oid '%s', guid: %s, index-oid: %s ",mail->uid,rmail->rados_mail->get_oid()->c_str(), guid_128_to_string(rmail->index_guid), guid_128_to_string(rmail->index_oid) ); rmail->rados_mail->set_oid(rmail->index_oid); } int ret_load_metadata = r_storage->ms->get_storage()->load_metadata(rmail->rados_mail); if (ret_load_metadata < 0) { - std::string metadata_key = librmb::rbox_metadata_key_to_char(key); + std::string metadata_key = storage_interface::rbox_metadata_key_to_char(key); if (ret_load_metadata == -ENOENT) { //i_debug("Errorcode: process %d returned with %d cannot get x_attr(%s,%c) from rados_object: %s",getpid(), ret_load_metadata, // metadata_key.c_str(), key, rmail->rados_mail != NULL ? rmail->rados_mail->to_string(" ").c_str() : " no rados_mail"); @@ -167,15 +168,28 @@ static int rbox_mail_metadata_get(struct rbox_mail *rmail, enum rbox_metadata_ke FUNC_END(); return -1; } - - // we need to copy the pointer. Because dovecots memory mgmnt will free it! +#ifdef DEBUG + for (std::map::iterator it = rmail->rados_mail->get_metadata()->begin(); + it != rmail->rados_mail->get_metadata()->end(); ++it) { + void* t = (void*) (*it).second; + ceph::bufferlist bl = *(ceph::bufferlist*)t; + char* key = (*it).first.c_str(); + i_debug("read: metadata: '%s'='%s'",key, bl.to_str().c_str()); + } +#endif char *val = NULL; - librmb::RadosUtils::get_metadata(key, rmail->rados_mail->get_metadata(), &val); + std::string metadata_key = storage_interface::rbox_metadata_key_to_char(key); + + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->get_metadata(metadata_key, rmail->rados_mail->get_metadata(), &val); if (val != NULL) { *value_r = i_strdup(val); } else { return -1; } + delete rados_utils; + rados_utils = nullptr; FUNC_END(); return 0; } @@ -207,7 +221,9 @@ int rbox_mail_get_received_date(struct mail *_mail, time_t *date_r) { return -1; } } - librmb::RadosUtils::get_metadata(rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, rmail->rados_mail->get_metadata(), + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->get_metadata(rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, rmail->rados_mail->get_metadata(), &value); if (value == NULL) { @@ -248,6 +264,9 @@ int rbox_mail_get_received_date(struct mail *_mail, time_t *date_r) { if (value != NULL && free_value) { i_free(value); } + + delete rados_utils; + rados_utils = nullptr; FUNC_END(); return ret; } @@ -290,7 +309,7 @@ static int rbox_mail_get_save_date(struct mail *_mail, time_t *date_r) { return -1; } - librmb::RadosStorage *rados_storage = alt_storage ? r_storage->alt : r_storage->s; + storage_interface::RadosStorage *rados_storage = alt_storage ? r_storage->alt : r_storage->s; int ret_val = rados_storage->stat_mail(*rmail->rados_mail->get_oid(), &object_size, &save_date_rados); if (ret_val < 0) { if (ret_val != -ENOENT) { @@ -325,7 +344,9 @@ int rbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) { FUNC_END_RET("ret == -1; mail_object == nullptr "); return -1; } - librmb::RadosUtils::get_metadata(rbox_metadata_key::RBOX_METADATA_VIRTUAL_SIZE, rmail->rados_mail->get_metadata(), + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->get_metadata(rbox_metadata_key::RBOX_METADATA_VIRTUAL_SIZE, rmail->rados_mail->get_metadata(), &value); if (value == NULL) { @@ -357,12 +378,12 @@ int rbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) { oid.c_str()); ret = -1; } - librmb::RadosMetadata metadata_phy(rbox_metadata_key::RBOX_METADATA_VIRTUAL_SIZE, data->virtual_size); - rmail->rados_mail->add_metadata(metadata_phy); - + if (value != NULL && free_value) { i_free(value); } + delete rados_utils; + rados_utils=nullptr; return ret; } @@ -384,8 +405,9 @@ static int rbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) { FUNC_END_RET("ret == -1; mail_object == nullptr "); return -1; } - - librmb::RadosUtils::get_metadata(rbox_metadata_key::RBOX_METADATA_PHYSICAL_SIZE, rmail->rados_mail->get_metadata(), + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->get_metadata(rbox_metadata_key::RBOX_METADATA_PHYSICAL_SIZE, rmail->rados_mail->get_metadata(), &value); if (value == NULL) { @@ -415,17 +437,18 @@ static int rbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) { if (value != NULL && free_value) { i_free(value); } - + delete rados_utils; + rados_utils = nullptr; FUNC_END(); return ret; } -static int get_mail_stream(struct rbox_mail *mail, librados::bufferlist *buffer, const size_t physical_size, +static int get_mail_stream(struct rbox_mail *mail,const char *mail_buff,const size_t physical_size, struct istream **stream_r) { struct mail_private *pmail = &mail->imail.mail; int ret = 0; - struct istream *input = i_stream_create_from_bufferlist(buffer, physical_size); + struct istream *input = i_stream_create_from_bufferlist(mail->rados_mail->get_mail_buffer(),mail_buff,physical_size); i_stream_seek(input, 0); *stream_r = input; @@ -438,32 +461,6 @@ static int get_mail_stream(struct rbox_mail *mail, librados::bufferlist *buffer, return ret; } - -static int read_mail_from_storage(librmb::RadosStorage *rados_storage, - struct rbox_mail *rmail, - uint64_t *psize, - time_t *save_date) { - - int stat_err = 0; - int read_err = 0; - - /* duplicate code: get_attribute */ - librados::ObjectReadOperation *read_mail = new librados::ObjectReadOperation(); - read_mail->read(0, INT_MAX, rmail->rados_mail->get_mail_buffer(), &read_err); - read_mail->stat(psize, save_date, &stat_err); - - //TODO: refactore to use operate instead of aio_operate. - librados::AioCompletion *completion = librados::Rados::aio_create_completion(); - int ret = rados_storage->get_io_ctx().aio_operate(*rmail->rados_mail->get_oid(), completion, read_mail, - rmail->rados_mail->get_mail_buffer()); - completion->wait_for_complete_and_cb(); - ret = completion->get_return_value(); - completion->release(); - delete read_mail; - - return ret; -} - static int rbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, struct message_size *hdr_size, struct message_size *body_size, struct istream **stream_r) { FUNC_START(); @@ -481,44 +478,24 @@ static int rbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, s return -1; } - librmb::RadosStorage *rados_storage = alt_storage ? ((struct rbox_storage *)_mail->box->storage)->alt + storage_interface::RadosStorage *rados_storage = alt_storage ? ((struct rbox_storage *)_mail->box->storage)->alt : ((struct rbox_storage *)_mail->box->storage)->s; if (alt_storage) { rados_storage->set_namespace(rados_storage->get_namespace()); } - + /* Pop3 and virtual box needs this. it looks like rbox_index_mail_set_seq is not called. */ if (rmail->rados_mail == nullptr) { - // make sure that mail_object is initialized, - // else create and load guid from index. - rmail->rados_mail = rados_storage->alloc_rados_mail(); - if (rbox_get_index_record(_mail) < 0) { - i_error("Error rbox_get_index uid(%d)", _mail->uid); - FUNC_END(); - return -1; - } - } - // create mail buffer! - rmail->rados_mail->set_mail_buffer(new librados::bufferlist()); - - uint64_t psize; - time_t save_date; - - ret = read_mail_from_storage(rados_storage, rmail,&psize,&save_date); - - /* duplicate code: get_attribute - librados::ObjectReadOperation *read_mail = new librados::ObjectReadOperation(); - read_mail->read(0, INT_MAX, rmail->rados_mail->get_mail_buffer(), &read_err); - read_mail->stat(&psize, &save_date, &stat_err); - - librados::AioCompletion *completion = librados::Rados::aio_create_completion(); - ret = rados_storage->get_io_ctx().aio_operate(*rmail->rados_mail->get_oid(), completion, read_mail, - rmail->rados_mail->get_mail_buffer()); - completion->wait_for_complete_and_cb(); - ret = completion->get_return_value(); - completion->release(); - delete read_mail; - */ + rmail->rados_mail = rados_storage->alloc_rados_mail(); + if (rbox_get_index_record(_mail) < 0) { + i_error("Error rbox_get_index uid(%d)", _mail->uid); + FUNC_END(); + return -1; + } + } + const std::string mail_oid=guid_128_to_string(rmail->index_oid); + ret=rados_storage->read_mail(mail_oid,rmail->rados_mail,0); + if (ret < 0) { if (ret == -ENOENT) { // This can happen, if we have more then 2 processes running at the same time. @@ -529,40 +506,19 @@ static int rbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, s rmail->rados_mail->get_oid()->c_str(), rados_storage->get_namespace().c_str(), getpid(), alt_storage); rbox_mail_set_expunged(rmail); FUNC_END_RET("ret == -1"); - delete rmail->rados_mail->get_mail_buffer(); + // rados_storage->free_mail_buffer(rmail->rados_mail->get_mail_buffer()); return -1; } - else if(ret == -ETIMEDOUT) { - int max_retry = 10; - for(int i=0;i= 0){ - i_error("READ TIMEOUT %d reading mail object %s ", ret,rmail->rados_mail != NULL ? rmail->rados_mail->to_string(" ").c_str() : " no rados_mail"); - break; - } - i_warning("READ TIMEOUT retry(%d) %d reading mail object %s ",i, ret,rmail->rados_mail != NULL ? rmail->rados_mail->to_string(" ").c_str() : " no rados_mail"); - // wait random time before try again!! - usleep(((rand() % 5) + 1) * 10000); - } - - if(ret <0){ - delete rmail->rados_mail->get_mail_buffer(); - FUNC_END(); - return -1; - } - } else { i_error("reading mail return code(%d), oid(%s),namespace(%s), alt_storage(%d)", ret, rmail->rados_mail->get_oid()->c_str(), rados_storage->get_namespace().c_str(), alt_storage); - FUNC_END_RET("ret == -1"); - delete rmail->rados_mail->get_mail_buffer(); + FUNC_END_RET("ret == -1"); + //rados_storage->free_mail_buffer(rmail->rados_mail->get_mail_buffer()); return -1; } } - int physical_size = psize; - rmail->rados_mail->set_mail_size(psize); - rmail->rados_mail->set_rados_save_date(save_date); - + int physical_size = rmail->rados_mail->get_mail_size(); + if (physical_size == 0) { i_error( "trying to read a mail(%s) with size = 0, namespace(%s), alt_storage(%d) uid(%d), which is currently copied, " @@ -571,23 +527,45 @@ static int rbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, s "expunging mail ", rmail->rados_mail->get_oid()->c_str(), rados_storage->get_namespace().c_str(), alt_storage, _mail->uid); FUNC_END_RET("ret == 0"); - rbox_mail_set_expunged(rmail); - delete rmail->rados_mail->get_mail_buffer(); + rbox_mail_set_expunged(rmail); + //rados_storage->free_mail_buffer(rmail->rados_mail->get_mail_buffer()); return -1; } else if (physical_size == INT_MAX) { i_error("trying to read a mail with INT_MAX size. (uid=%d,oid=%s,namespace=%s,alt_storage=%d)", _mail->uid, rmail->rados_mail->get_oid()->c_str(), rados_storage->get_namespace().c_str(), alt_storage); FUNC_END_RET("ret == -1"); - delete rmail->rados_mail->get_mail_buffer(); + //rados_storage->free_mail_buffer(rmail->rados_mail->get_mail_buffer()); return -1; } - - if (get_mail_stream(rmail, rmail->rados_mail->get_mail_buffer(), physical_size, &input) < 0) { + int mail_buff_size=2; + const char *mail_buff_char=rados_storage->get_mail_buffer(rmail->rados_mail->get_mail_buffer(),&mail_buff_size); + i_debug("reading stream for oid: %s, phy: %d, buffer: %d", rmail->rados_mail->get_oid()->c_str(), + physical_size, + mail_buff_size); + + // validates if object is in zlib format (first 2 byte) + bool isGzip = check_is_zlib(mail_buff_char); + if(isGzip) { + + uint32_t result = zlib_trailer_msg_length(mail_buff_char,physical_size); + + // get mails real physical size and compare against trailer length + uoff_t real_physical_size; + rbox_mail_get_physical_size(_mail, &real_physical_size); + // in case we have corrupt trailer, + if(result-real_physical_size > zlib_header_length(mail_buff_char)){ + i_warning("zlib size check failed %d trailer not as expected, fixing by adding 0x00 to msb",(result-real_physical_size)); + std::stringstream appended_zero; + appended_zero<<0x00; + rados_storage->append_to_buffer(rmail->rados_mail->get_mail_buffer(),appended_zero.str().c_str(),1); + physical_size+=1; + } + } + if (get_mail_stream(rmail,mail_buff_char, physical_size, &input) < 0) { FUNC_END_RET("ret == -1"); - delete rmail->rados_mail->get_mail_buffer(); + //rados_storage->free_mail_buffer(rmail->rados_mail->get_mail_buffer());; return -1; } - data->stream = input; index_mail_set_read_buffer_size(_mail, input); } @@ -595,6 +573,82 @@ static int rbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, s FUNC_END(); return ret; } +uint32_t zlib_trailer_msg_length(char* mail_buff_char, int physical_size) { + unsigned char gzip_size[] = { + mail_buff_char[physical_size-1], + mail_buff_char[physical_size-2], + mail_buff_char[physical_size-3], + mail_buff_char[physical_size-4] + }; + uint32_t result = (gzip_size[0] << 24 | gzip_size[1] << 16 | gzip_size[2] << 8 | gzip_size[3]); + + i_debug("length of message(%d) last byte of trailer %d / %d / %d / %d sizeof(char %d), sizeof(unsingned int: %d) ", + result, + gzip_size[0], gzip_size[1], gzip_size[2], gzip_size[3], + sizeof(char), sizeof(unsigned int)); + return result; + +} +bool check_is_zlib(char* mail_buff_char) { + + unsigned char magic1 = mail_buff_char[0]; + unsigned char magic2 = mail_buff_char[1]; + + i_debug("checking for z_lib header magic bytes check %x : %x compared to %x : %x", + magic1,magic2, + 0x1f,0x8b); + + if(magic1 == 0x1f && magic2 == 0x8b){ + i_debug("magic bytes 0x1f 0x8b found"); + return true; + } + i_debug("magic bytes 0x1f 0x8b not found"); + return false; +} + +int zlib_header_length(char* mail_buff_char) { + + int header_length = 11; + const unsigned char FLG=mail_buff_char[3]; + + switch (FLG){ + //ETEXT + case (unsigned char) 0x01: + header_length=10; + break; + case (unsigned char) 0x02: + header_length=10 + 2; + break; + //FXTERA + case (unsigned char) 0x04: + header_length=header_extra_size(mail_buff_char); + break; + case (unsigned char) 0x08: + header_length=header_dynamic_size(mail_buff_char); + break; + case (unsigned char) 0x10: + header_length=header_dynamic_size(mail_buff_char); + break; + default: + break; + } + i_debug("found header Type %x header size is %d", FLG, header_length); + + return header_length; +} +int header_dynamic_size(const unsigned char *data){ + + int i=10 - 1; + do{ + i++; + }while(data[i]!=(unsigned char) 0 ); + return i++; +} + +int header_extra_size(const unsigned char *data){ + int extera_part_size= int(data[10] + data[11]); + return 10 + 2 + extera_part_size; +} // guid is saved in the obox header, and should be available when rbox_mail does exist. (rbox_get_index_record) int rbox_get_guid_metadata(struct rbox_mail *mail, const char **value_r) { diff --git a/src/storage-rbox/rbox-mail.h b/src/storage-rbox/rbox-mail.h index db5727f1..ba999c1b 100644 --- a/src/storage-rbox/rbox-mail.h +++ b/src/storage-rbox/rbox-mail.h @@ -14,7 +14,7 @@ #include "index-mail.h" #include -#include "../librmb/rados-mail.h" +#include "../storage-interface/rados-mail.h" /** * @brief: holds the rados mail object. @@ -25,7 +25,7 @@ struct rbox_mail { guid_128_t index_guid; guid_128_t index_oid; /** refrence to rados mail object **/ - librmb::RadosMail *rados_mail; + storage_interface::RadosMail *rados_mail; uint32_t last_seq; // TODO(jrse): init with -1 }; extern void rbox_mail_set_expunged(struct rbox_mail *mail); @@ -35,9 +35,11 @@ extern struct mail *rbox_mail_alloc(struct mailbox_transaction_context *t, enum extern int rbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r); extern int rbox_get_guid_metadata(struct rbox_mail *mail, const char **value_r); +extern bool check_is_zlib(char* mail_buff_char); +extern int zlib_header_length(char* mail_buff_char); +extern uint32_t zlib_trailer_msg_length(char* mail_buff_char, int physical_size); + +extern int header_dynamic_size(const unsigned char *data); +extern int header_extra_size(const unsigned char *data); -extern int read_mail_from_storage(librmb::RadosStorage *rados_storage, - struct rbox_mail *rmail, - uint64_t *psize, - time_t *save_date); #endif // SRC_STORAGE_RBOX_RBOX_MAIL_H_ diff --git a/src/storage-rbox/rbox-save.cpp b/src/storage-rbox/rbox-save.cpp index 4670a41d..a06dc4c1 100644 --- a/src/storage-rbox/rbox-save.cpp +++ b/src/storage-rbox/rbox-save.cpp @@ -28,27 +28,28 @@ extern "C" { #include "str.h" #include "rbox-sync.h" -#include "rados-types.h" +#include "../storage-interface/rados-types.h" #include "debug-helper.h" #if DOVECOT_PREREQ(2, 3) #include "index-pop3-uidl.h" #endif } -#include "../librmb/rados-mail.h" +#include "../storage-interface/rados-mail.h" #include "rbox-storage.hpp" #include "rbox-save.h" -#include "rados-util.h" +#include "../storage-interface/rados-util.h" #include "rbox-mail.h" #include "ostream-bufferlist.h" +#include "../storage-engine/storage-backend-factory.h" using ceph::bufferlist; -using librmb::RadosMail; -using librmb::RadosMetadata; -using librmb::RadosStorage; -using librmb::rbox_metadata_key; - +using storage_interface::RadosMail; +using storage_interface::RadosMetadata; +using storage_interface::RadosStorage; +using storage_interface::rbox_metadata_key; +using storage_engine::StorageBackendFactory; using std::string; static const char X_ATTR_VERSION_VALUE[] = "0.1"; @@ -94,8 +95,12 @@ void setup_mail_object(struct mail_save_context *_ctx) { if (str.find("-")rados_mail->set_deprecated_uid(true); } - librmb::RadosUtils::find_and_replace(&str, "-", ""); // remove hyphens if they exist + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->find_and_replace(&str, "-", ""); // remove hyphens if they exist mail_generate_guid_128_hash(str.c_str(), r_ctx->mail_guid); + delete rados_utils; + rados_utils = nullptr; } else { guid_128_generate(r_ctx->mail_guid); } @@ -171,8 +176,12 @@ void rbox_move_index(struct mail_save_context *_ctx, struct mail *src_mail) { if (str.find("-")rados_mail->set_deprecated_uid(true); } - librmb::RadosUtils::find_and_replace(&str, "-", ""); // remove hyphens if they exist + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->find_and_replace(&str, "-", ""); // remove hyphens if they exist mail_generate_guid_128_hash(str.c_str(), r_ctx->mail_guid); + delete rados_utils; + rados_utils=nullptr; } else { guid_128_generate(r_ctx->mail_guid); } @@ -198,9 +207,6 @@ void init_output_stream(mail_save_context *_ctx) { i_debug("freing data output stream!"); o_stream_unref(&_ctx->data.output); } - - // create buffer ( delete is in wait_for_write_operations) - r_ctx->rados_mail->set_mail_buffer(new librados::bufferlist()); r_ctx->output_stream = o_stream_create_bufferlist(r_ctx->rados_mail, &r_ctx->rados_storage, rbox->storage->config->is_write_chunks()); o_stream_cork(r_ctx->output_stream); @@ -299,39 +305,56 @@ int rbox_save_continue(struct mail_save_context *_ctx) { return 0; } -static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb::RadosMail *mail_object) { +static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, storage_interface::RadosMail *mail_object) { FUNC_START(); struct mail_save_data *mdata = &r_ctx->ctx.data; struct rbox_storage *r_storage = (struct rbox_storage *)&r_ctx->mbox->storage->storage; - + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_VERSION)) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_VERSION, X_ATTR_VERSION_VALUE); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_char( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_VERSION, X_ATTR_VERSION_VALUE); + mail_object->add_metadata(xattr); } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID)) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, guid_128_to_string(r_ctx->mbox->mailbox_guid)); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_uint( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, + guid_128_to_string(r_ctx->mbox->mailbox_guid)); + mail_object->add_metadata(xattr); } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_GUID)) { //#286: use deprecated_flag to save original uuid format to G xattr if(!mail_object->is_deprecated_uid()){ - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_GUID, guid_128_to_string(r_ctx->mail_guid)); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_GUID, guid_128_to_string(r_ctx->mail_guid)); + mail_object->add_metadata(xattr); } else{ - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_GUID, guid_128_to_uuid_string(r_ctx->mail_guid,FORMAT_RECORD)); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_GUID, guid_128_to_uuid_string(r_ctx->mail_guid,FORMAT_RECORD)); mail_object->add_metadata(xattr); } } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME)) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, mdata->received_date); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_time( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, mdata->received_date); mail_object->add_metadata(xattr); } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_POP3_UIDL)) { if (mdata->pop3_uidl != NULL) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_char( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); mail_object->add_metadata(xattr); r_ctx->have_pop3_uidls = TRUE; #if DOVECOT_PREREQ(2, 3) @@ -341,7 +364,9 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb:: } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_POP3_ORDER)) { if (mdata->pop3_order != 0) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_ORDER, mdata->pop3_order); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_uint( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_POP3_ORDER, mdata->pop3_order); mail_object->add_metadata(xattr); r_ctx->have_pop3_orders = TRUE; #if DOVECOT_PREREQ(2, 3) @@ -351,7 +376,9 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb:: } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_FROM_ENVELOPE)) { if (mdata->from_envelope != NULL) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_FROM_ENVELOPE, mdata->from_envelope); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_char( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_FROM_ENVELOPE, mdata->from_envelope); mail_object->add_metadata(xattr); } } @@ -361,18 +388,24 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb:: i_warning("unable to determine virtual size, using physical size instead."); vsize = r_ctx->input->v_offset; } - librmb::RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_VIRTUAL_SIZE, vsize); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_uint( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_VIRTUAL_SIZE, vsize); mail_object->add_metadata(xattr); } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_PHYSICAL_SIZE)) { - librmb::RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_PHYSICAL_SIZE, r_ctx->input->v_offset); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_uint( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_PHYSICAL_SIZE, r_ctx->input->v_offset); mail_object->add_metadata(xattr); } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_OLDV1_FLAGS)) { if (mdata->flags != 0) { std::string flags; - if (librmb::RadosUtils::flags_to_string(mdata->flags, &flags)) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_OLDV1_FLAGS, flags); + if (rados_utils->flags_to_string(mdata->flags, &flags)) { + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_OLDV1_FLAGS, flags); mail_object->add_metadata(xattr); } } @@ -381,14 +414,18 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb:: if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_PVT_FLAGS)) { if (mdata->pvt_flags != 0) { std::string pvt_flags; - if (librmb::RadosUtils::flags_to_string(mdata->pvt_flags, &pvt_flags)) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_PVT_FLAGS, pvt_flags); + if (rados_utils->flags_to_string(mdata->pvt_flags, &pvt_flags)) { + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_PVT_FLAGS, pvt_flags); mail_object->add_metadata(xattr); } } } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX)) { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, r_ctx->mbox->box.name); + RadosMetadata *xattr= + StorageBackendFactory::create_metadata_char( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, r_ctx->mbox->box.name); mail_object->add_metadata(xattr); } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_OLDV1_KEYWORDS)) { @@ -408,13 +445,15 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb:: // set keyword_idx : keyword_value std::string key_idx = std::to_string(keyword_indexes[i]); std::string keyword_value = keywords[i]; - RadosMetadata ext_metadata(key_idx, keyword_value); + RadosMetadata *ext_metadata= + StorageBackendFactory::create_metadata_str_key_val(storage_engine::CEPH, key_idx, keyword_value); mail_object->add_extended_metadata(ext_metadata); } } mail_object->set_rados_save_date(mdata->save_date); - + delete rados_utils; + rados_utils=nullptr; FUNC_END(); return 0; } @@ -427,7 +466,7 @@ static void clean_up_failed(struct rbox_save_context *r_ctx, bool wait_for_opera // try to clean up! for (std::list::iterator it_cur_obj = r_ctx->rados_mails.begin(); it_cur_obj != r_ctx->rados_mails.end(); ++it_cur_obj) { - int delete_ret = r_storage->s->delete_mail(*it_cur_obj); + int delete_ret = r_storage->s->delete_mail(*(*it_cur_obj)->get_oid()); if (delete_ret < 0 && delete_ret != -ENOENT) { i_error("Librados obj: %s, could not be removed", (*it_cur_obj)->get_oid()->c_str()); } @@ -468,313 +507,6 @@ static void clean_up_write_finish(struct mail_save_context *_ctx) { FUNC_END(); } -int save_mail_async(RadosStorage *rados_storage, - RadosMail *current_object, - librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write) { - - librados::AioCompletion *completion = librados::Rados::aio_create_completion(); - - int ret_val = 0; - uint64_t write_buffer_size = current_object->get_mail_size() -1; // write buffer size needs to be length -1 - - assert(max_write > 0); - - if (write_buffer_size == 0 || max_write == 0) { - ret_val = -1; - i_debug("write_buffer_size == 0 or max_write <=0 < -1" ); - return ret_val; - } - - ret_val = rados_storage->aio_operate(&rados_storage->get_io_ctx(),*current_object->get_oid(), completion, write_op_xattr); - - if(ret_val< 0){ - i_debug("write metadata did not work: %d",ret_val); - ret_val = -1; - return ret_val; - } - - uint64_t rest = write_buffer_size % max_write; - int div = write_buffer_size / max_write + (rest > 0 ? 1 : 0); - for (int i = 0; i < div; ++i) { - - // split the buffer. - librados::bufferlist tmp_buffer; - - librados::ObjectWriteOperation write_op; - - int offset = i * max_write; - - uint64_t length = max_write; - if (write_buffer_size < ((i + 1) * length)) { - length = rest; - } - - if (div == 1) { - write_op.write(0, *current_object->get_mail_buffer()); - } else { - i_debug("write chunk size %d, offset=%d,lenght=%d",write_buffer_size,offset,length); - if(offset + length > write_buffer_size){ - i_error("offset and length (%d) is bigger then write_buffer size (%d)", (offset+length), write_buffer_size); - return -1; - }else{ - tmp_buffer.substr_of(*current_object->get_mail_buffer(), offset, length); - } - - if(tmp_buffer.length() == 0){ - i_info("save buffer from %d to %d is empty => done",offset, length); - break; - } - write_op.write(offset, tmp_buffer); - - } - - ret_val = rados_storage->aio_operate(&rados_storage->get_io_ctx(),*current_object->get_oid(), completion, &write_op); - - i_debug("append mail (aio_operate) return value: %d",ret_val); - if(ret_val < 0){ - ret_val = -1; - break; - } - } - - bool failed = rados_storage->wait_for_write_operations_complete(completion,nullptr); - - if(ret_val >= 0){ - ret_val = failed ? -1 : 0; - i_debug("completion return value %d",ret_val); - } - - // deprecated unused - current_object->set_write_operation(nullptr); - current_object->set_completion(nullptr); - current_object->set_active_op(0); - // free mail's buffer cause we don't need it anymore - librados::bufferlist *mail_buffer = current_object->get_mail_buffer(); - delete mail_buffer; - current_object->set_mail_buffer(nullptr); - - return ret_val; -} - -int save_mail_sync(RadosStorage *rados_storage, - RadosMail *current_object, - librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write) { - - int ret_val = 0; - uint64_t write_buffer_size = current_object->get_mail_size() -1; // write buffer size needs to be length -1 - - assert(max_write > 0); - - if (write_buffer_size == 0 || max_write == 0) { - ret_val = -1; - i_debug("write_buffer_size == 0 or max_write <=0 < -1" ); - return ret_val; - } - - ret_val = rados_storage->get_io_ctx().operate(*current_object->get_oid(), write_op_xattr); - - if(ret_val< 0){ - i_debug("write metadata did not work: %d",ret_val); - ret_val = -1; - return ret_val; - } - - uint64_t rest = write_buffer_size % max_write; - int div = write_buffer_size / max_write + (rest > 0 ? 1 : 0); - for (int i = 0; i < div; ++i) { - - // split the buffer. - librados::bufferlist tmp_buffer; - librados::ObjectWriteOperation write_op; - - int offset = i * max_write; - - uint64_t length = max_write; - if (write_buffer_size < ((i + 1) * length)) { - length = rest; - } - - if (div == 1) { - write_op.write(0, *current_object->get_mail_buffer()); - } else { - i_debug("write chunk size %d, offset=%d,lenght=%d",write_buffer_size,offset,length); - if(offset + length > write_buffer_size){ - i_error("offset and length (%d) is bigger then write_buffer size (%d)", (offset+length), write_buffer_size); - return -1; - }else{ - tmp_buffer.substr_of(*current_object->get_mail_buffer(), offset, length); - } - write_op.write(offset, tmp_buffer); - } - ret_val = rados_storage->get_io_ctx().operate(*current_object->get_oid(), &write_op); - i_debug("append mail (operate) return value: %d",ret_val); - if(ret_val < 0){ - ret_val = -1; - break; - } - } - // deprecated unused - current_object->set_write_operation(nullptr); - current_object->set_completion(nullptr); - current_object->set_active_op(0); - - i_debug("freeing mailbuffer"); - // free mail's buffer cause we don't need it anymore - librados::bufferlist *mail_buffer = current_object->get_mail_buffer(); - delete mail_buffer; - - return ret_val; -} - -int save_mail_write_append(RadosStorage *rados_storage, - RadosMail *current_object, - librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write) { - - int ret_val = 0; - uint64_t write_buffer_size = current_object->get_mail_size() -1; - - assert(max_write > 0); - - if (write_buffer_size == 0 || max_write <= 0) { - ret_val = -1; - i_debug("write_buffer_size == 0 or max_write <=0 < -1" ); - return ret_val; - } - - ret_val = rados_storage->get_io_ctx().operate(*current_object->get_oid(), write_op_xattr); - - if(ret_val< 0){ - i_debug("write metadata did not work: %d",ret_val); - ret_val = -1; - return ret_val; - } - - uint64_t rest = write_buffer_size % max_write; - int div = write_buffer_size / max_write + (rest > 0 ? 1 : 0); - for (int i = 0; i < div; ++i) { - - // split the buffer. - librados::bufferlist tmp_buffer; - - librados::ObjectWriteOperation write_op; - - int offset = i * max_write; - - uint64_t length = max_write; - if (write_buffer_size < ((i + 1) * length)) { - length = rest; - } - - if (div == 1) { - write_op.write(0, *current_object->get_mail_buffer()); - ret_val = rados_storage->get_io_ctx().operate(*current_object->get_oid(), &write_op); - - } else { - i_debug("write chunk size %d, offset=%d,lenght=%d",write_buffer_size,offset,length); - if(offset + length > write_buffer_size){ - i_error("offset and length (%d) is bigger then write_buffer size (%d)", (offset+length), write_buffer_size); - return -1; - }else{ - tmp_buffer.substr_of(*current_object->get_mail_buffer(), offset, length); - } - i_debug("tmp_buffer %d ",tmp_buffer.length()); - ret_val = rados_storage->get_io_ctx().append(*current_object->get_oid(), tmp_buffer, length); - } - i_debug("append mail (append) return value: %d",ret_val); - if(ret_val < 0){ - ret_val = -1; - break; - } - } - // deprecated unused - current_object->set_write_operation(nullptr); - current_object->set_completion(nullptr); - current_object->set_active_op(0); - - i_debug("freeing mailbuffer"); - // free mail's buffer cause we don't need it anymore - librados::bufferlist *mail_buffer = current_object->get_mail_buffer(); - delete mail_buffer; - - return ret_val; -} -int save_mail_write_ioctx(RadosStorage *rados_storage, - RadosMail *current_object, - librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write) { - - int ret_val = 0; - uint64_t write_buffer_size = current_object->get_mail_size() -1; - - assert(max_write > 0); - - if (write_buffer_size == 0 || max_write == 0) { - ret_val = -1; - i_debug("write_buffer_size == 0 or max_write <=0 < -1" ); - return ret_val; - } - - ret_val = rados_storage->get_io_ctx().operate(*current_object->get_oid(), write_op_xattr); - - if(ret_val< 0){ - i_debug("write metadata did not work: %d",ret_val); - ret_val = -1; - return ret_val; - } - - uint64_t rest = write_buffer_size % max_write; - int div = write_buffer_size / max_write + (rest > 0 ? 1 : 0); - for (int i = 0; i < div; ++i) { - - // split the buffer. - librados::bufferlist tmp_buffer; - - librados::ObjectWriteOperation write_op; - - int offset = i * max_write; - - uint64_t length = max_write; - if (write_buffer_size < ((i + 1) * length)) { - length = rest; - } - - if (div == 1) { - write_op.write(0, *current_object->get_mail_buffer()); - ret_val = rados_storage->get_io_ctx().operate(*current_object->get_oid(), &write_op); - - } else { - i_debug("write chunk size %d, offset=%d,lenght=%d",write_buffer_size,offset,length); - if(offset + length > write_buffer_size){ - i_error("offset and length (%d) is bigger then write_buffer size (%d)", (offset+length), write_buffer_size); - return -1; - }else{ - tmp_buffer.substr_of(*current_object->get_mail_buffer(), offset, length); - } - i_debug("tmp_buffer %d ",tmp_buffer.length()); - ret_val = rados_storage->get_io_ctx().write(*current_object->get_oid(), tmp_buffer, length, offset); - } - i_debug("append mail (io_ctx->write) return value: %d",ret_val); - if(ret_val < 0){ - ret_val = -1; - break; - } - } - // deprecated unused - current_object->set_write_operation(nullptr); - current_object->set_completion(nullptr); - current_object->set_active_op(0); - - i_debug("freeing mailbuffer"); - // free mail's buffer cause we don't need it anymore - librados::bufferlist *mail_buffer = current_object->get_mail_buffer(); - delete mail_buffer; - - return ret_val; -} - int rbox_save_finish(struct mail_save_context *_ctx) { FUNC_START(); @@ -788,15 +520,11 @@ int rbox_save_finish(struct mail_save_context *_ctx) { if (r_ctx->ctx.data.output != r_ctx->output_stream ) { /* e.g. zlib plugin had changed this. make sure we successfully write the trailer. */ - i_info("check state2 %ld",r_ctx->ctx.data.output); - ret = o_stream_finish(r_ctx->ctx.data.output); } else if(r_ctx->ctx.data.output != NULL) { //TODO: check for error in the stream before using flush /* no plugins - flush the output so far */ - ret = o_stream_flush(r_ctx->ctx.data.output); - } if (ret < 0) { @@ -837,59 +565,32 @@ int rbox_save_finish(struct mail_save_context *_ctx) { i_error("ERROR, mailsize is <= 0 "); } else { - - if (!zlib_plugin_active) { - // write \0 to ceph (length()+1) if stream is not binary - r_ctx->rados_mail->set_mail_size(r_ctx->output_stream->offset + 1); - - } else { - // binary stream, do not modify the length of stream. - r_ctx->rados_mail->set_mail_size(r_ctx->output_stream->offset); - } - - rbox_save_mail_set_metadata(r_ctx, r_ctx->rados_mail); - - librados::ObjectWriteOperation write_op; struct rbox_storage *r_storage = (struct rbox_storage *)&r_ctx->mbox->storage->storage; + + r_ctx->rados_mail->set_mail_size(r_ctx->output_stream->offset); + rbox_save_mail_set_metadata(r_ctx, r_ctx->rados_mail); - r_storage->ms->get_storage()->save_metadata(&write_op, r_ctx->rados_mail); - - int max_object_size = r_storage->s->get_max_object_size(); - i_debug("max_object_size %d mail_size %d",max_object_size, r_ctx->rados_mail->get_mail_size() ); - if(max_object_size < r_ctx->rados_mail->get_mail_size()) { - i_error("configured CEPH Object size %d < then mail size %d ", r_storage->s->get_max_object_size(), r_ctx->rados_mail->get_mail_size() ); - mail_set_critical(r_ctx->ctx.dest_mail, "write(%s) failed: %s", o_stream_get_name(r_ctx->ctx.data.output),"MAX OBJECT SIZE REACHED"); - r_ctx->failed = true; - }else { - - time_t save_date = r_ctx->rados_mail->get_rados_save_date(); - write_op.mtime(&save_date); - - uint32_t config_chunk_size = r_storage->config->get_chunk_size(); - if(config_chunk_size > r_storage->s->get_max_write_size_bytes()){ - config_chunk_size = r_storage->s->get_max_write_size_bytes(); - } - i_debug("max chunk write size: %d ", config_chunk_size ); - uint32_t write_method = r_storage->config->get_write_method(); - - int ret = 0; - i_debug("write method: %d",write_method); - if(write_method == 0) { - ret = save_mail_sync(r_storage->s,r_ctx->rados_mail, &write_op, config_chunk_size); - }else if (write_method == 1){ - ret = save_mail_async(r_storage->s,r_ctx->rados_mail, &write_op, config_chunk_size); - }else if (write_method == 2){ - ret = save_mail_write_append(r_storage->s,r_ctx->rados_mail, &write_op, config_chunk_size); - }else if (write_method == 3){ - ret = save_mail_write_ioctx(r_storage->s,r_ctx->rados_mail, &write_op, config_chunk_size); - }else { - r_ctx->failed = true; - i_error("SAVE_MAIL result: %d no save method set", r_ctx->failed); - } - - r_ctx->failed = ret < 0; - i_debug("SAVE_MAIL result: %d", r_ctx->failed); +#ifdef DEBUG + i_debug("save mail metadata %s . Metadata_count %ld, mail_size (%d)", r_ctx->rados_mail->get_oid()->c_str(), + r_ctx->rados_mail->get_metadata()->size(), r_ctx->rados_mail->get_mail_size()); + for (std::map::iterator it = r_ctx->rados_mail->get_metadata()->begin(); + it != r_ctx->rados_mail->get_metadata()->end(); ++it) { + void* t = (void*) (*it).second; + ceph::bufferlist bl = *(ceph::bufferlist*)t; + ceph::bufferlist bb; + bb.append("dfdf"); + char* key = (*it).first.c_str(); + i_debug("metadata: %s %s",key, bl.to_str().c_str()); + } +#endif + if(r_storage->ms->get_storage()->set_metadata(r_ctx->rados_mail) < 0) { + i_error("save mail metadata failed %s failed. Metadata_count %ld, mail_size (%d)", r_ctx->rados_mail->get_oid()->c_str(), + r_ctx->rados_mail->get_metadata()->size(), r_ctx->rados_mail->get_mail_size()); + r_ctx->failed = true; } + + r_ctx->failed=r_storage->s->save_mail(r_ctx->rados_mail) ? false : true; + if (r_ctx->failed) { i_error("saved mail: %s failed. Metadata_count %ld, mail_size (%d)", r_ctx->rados_mail->get_oid()->c_str(), r_ctx->rados_mail->get_metadata()->size(), r_ctx->rados_mail->get_mail_size()); @@ -897,6 +598,7 @@ int rbox_save_finish(struct mail_save_context *_ctx) { if( r_storage->config->get_object_search_method() == 2){ // ceph config schalter an oder aus! + r_storage->s->ceph_index_append(*r_ctx->rados_mail->get_oid()); uint64_t index_size = r_storage->s->ceph_index_size(); // WARN if index reaches 80% of max object size @@ -906,15 +608,22 @@ int rbox_save_finish(struct mail_save_context *_ctx) { } } if (r_storage->save_log->is_open()) { + std::cout<<"r_storage->save_log->is_open()"<save_log->append( - librmb::RadosSaveLogEntry(*r_ctx->rados_mail->get_oid(), r_storage->s->get_namespace(), - r_storage->s->get_pool_name(), librmb::RadosSaveLogEntry::op_save())); + storage_engine::StorageBackendFactory::create_save_log_entry( + storage_engine::CEPH, + *r_ctx->rados_mail->get_oid(), r_storage->s->get_namespace(), + r_storage->s->get_pool_name(), save_log_entry->op_save())); + delete save_log_entry; + save_log_entry=nullptr; } } + std::cout<<"save_log_entry"<failed ? -1 : 0; } @@ -940,19 +649,22 @@ static int rbox_save_assign_uids(struct rbox_save_context *r_ctx, const ARRAY_TY unsigned int n = 0; seq_range_array_iter_init(&iter, uids); struct rbox_storage *r_storage = (struct rbox_storage *)&r_ctx->mbox->storage->storage; - RadosMetadata metadata; - for (std::list::iterator it = r_ctx->rados_mails.begin(); it != r_ctx->rados_mails.end(); ++it) { + + for (std::list::iterator it = r_ctx->rados_mails.begin(); + it != r_ctx->rados_mails.end(); ++it) { r_ctx->rados_mail = *it; bool ret = seq_range_array_iter_nth(&iter, n++, &uid); i_assert(ret); if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_MAIL_UID)) { - metadata.convert(rbox_metadata_key::RBOX_METADATA_MAIL_UID, uid); - - librados::ObjectWriteOperation write_mail_uid; - write_mail_uid.setxattr(metadata.key.c_str(), metadata.bl); - - if (r_storage->ms->get_storage()->set_metadata(r_ctx->rados_mail, metadata, &write_mail_uid) < 0) { - return -1; + + // deletion will be done when rbox mail is deleted + storage_interface::RadosMetadata *metadata= + storage_engine::StorageBackendFactory::create_metadata_default(storage_engine::CEPH); + + metadata->convert(rbox_metadata_key::RBOX_METADATA_MAIL_UID, std::to_string(uid)); + r_ctx->rados_mail->add_metadata(metadata); + if(r_storage->ms->get_storage()->set_metadata(r_ctx->rados_mail) < 0) { + return -1; } } #if DOVECOT_PREREQ(2, 3) @@ -961,6 +673,7 @@ static int rbox_save_assign_uids(struct rbox_save_context *r_ctx, const ARRAY_TY } #endif } + i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); } @@ -1079,7 +792,6 @@ void rbox_transaction_save_rollback(struct mail_save_context *_ctx) { FUNC_START(); struct rbox_save_context *r_ctx = (struct rbox_save_context *)_ctx; - librmb::RadosStorage *storage = ((struct rbox_storage *)&r_ctx->mbox->storage->storage)->s; if (!r_ctx->finished) { rbox_save_cancel(&r_ctx->ctx); diff --git a/src/storage-rbox/rbox-save.h b/src/storage-rbox/rbox-save.h index 36fcc3b8..e2fead86 100644 --- a/src/storage-rbox/rbox-save.h +++ b/src/storage-rbox/rbox-save.h @@ -14,11 +14,9 @@ #include #include - -#include "../librmb/rados-storage-impl.h" #include "mail-storage-private.h" - -#include "../librmb/rados-mail.h" +#include "../storage-interface/rados-storage.h" +#include "../storage-interface/rados-mail.h" /** * @brief: rbox_save_context * class is holding all references to @@ -28,7 +26,7 @@ */ class rbox_save_context { public: - explicit rbox_save_context(const librmb::RadosStorage &_rados_storage) + explicit rbox_save_context(const storage_interface::RadosStorage &_rados_storage) : ctx({}), mbox(NULL), trans(NULL), @@ -70,11 +68,11 @@ class rbox_save_context { /** stream to write the mail data to **/ struct ostream *output_stream; /** storage rerference **/ - const librmb::RadosStorage &rados_storage; + const storage_interface::RadosStorage &rados_storage; /** mails in the current save context **/ - std::list rados_mails; + std::list rados_mails; /** current mail in the context **/ - librmb::RadosMail *rados_mail; + storage_interface::RadosMail *rados_mail; #if DOVECOT_PREREQ(2, 3) unsigned int highest_pop3_uidl_seq : 1; #endif diff --git a/src/storage-rbox/rbox-storage.cpp b/src/storage-rbox/rbox-storage.cpp index f7f19b50..89f907fb 100644 --- a/src/storage-rbox/rbox-storage.cpp +++ b/src/storage-rbox/rbox-storage.cpp @@ -34,21 +34,16 @@ extern "C" { } #include "rbox-mailbox-list-fs.h" - -#include "../librmb/rados-cluster-impl.h" -#include "../librmb/rados-storage-impl.h" -#include "../librmb/rados-namespace-manager.h" -#include "../librmb/rados-dovecot-ceph-cfg-impl.h" -#include "../librmb/rados-guid-generator.h" -#include "../librmb/rados-metadata-storage-impl.h" - +#include "../storage-interface/rados-namespace-manager.h" +#include "../storage-interface/rados-guid-generator.h" +#include "../storage-engine/storage-backend-factory.h" +#include "../storage-interface/rados-types.h" +#include "../storage-interface/rados-storage.h" #include "rbox-copy.h" #include "rbox-mail.h" -#include "rados-types.h" using std::string; - -class RboxGuidGenerator : public librmb::RadosGuidGenerator { +class RboxGuidGenerator : public storage_interface::RadosGuidGenerator { public: void generate_guid(std::string *guid_) override { guid_128_t namespace_guid; @@ -62,7 +57,6 @@ extern struct mailbox_vfuncs rbox_mailbox_vfuncs; struct mail_storage *rbox_storage_alloc(void) { FUNC_START(); - struct rbox_storage *r_storage; pool_t pool; pool = pool_alloconly_create("rbox storage", 512 + 256); @@ -70,23 +64,34 @@ struct mail_storage *rbox_storage_alloc(void) { i_zero(r_storage); r_storage->storage = rbox_storage; r_storage->storage.pool = pool; - r_storage->cluster = new librmb::RadosClusterImpl(); - r_storage->s = new librmb::RadosStorageImpl(r_storage->cluster); - r_storage->config = new librmb::RadosDovecotCephCfgImpl(&r_storage->s->get_io_ctx()); - r_storage->ns_mgr = new librmb::RadosNamespaceManager(r_storage->config); - r_storage->ms = new librmb::RadosMetadataStorageImpl(); - r_storage->alt = new librmb::RadosStorageImpl(r_storage->cluster); + r_storage->cluster = + storage_engine::StorageBackendFactory::create_cluster(storage_engine::CEPH); + r_storage->s = + storage_engine::StorageBackendFactory::create_storage(storage_engine::CEPH,r_storage->cluster); + r_storage->config = + storage_engine::StorageBackendFactory::create_dovecot_ceph_cfg_io( + storage_engine::CEPH,r_storage->s->get_io_ctx_wrapper()); + r_storage->ns_mgr = + storage_engine::StorageBackendFactory::create_namespace_manager(storage_engine::CEPH,r_storage->config); + r_storage->ms = + storage_engine::StorageBackendFactory::create_metadata_storage( + storage_engine::CEPH,r_storage->s->get_io_ctx_wrapper(),r_storage->config); + r_storage->alt = + storage_engine::StorageBackendFactory::create_storage(storage_engine::CEPH,r_storage->cluster); // logfile is set when 90-plugin.conf param rados_save_cfg is evaluated. - r_storage->save_log = new librmb::RadosSaveLog(); - + r_storage->save_log = + storage_engine::StorageBackendFactory::create_save_log_default(storage_engine::CEPH); + + // set ceph-plugin logger for c++ components + storage_engine::StorageBackendFactory::init_logger(r_storage->config); + FUNC_END(); return &r_storage->storage; } void rbox_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, struct mailbox_list_settings *set) { FUNC_START(); - if (set->layout == NULL) { set->layout = MAILBOX_LIST_NAME_FS; // TODO(peter): better default? set->layout = MAILBOX_LIST_NAME_INDEX; @@ -434,7 +439,7 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { struct rbox_mailbox *rbox = (struct rbox_mailbox *)box; struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librmb::RadosStorage *rados_storage = rbox->storage->s; + storage_interface::RadosStorage *rados_storage = rbox->storage->s; if (!r_storage->config->is_config_valid()) { // initialize storage with plugin configuration read_plugin_configuration(box); @@ -444,8 +449,8 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { int ret = 0; try { rados_storage->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb() - ? librmb::WAIT_FOR_SAFE_AND_CB - : librmb::WAIT_FOR_COMPLETE_AND_CB); + ? storage_interface::WAIT_FOR_SAFE_AND_CB + : storage_interface::WAIT_FOR_COMPLETE_AND_CB); /* open connection to primary and alternative storage */ ret = rados_storage->open_connection(rbox->storage->config->get_pool_name(), rbox->storage->config->get_index_pool_name(), @@ -459,8 +464,8 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { rbox->storage->config->get_rados_username()); rbox->storage->alt->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb() - ? librmb::WAIT_FOR_SAFE_AND_CB - : librmb::WAIT_FOR_COMPLETE_AND_CB); + ? storage_interface::WAIT_FOR_SAFE_AND_CB + : storage_interface::WAIT_FOR_COMPLETE_AND_CB); } } catch (std::exception &e) { i_error("Exception: setting up ceph connection: %s",e.what()); @@ -478,10 +483,11 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { if (ret < 0) { i_error( - "Open rados connection. Error(%d,%s) (pool_name(%s), cluster_name(%s), rados_user_name(%s), " + "Open rados connection. Error(%d,%s) (pool_name(%s), (index_pool(%s)), cluster_name(%s), rados_user_name(%s), " "alt_storage(%d), " "alt_dir(%s) )", ret, strerror(ret * -1), rbox->storage->config->get_pool_name().c_str(), + rbox->storage->config->get_index_pool_name().c_str(), rbox->storage->config->get_rados_cluster_name().c_str(), rbox->storage->config->get_rados_username().c_str(), alt_storage, box->list->set.alt_dir); return ret; @@ -498,7 +504,7 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { assert(ret == 0); return ret; } - rbox->storage->ms->create_metadata_storage(&rbox->storage->s->get_io_ctx(), rbox->storage->config); + // rbox->storage->ms->create_metadata_storage(rbox->storage->s->get_io_ctx_wrapper(), rbox->storage->config); std::string uid; if (box->list->ns->owner != nullptr) { @@ -509,8 +515,8 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { } std::string ns; if (!rbox->storage->ns_mgr->lookup_key(uid, &ns)) { - RboxGuidGenerator guid_generator; - ret = rbox->storage->ns_mgr->add_namespace_entry(uid, &ns, &guid_generator) ? 0 : -1; + storage_interface::RadosGuidGenerator *guid_generator = new RboxGuidGenerator() ; + ret = rbox->storage->ns_mgr->add_namespace_entry(uid, &ns, guid_generator) ? 0 : -1; } if (ret >= 0) { rados_storage->set_namespace(ns); @@ -520,7 +526,6 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { } else { i_error("error namespace not set: for uid %s error code is: %d", uid.c_str(), ret); } - FUNC_END(); return ret; } @@ -875,8 +880,8 @@ void rbox_notify_changes(struct mailbox *box) { FUNC_END(); } -int check_users_mailbox_delete_ns_object(struct mail_user *user, librmb::RadosDovecotCephCfg *config, - librmb::RadosNamespaceManager *ns_mgr, librmb::RadosStorage *storage) { +int check_users_mailbox_delete_ns_object(struct mail_user *user, storage_interface::RadosDovecotCephCfg *config, + storage_interface::RadosNamespaceManager *ns_mgr, storage_interface::RadosStorage *storage) { FUNC_START(); int ret = 0; struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces); diff --git a/src/storage-rbox/rbox-storage.hpp b/src/storage-rbox/rbox-storage.hpp index f1bd7c37..5eb27f88 100644 --- a/src/storage-rbox/rbox-storage.hpp +++ b/src/storage-rbox/rbox-storage.hpp @@ -24,25 +24,22 @@ #define RBOX_MAILDIR_NAME "rbox-Mails" #ifdef __cplusplus -#include "../librmb/rados-cluster-impl.h" -#include "../librmb/rados-storage-impl.h" -#include "../librmb/rados-namespace-manager.h" -#include "../librmb/rados-dovecot-ceph-cfg.h" -#include "../librmb/rados-metadata-storage-impl.h" -#include "../librmb/rados-save-log.h" - +#include "../storage-interface/rados-namespace-manager.h" +#include "../storage-interface/rados-dovecot-ceph-cfg.h" +#include "../storage-interface/rados-metadata-storage.h" +#include "../storage-interface/rados-save-log.h" #include "rbox-storage-struct.h" struct rbox_storage { struct mail_storage storage; - librmb::RadosCluster *cluster; - librmb::RadosStorage *s; - librmb::RadosDovecotCephCfg *config; - librmb::RadosNamespaceManager *ns_mgr; - librmb::RadosMetadataStorage *ms; - librmb::RadosStorage *alt; - librmb::RadosSaveLog *save_log; + storage_interface::RadosCluster *cluster; + storage_interface::RadosStorage *s; + storage_interface::RadosDovecotCephCfg *config; + storage_interface::RadosNamespaceManager *ns_mgr; + storage_interface::RadosMetadataStorage *ms; + storage_interface::RadosStorage *alt; + storage_interface::RadosSaveLog *save_log; uint32_t corrupted_rebuild_count; bool corrupted; @@ -75,7 +72,7 @@ extern int rbox_read_header(struct rbox_mailbox *mbox, struct rbox_index_header extern int rbox_mailbox_create_indexes(struct mailbox *box, const struct mailbox_update *update, struct mail_index_transaction *trans); -extern int check_users_mailbox_delete_ns_object(struct mail_user *user, librmb::RadosDovecotCephCfg *config, - librmb::RadosNamespaceManager *ns_mgr, librmb::RadosStorage *storage); +extern int check_users_mailbox_delete_ns_object(struct mail_user *user, storage_interface::RadosDovecotCephCfg *config, + storage_interface::RadosNamespaceManager *ns_mgr, storage_interface::RadosStorage *storage); #endif // SRC_STORAGE_RBOX_RBOX_STORAGE_HPP_ diff --git a/src/storage-rbox/rbox-sync-rebuild.cpp b/src/storage-rbox/rbox-sync-rebuild.cpp index 1abc4889..75cedea2 100644 --- a/src/storage-rbox/rbox-sync-rebuild.cpp +++ b/src/storage-rbox/rbox-sync-rebuild.cpp @@ -25,23 +25,25 @@ extern "C" { #include "rbox-storage.hpp" #include "rbox-mail.h" #include "encoding.h" -#include "../librmb/rados-mail.h" -#include "../librmb/rados-util.h" -#include "rados-types.h" +#include "../storage-interface/rados-mail.h" +#include "../storage-interface/rados-util.h" +#include "../storage-interface/rados-types.h" +#include "../storage-engine/storage-backend-factory.h" +using storage_interface::rbox_metadata_key; -using librmb::RadosMail; -using librmb::rbox_metadata_key; -int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &oi, librmb::RadosMail *mail_obj, +int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &oi, storage_interface::RadosMail *mail_obj, bool alt_storage, uint32_t next_uid) { FUNC_START(); + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); struct rbox_mailbox *rbox = (struct rbox_mailbox *)ctx->box; char *xattr_mail_uid = NULL; - librmb::RadosUtils::get_metadata(rbox_metadata_key::RBOX_METADATA_MAIL_UID, mail_obj->get_metadata(), + rados_utils->get_metadata(rbox_metadata_key::RBOX_METADATA_MAIL_UID, mail_obj->get_metadata(), &xattr_mail_uid); char *xattr_guid = NULL; - librmb::RadosUtils::get_metadata(rbox_metadata_key::RBOX_METADATA_GUID, mail_obj->get_metadata(), &xattr_guid); + rados_utils->get_metadata(rbox_metadata_key::RBOX_METADATA_GUID, mail_obj->get_metadata(), &xattr_guid); struct mail_storage *storage = ctx->box->storage; struct rbox_storage *r_storage = (struct rbox_storage *)storage; uint32_t seq; @@ -84,44 +86,48 @@ int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &o } // update uid. - librmb::RadosMetadata mail_uid(librmb::RBOX_METADATA_MAIL_UID, next_uid); + storage_interface::RadosMetadata *mail_uid= + storage_engine::StorageBackendFactory::create_metadata_uint( + storage_engine::CEPH, storage_interface::RBOX_METADATA_MAIL_UID, next_uid); + mail_obj->add_metadata(mail_uid); std::string s_oid = *mail_obj->get_oid(); - std::list to_update; + std::list to_update; to_update.push_back(mail_uid); if (!r_storage->ms->get_storage()->update_metadata(s_oid, to_update)) { i_warning("update of MAIL_UID failed: for object: %s , uid: %d", mail_obj->get_oid()->c_str(), next_uid); } - + delete rados_utils; + rados_utils=nullptr; FUNC_END(); return 0; } -std::map> load_rados_mail_metadata( +std::map> load_rados_mail_metadata( bool alt_storage, struct rbox_storage *r_storage, std::set &mail_list) { - std::map> rados_mails; + std::map> rados_mails; std::set::iterator it; - for(it=mail_list.begin(); it!=mail_list.end(); ++it){ - - librmb::RadosMail mail_object; - mail_object.set_oid((*it)); + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + storage_interface::RadosMail *mail_object= + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + mail_object->set_oid((*it)); if (alt_storage) { - r_storage->ms->get_storage()->set_io_ctx(&r_storage->alt->get_io_ctx()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->alt->get_io_ctx_wrapper()); } - - int load_metadata_ret = r_storage->ms->get_storage()->load_metadata(&mail_object); - if (load_metadata_ret < 0 || !librmb::RadosUtils::validate_metadata(mail_object.get_metadata())) { - i_debug("metadata for object : %s is not valid, skipping object ", mail_object.get_oid()->c_str()); + int load_metadata_ret = r_storage->ms->get_storage()->load_metadata(mail_object); + if (load_metadata_ret < 0 || !rados_utils->validate_metadata(mail_object->get_metadata())) { + i_debug("metadata for object : %s is not valid, skipping object ", mail_object->get_oid()->c_str()); continue; } char *mailbox_guid = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, - mail_object.get_metadata(), + rados_utils->get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, + mail_object->get_metadata(), &mailbox_guid ); std::string mails_guid(mailbox_guid); @@ -130,17 +136,18 @@ std::map> load_rados_mail_metadata( rados_mails[mailbox_guid].push_back(mail_object); } else{ - std::list list_mail_objects; + std::list list_mail_objects; list_mail_objects.push_back(mail_object); rados_mails[mailbox_guid]= list_mail_objects; } - + delete rados_utils; + rados_utils=nullptr; } return rados_mails; } // find objects with mailbox_guid 'U' attribute -int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map> &rados_mails, +int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map> &rados_mails, struct rbox_sync_rebuild_ctx *rebuild_ctx) { FUNC_START(); struct mail_storage *storage = ctx->box->storage; @@ -163,29 +170,30 @@ int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map::iterator it; + std::list::iterator it; for(it=rados_mails[mailbox_guid].begin(); it!=rados_mails[mailbox_guid].end(); ++it){ - if(it->is_restored()){ + if((*it)->is_restored()){ // if this is second run, do not add the mail again. - i_debug("skipping already restored mail! oid: %s",it->get_oid()->c_str()); + i_debug("skipping already restored mail! oid: %s",(*it)->get_oid()->c_str()); continue; } sync_add_objects_ret = - rbox_sync_add_object(ctx, *it->get_oid(), &(*it), rebuild_ctx->alt_storage, rebuild_ctx->next_uid); - i_debug("re-adding mail oid:(%s) with uid: %d to mailbox %s (%s) ", it->get_oid()->c_str(), rebuild_ctx->next_uid, mailbox_guid.c_str(), ctx->box->name ); + rbox_sync_add_object(ctx, *(*it)->get_oid(), *it, rebuild_ctx->alt_storage, rebuild_ctx->next_uid); + i_debug("re-adding mail oid:(%s) with uid: %d to mailbox %s (%s) ", (*it)->get_oid()->c_str(), rebuild_ctx->next_uid, mailbox_guid.c_str(), ctx->box->name ); if (sync_add_objects_ret < 0) { - i_error("sync_add_object: oid(%s), alt_storage(%d),uid(%d)", it->get_oid()->c_str(), + i_error("sync_add_object: oid(%s), alt_storage(%d),uid(%d)", (*it)->get_oid()->c_str(), rebuild_ctx->alt_storage, rebuild_ctx->next_uid); break; } - it->set_restored(true); - it->set_index_ref(true); - i_debug("restored rados_mail: %s",it->to_string(" ").c_str()); + (*it)->set_restored(true); + (*it)->set_index_ref(true); + i_debug("restored rados_mail: %s",(*it)->to_string(" ").c_str()); rebuild_ctx->next_uid++; + /*TO DO::IF it is required to delete the RadosMail pointer you should continue path from here.*/ } if (sync_add_objects_ret < 0) { @@ -231,7 +239,7 @@ void rbox_sync_update_header(struct index_rebuild_context *ctx) { FUNC_END(); } -int rbox_sync_index_rebuild_objects(struct index_rebuild_context *ctx, std::map> &rados_mails) { +int rbox_sync_index_rebuild_objects(struct index_rebuild_context *ctx, std::map> &rados_mails) { FUNC_START(); int ret = 0; @@ -260,7 +268,7 @@ int rbox_storage_rebuild_in_context(struct rbox_storage *r_storage, bool force, FUNC_START(); struct mail_user *user = r_storage->storage.user; - std::map> rados_mails; + std::map> rados_mails; struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces); @@ -277,15 +285,14 @@ int rbox_storage_rebuild_in_context(struct rbox_storage *r_storage, bool force, } i_info("Repair done checking for unassigned mails "); - std::map>::iterator it; + std::map>::iterator it; long count_not_assigned = 0; long count_assigned = 0; for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){ - std::list::iterator list_it; + std::list::iterator list_it; for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){ - count_not_assigned += list_it->is_restored() ? 0 : 1; - count_assigned += list_it->is_restored() ? 1 : 0; - + count_not_assigned += (*list_it)->is_restored() ? 0 : 1; + count_assigned += (*list_it)->is_restored() ? 1 : 0; } } i_info("total unassigned mails %ld", count_not_assigned); @@ -300,47 +307,85 @@ int rbox_storage_rebuild_in_context(struct rbox_storage *r_storage, bool force, if(find_default_mailbox_guid(ns_second, &last_known_mailbox_guid) < 0){ // bad no default mailbox found i_warning("unable to find inbox guid => unreferenced mails can not automatically be re-assigned "); + for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){ + std::list::iterator list_it; + for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){ + delete *list_it; + *list_it=nullptr; + } + } return 0; } int unassigned_counter = 0; for(it=rados_mails.begin(); it!=rados_mails.end(); ++it) { - std::list::iterator list_it; + std::list::iterator list_it; for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){ - if(list_it->is_restored()){ + if((*list_it)->is_restored()){ continue; } - librmb::RadosMetadata metadata; - metadata.convert(rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, last_known_mailbox_guid); + storage_interface::RadosMetadata *metadata=storage_engine::StorageBackendFactory::create_metadata_default( + storage_engine::CEPH); + metadata->convert(rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, last_known_mailbox_guid); - librmb::RadosMetadata metadata_uid; - metadata_uid.convert(rbox_metadata_key::RBOX_METADATA_MAIL_UID, INT32_MAX); + storage_interface::RadosMetadata *metadata_uid=storage_engine::StorageBackendFactory::create_metadata_default( + storage_engine::CEPH); + metadata_uid->convert(rbox_metadata_key::RBOX_METADATA_MAIL_UID, INT32_MAX); librados::ObjectWriteOperation write_mail_uid; - write_mail_uid.setxattr(metadata_uid.key.c_str(), metadata_uid.bl); - write_mail_uid.setxattr(metadata.key.c_str(), metadata.bl); - - if (r_storage->s->get_io_ctx().operate(*list_it->get_oid(), &write_mail_uid) < 0) { + (*list_it)->add_metadata(metadata_uid); + void* bl_ptr_uid=(void*)metadata_uid->get_buffer(); + ceph::bufferlist bl_uid=*(ceph::bufferlist*)bl_ptr_uid; + write_mail_uid.setxattr(metadata_uid->get_key().c_str(), bl_uid); + (*list_it)->add_metadata(metadata); + void* bl_ptr=(void*)metadata->get_buffer(); + ceph::bufferlist bl=*(ceph::bufferlist*)bl_ptr; + write_mail_uid.setxattr(metadata->get_key().c_str(), bl); + + if (r_storage->s->get_io_ctx_wrapper()->operate(*(*list_it)->get_oid(), &write_mail_uid) < 0) { i_debug("Unable to reset metadata to guid : %s",last_known_mailbox_guid.c_str()); }else { - i_debug("(%d) Mailbox guid for mail (oid=%s) restored to %s (INBOX) => re-run force-resync to assign them ",unassigned_counter, list_it->get_oid()->c_str(),last_known_mailbox_guid.c_str()); + i_debug("(%d) Mailbox guid for mail (oid=%s) restored to %s (INBOX) => re-run force-resync to assign them ",unassigned_counter, (*list_it)->get_oid()->c_str(),last_known_mailbox_guid.c_str()); } unassigned_counter++; - list_it->set_lost_object(true); + (*list_it)->set_lost_object(true); } } if(unassigned_counter > 0){ if(firstTry){ // try again.... but only once, as we do not want to end up in an endless loop. - return rbox_storage_rebuild_in_context(r_storage,force, false); + int ret = 0; + ret= rbox_storage_rebuild_in_context(r_storage,force, false); + for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){ + std::list::iterator list_it; + for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){ + delete *list_it; + *list_it=nullptr; + } + } + return ret; }else{ i_error("still unassigned mail objects in ceph namespace, manual intervention required."); + for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){ + std::list::iterator list_it; + for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){ + delete *list_it; + *list_it=nullptr; + } + } return -1; } } } - + + for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){ + std::list::iterator list_it; + for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){ + delete *list_it; + *list_it=nullptr; + } + } FUNC_END(); return 0; } @@ -402,7 +447,7 @@ void cb(std::string &pg){ i_debug("processing: %s",pg.c_str()); } -int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage *r_storage, std::map> &rados_mails) { +int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage *r_storage, std::map> &rados_mails) { FUNC_START(); const struct mailbox_info *info; @@ -445,13 +490,15 @@ int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage if( r_storage->config->get_object_search_method() == 1) { //all mail first end up in the inbox - librmb::RadosMetadata filter(rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, "INBOX"); + storage_interface::RadosMetadata *filter= + storage_engine::StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, "INBOX"); long milli_time, seconds, useconds; struct timeval start_time, end_time; gettimeofday(&start_time, NULL); - mail_list = r_storage->s->find_mails_async(&filter, + mail_list = r_storage->s->find_mails_async(filter, pool_name, r_storage->config->get_object_search_threads(), &cb); @@ -461,6 +508,8 @@ int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage milli_time = ((seconds) * 1000 + useconds/1000.0); i_debug("multithreading done : took: %ld ms", (milli_time)); + delete filter; + filter=nullptr; } else if( r_storage->config->get_object_search_method() == 2){ @@ -480,18 +529,14 @@ int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage i_debug("processing ceph index done : took: %ld ms", (milli_time)); } else { - librados::NObjectIterator iter_guid = r_storage->s->find_mails(nullptr); - while (iter_guid != librados::NObjectIterator::__EndObjectIterator) { - mail_list.insert((*iter_guid).get_oid()); - iter_guid++; - } + mail_list = r_storage->s->find_mails(nullptr); } i_info("Loading mail metadata..."); rados_mails = load_rados_mail_metadata(false,r_storage, mail_list); i_info("Mails completely loaded "); #ifdef DEBUG - std::map>::iterator it; + std::map>::iterator it; for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){ i_debug("Found mails for mailbox_guid: %s: mails : %ld", it->first.c_str(), it->second.size()); } @@ -500,9 +545,9 @@ int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage if( r_storage->config->get_object_search_method() == 2){ //TODO: make this more efficient : restore the valid objects std::set valid_objects; - for(std::map>::iterator boxes = rados_mails.begin(); boxes != rados_mails.end(); ++boxes) { - for (librmb::RadosMail const& m : boxes->second) { - valid_objects.insert(*m.get_oid()); + for(std::map>::iterator boxes = rados_mails.begin(); boxes != rados_mails.end(); ++boxes) { + for (storage_interface::RadosMail* const m : boxes->second) { + valid_objects.insert(*m->get_oid()); } } @@ -533,7 +578,7 @@ int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage return ret; } -int rbox_sync_index_rebuild(struct rbox_mailbox *rbox, bool force, std::map> &rados_mails) { +int rbox_sync_index_rebuild(struct rbox_mailbox *rbox, bool force, std::map> &rados_mails) { struct index_rebuild_context *ctx; struct mail_index_view *view; struct mail_index_transaction *trans; @@ -577,6 +622,7 @@ int rbox_sync_index_rebuild(struct rbox_mailbox *rbox, bool force, std::mapstorage->corrupted_rebuild_count = 0; mail_index_view_close(&view); + /*TO DO if there is a memoryleak problem delete RadosMail object here*/ FUNC_END(); return ret; } diff --git a/src/storage-rbox/rbox-sync-rebuild.h b/src/storage-rbox/rbox-sync-rebuild.h index 16bed509..2c0a8e9d 100644 --- a/src/storage-rbox/rbox-sync-rebuild.h +++ b/src/storage-rbox/rbox-sync-rebuild.h @@ -17,7 +17,7 @@ #include #include -#include "../librmb/rados-mail.h" +#include "../storage-interface/rados-mail.h" extern "C" { #include "index-rebuild.h" @@ -29,21 +29,19 @@ struct rbox_sync_rebuild_ctx { }; extern void rbox_sync_update_header(struct index_rebuild_context *ctx); -extern int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &oi, librmb::RadosMail *mail_obj, +extern int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &oi, storage_interface::RadosMail *mail_obj, bool alt_storage, uint32_t next_uid); -extern int rbox_sync_index_rebuild(struct index_rebuild_context *ctx, std::map> &rados_mails, - struct rbox_sync_rebuild_ctx *rebuild_ctx); extern void rbox_sync_set_uidvalidity(struct index_rebuild_context *ctx); -extern int rbox_sync_index_rebuild_objects(struct index_rebuild_context *ctx, std::map> &rados_mails); -extern int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map> &rados_mails, +extern int rbox_sync_index_rebuild_objects(struct index_rebuild_context *ctx, std::map> &rados_mails); +extern int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map> &rados_mails, struct rbox_sync_rebuild_ctx *rebuild_ctx); -extern int rbox_sync_index_rebuild(struct rbox_mailbox *rbox, bool force, std::map> &rados_mails); +extern int rbox_sync_index_rebuild(struct rbox_mailbox *rbox, bool force, std::map> &rados_mails); extern int rbox_storage_rebuild_in_context(struct rbox_storage *r_storage, bool force, bool firstTry); -extern int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage *r_storage, std::map> &rados_mails); +extern int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage *r_storage, std::map> &rados_mails); -extern std::map> load_rados_mail_metadata(bool alt_storage, struct rbox_storage *r_storage, std::list &mail_list); +extern std::map> load_rados_mail_metadata(bool alt_storage, struct rbox_storage *r_storage, std::list &mail_list); extern int find_default_mailbox_guid(struct mail_namespace *ns, std::string *mailbox_guid); extern int find_inbox_mailbox_guid(struct mail_namespace *ns, std::string *mailbox_guid); diff --git a/src/storage-rbox/rbox-sync.cpp b/src/storage-rbox/rbox-sync.cpp index 3374b99c..4b33e9cb 100644 --- a/src/storage-rbox/rbox-sync.cpp +++ b/src/storage-rbox/rbox-sync.cpp @@ -21,10 +21,11 @@ extern "C" { #include "rbox-sync.h" #include "debug-helper.h" } -#include "rados-util.h" +#include "../storage-interface/rados-util.h" #include "rbox-storage.hpp" #include "rbox-mail.h" #include "rbox-sync-rebuild.h" +#include "../storage-engine/storage-backend-factory.h" #define RBOX_REBUILD_COUNT 3 @@ -104,8 +105,8 @@ static int update_extended_metadata(struct rbox_sync_context *ctx, uint32_t seq1 } r_storage->ms->get_storage()->set_io_ctx(alt_storage ? - &r_storage->alt->get_io_ctx() : - &r_storage->s->get_io_ctx() ); + r_storage->alt->get_io_ctx_wrapper(): + r_storage->s->get_io_ctx_wrapper()); guid_128_t index_oid; if (rbox_get_oid_from_index(ctx->sync_view, seq1, ((struct rbox_mailbox *)box)->ext_id, &index_oid) >= 0) { @@ -124,8 +125,12 @@ static int update_extended_metadata(struct rbox_sync_context *ctx, uint32_t seq1 continue; } std::string key_value = keywords[keyword_idx]; - librmb::RadosMetadata ext_metata(ext_key, key_value); - ret = r_storage->ms->get_storage()->update_keyword_metadata(key_oid, &ext_metata); + storage_interface::RadosMetadata *ext_metata= + storage_engine::StorageBackendFactory::create_metadata_str_key_val( + storage_engine::CEPH, ext_key, key_value); + ret = r_storage->ms->get_storage()->update_keyword_metadata(key_oid, ext_metata); + delete ext_metata; + ext_metata=nullptr; } if (ret < 0) { break; @@ -133,7 +138,7 @@ static int update_extended_metadata(struct rbox_sync_context *ctx, uint32_t seq1 } } // reset metadatas storage - r_storage->ms->get_storage()->set_io_ctx(&r_storage->s->get_io_ctx()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->s->get_io_ctx_wrapper()); FUNC_END(); return ret; } @@ -151,7 +156,9 @@ static int move_to_alt(struct rbox_sync_context *ctx, uint32_t seq1, uint32_t se guid_128_t index_oid; if (rbox_get_oid_from_index(ctx->sync_view, seq1, ((struct rbox_mailbox *)&ctx->rbox->box)->ext_id, &index_oid) >= 0) { std::string oid = guid_128_to_string(index_oid); - ret = librmb::RadosUtils::move_to_alt(oid, r_storage->s, r_storage->alt, r_storage->ms, inverse); + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + ret = rados_utils->move_to_alt(oid, r_storage->s, r_storage->alt, r_storage->ms, inverse); if (ret >= 0) { if (inverse) { mail_index_update_flags(ctx->trans, seq1, MODIFY_REMOVE, (enum mail_flags)RBOX_INDEX_FLAG_ALT); @@ -159,6 +166,8 @@ static int move_to_alt(struct rbox_sync_context *ctx, uint32_t seq1, uint32_t se mail_index_update_flags(ctx->trans, seq1, MODIFY_ADD, (enum mail_flags)RBOX_INDEX_FLAG_ALT); } } + delete rados_utils; + rados_utils = nullptr; } } return ret; @@ -189,43 +198,55 @@ static int update_flags(struct rbox_sync_context *ctx, uint32_t seq1, uint32_t s } r_storage->ms->get_storage()->set_io_ctx( alt_storage ? - &r_storage->alt->get_io_ctx() : - &r_storage->s->get_io_ctx() ); + r_storage->alt->get_io_ctx_wrapper(): + r_storage->s->get_io_ctx_wrapper()); guid_128_t index_oid; if (rbox_get_oid_from_index(ctx->sync_view, seq1, ((struct rbox_mailbox *)box)->ext_id, &index_oid) >= 0) { const char *oid = guid_128_to_string(index_oid); - librmb::RadosMail mail_object; - mail_object.set_oid(oid); - if (r_storage->ms->get_storage()->load_metadata(&mail_object) < 0) { + storage_interface::RadosMail *mail_object= + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + mail_object->set_oid(oid); + if (r_storage->ms->get_storage()->load_metadata(mail_object) < 0) { i_error("update_flags: load_metadata failed! for %d, oid(%s)", seq1, oid); continue; } char *flags_metadata = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail_object.get_metadata(), &flags_metadata); + storage_interface::RadosUtils *rados_utils= + storage_engine::StorageBackendFactory::create_rados_utils(storage_engine::CEPH); + rados_utils->get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, mail_object->get_metadata(), &flags_metadata); uint8_t flags = 0x0; - if (librmb::RadosUtils::string_to_flags(flags_metadata, &flags)) { + if (rados_utils->string_to_flags(flags_metadata, &flags)) { if (add_flags != 0) { flags |= add_flags; } if (remove_flags != 0) { flags &= ~remove_flags; } + std::list to_update; std::string str_flags_metadata; - if (librmb::RadosUtils::flags_to_string(flags, &str_flags_metadata)) { - librmb::RadosMetadata update(librmb::RBOX_METADATA_OLDV1_FLAGS, str_flags_metadata); - ret = r_storage->ms->get_storage()->set_metadata(&mail_object, update); + if (rados_utils->flags_to_string(flags, &str_flags_metadata)) { + storage_interface::RadosMetadata *update= + storage_engine::StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, storage_interface::RBOX_METADATA_OLDV1_FLAGS, str_flags_metadata); + mail_object->add_metadata(update); + to_update.push_back(update); + ret = r_storage->ms->get_storage()->update_metadata(*mail_object->get_oid(), to_update); if (ret < 0) { i_warning("updating metadata for object : oid(%s), seq (%d) failed with ceph errorcode: %d", - mail_object.get_oid()->c_str(), seq1, ret); + mail_object->get_oid()->c_str(), seq1, ret); } } } + delete mail_object; + mail_object=nullptr; + delete rados_utils; + rados_utils=nullptr; } } // reset metadata storage - r_storage->ms->get_storage()->set_io_ctx(&r_storage->s->get_io_ctx()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->s->get_io_ctx_wrapper()); FUNC_END(); return ret; } @@ -284,9 +305,9 @@ static int rbox_sync_index(struct rbox_sync_context *ctx) { if (ret < 0) { i_error("Error moving seq (%d) to alt storage", seq1); } - } else if (r_storage->config->is_mail_attribute(librmb::RBOX_METADATA_OLDV1_FLAGS) && + } else if (r_storage->config->is_mail_attribute(storage_interface::RBOX_METADATA_OLDV1_FLAGS) && r_storage->config->is_update_attributes() && - r_storage->config->is_updateable_attribute(librmb::RBOX_METADATA_OLDV1_FLAGS)) { + r_storage->config->is_updateable_attribute(storage_interface::RBOX_METADATA_OLDV1_FLAGS)) { if (update_flags(ctx, seq1, seq2, sync_rec.add_flags, sync_rec.remove_flags) < 0) { i_error("Error updating flags seq (%d)", seq1); } @@ -294,9 +315,9 @@ static int rbox_sync_index(struct rbox_sync_context *ctx) { break; case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: - if (r_storage->config->is_mail_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS) && + if (r_storage->config->is_mail_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS) && r_storage->config->is_update_attributes() && - r_storage->config->is_updateable_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS)) { + r_storage->config->is_updateable_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS)) { // sync_rec.keyword_idx; if (update_extended_metadata(ctx, seq1, seq2, sync_rec.keyword_idx, false) < 0) { return -1; @@ -304,9 +325,9 @@ static int rbox_sync_index(struct rbox_sync_context *ctx) { } break; case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: - if (r_storage->config->is_mail_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS) && + if (r_storage->config->is_mail_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS) && r_storage->config->is_update_attributes() && - r_storage->config->is_updateable_attribute(librmb::RBOX_METADATA_OLDV1_KEYWORDS)) { + r_storage->config->is_updateable_attribute(storage_interface::RBOX_METADATA_OLDV1_KEYWORDS)) { /* FIXME: should be bother calling sync_notify()? */ // sync_rec.keyword_idx if (update_extended_metadata(ctx, seq1, seq2, sync_rec.keyword_idx, true) < 0) { @@ -451,13 +472,13 @@ static int rbox_sync_object_expunge(struct rbox_sync_context *ctx, struct expung FUNC_END(); return ret_remove; } - librmb::RadosStorage *rados_storage = item->alt_storage ? r_storage->alt : r_storage->s; - ret_remove = rados_storage->get_io_ctx().remove(oid); + storage_interface::RadosStorage *rados_storage = item->alt_storage ? r_storage->alt : r_storage->s; + ret_remove = rados_storage->get_io_ctx_wrapper()->get_io_ctx().remove(oid); if (ret_remove < 0) { if(ret_remove == -ETIMEDOUT) { int max_retry = 10; for(int i = 0;iget_io_ctx().remove(oid); + ret_remove = rados_storage->get_io_ctx_wrapper()->get_io_ctx().remove(oid); if(ret_remove >=0){ i_error("rbox_sync connection timeout during oid (%s) deletion, mail stays in object store.",oid); break; @@ -528,7 +549,6 @@ int rbox_sync_finish(struct rbox_sync_context **_ctx, bool success) { } else { mail_index_sync_rollback(&ctx->index_sync_ctx); } - i_info("EXPUNGE: calling deinit"); index_storage_expunging_deinit(&ctx->rbox->box); if (array_is_created(&ctx->expunged_items)) { diff --git a/src/tests/Makefile.am b/src/tests/Makefile.am index d90a281c..408e7bd2 100644 --- a/src/tests/Makefile.am +++ b/src/tests/Makefile.am @@ -55,17 +55,14 @@ gtest_shlibs = libgmockgtest.la TESTS = test_rmb test_rmb_SOURCES = rmb/test_rmb.cpp mocks/mock_test.h -test_rmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/rmb-commands.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) +test_rmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser_impl.o $(top_builddir)/src/librmb/tools/rmb/rmb-commands-impl.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools_impl.o $(gtest_shlibs) + TESTS += test_storage_mock_rbox test_storage_mock_rbox_SOURCES = storage-mock-rbox/test_storage_mock_rbox.cpp storage-mock-rbox/TestCase.cpp storage-mock-rbox/TestCase.h mocks/mock_test.h test-utils/it_utils.cpp test-utils/it_utils.h test_storage_mock_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) test_storage_mock_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) - -TESTS += test_storage_mock_rbox_bugs -test_storage_mock_rbox_bugs_SOURCES = storage-mock-rbox/test_storage_mock_rbox_bugs.cpp storage-mock-rbox/TestCase.cpp storage-mock-rbox/TestCase.h mocks/mock_test.h test-utils/it_utils.cpp test-utils/it_utils.h -test_storage_mock_rbox_bugs_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -test_storage_mock_rbox_bugs_LDADD = $(storage_shlibs) $(gtest_shlibs) + TESTS += test_repair_rbox test_repair_rbox_SOURCES = storage-mock-rbox/test_repair_rbox.cpp storage-mock-rbox/TestCase.cpp storage-mock-rbox/TestCase.h mocks/mock_test.h test-utils/it_utils.cpp test-utils/it_utils.h @@ -75,29 +72,27 @@ test_repair_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) TESTS += test_librmb_utils test_librmb_utils_SOURCES = librmb/test_librmb_utils.cpp -test_librmb_utils_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) +test_librmb_utils_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser_impl.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools_impl.o $(gtest_shlibs) if BUILD_INTEGRATION_TESTS TESTS += it_test_librmb it_test_librmb_SOURCES = librmb/it_test_librmb.cpp -it_test_librmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/rmb-commands.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) +it_test_librmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser_impl.o $(top_builddir)/src/librmb/tools/rmb/rmb-commands-impl.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools_impl.o $(gtest_shlibs) -TESTS += it_test_dict_rados -it_test_dict_rados_SOURCES = dict-rados/it_test_dict_rados.cpp dict-rados/TestCase.cpp dict-rados/TestCase.h -it_test_dict_rados_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_dict_rados_LDADD = $(dict_shlibs) $(gtest_shlibs) + TESTS += it_test_storage_rbox it_test_storage_rbox_SOURCES = storage-rbox/it_test_storage_rbox.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h it_test_storage_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_storage_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_storage_rbox_2 -it_test_storage_rbox_2_SOURCES = storage-rbox/it_test_storage_rbox.cpp storage-rbox2/TestCase.cpp storage-rbox2/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_storage_rbox_2_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_storage_rbox_2_LDADD = $(storage_shlibs) $(gtest_shlibs) +# storage_rbox_2 uses same code as storage-rbox but uses write chunks which is tested with the above testcase. +#TESTS += it_test_storage_rbox_2 +#it_test_storage_rbox_2_SOURCES = storage-rbox/it_test_storage_rbox.cpp storage-rbox2/TestCase.cpp storage-rbox2/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h +#it_test_storage_rbox_2_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) +#it_test_storage_rbox_2_LDADD = $(storage_shlibs) $(gtest_shlibs) TESTS += it_test_copy_rbox @@ -105,15 +100,7 @@ it_test_copy_rbox_SOURCES = storage-rbox/it_test_copy_rbox.cpp storage-rbox/Test it_test_copy_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_copy_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_check_metadata -it_test_check_metadata_SOURCES = storage-rbox/it_test_check_metadata.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_check_metadata_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_check_metadata_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_copy_rbox_alt -it_test_copy_rbox_alt_SOURCES = storage-rbox/it_test_copy_rbox_alt.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_copy_rbox_alt_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_copy_rbox_alt_LDADD = $(storage_shlibs) $(gtest_shlibs) TESTS += it_test_copy_rbox_fail it_test_copy_rbox_fail_SOURCES = storage-rbox/it_test_copy_rbox_fail.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h @@ -125,30 +112,25 @@ it_test_move_rbox_SOURCES = storage-rbox/it_test_move_rbox.cpp storage-rbox/Test it_test_move_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_move_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_move_rbox_alt -it_test_move_rbox_alt_SOURCES = storage-rbox/it_test_move_rbox_alt.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_move_rbox_alt_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_move_rbox_alt_LDADD = $(storage_shlibs) $(gtest_shlibs) + TESTS += it_test_lda_rbox it_test_lda_rbox_SOURCES = storage-rbox/it_test_lda_rbox.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h it_test_lda_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_lda_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_doveadm_backup_rbox -it_test_doveadm_backup_rbox_SOURCES = storage-rbox/it_test_doveadm_backup_rbox.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_doveadm_backup_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_doveadm_backup_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) + TESTS += it_test_read_mail_rbox it_test_read_mail_rbox_SOURCES = storage-rbox/it_test_read_mail_rbox.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h it_test_read_mail_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_read_mail_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_read_mail_rbox_alt -it_test_read_mail_rbox_alt_SOURCES = storage-rbox/it_test_read_mail_rbox_alt.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_read_mail_rbox_alt_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_read_mail_rbox_alt_LDADD = $(storage_shlibs) $(gtest_shlibs) +#NOTE: ALT STORAGE is not really supported +#TESTS += it_test_read_mail_rbox_alt +#it_test_read_mail_rbox_alt_SOURCES = storage-rbox/it_test_read_mail_rbox_alt.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h +#it_test_read_mail_rbox_alt_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) +#it_test_read_mail_rbox_alt_LDADD = $(storage_shlibs) $(gtest_shlibs) TESTS += it_test_read_mail_failed_rbox it_test_read_mail_failed_rbox_SOURCES = storage-rbox/it_test_read_mail_failed_rbox.cpp storage-rbox/TestCase.cpp storage-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h @@ -165,10 +147,7 @@ it_test_sync_rbox_2_SOURCES = sync-rbox/it_test_sync_rbox_2.cpp sync-rbox/TestCa it_test_sync_rbox_2_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_sync_rbox_2_LDADD = $(storage_shlibs) $(gtest_shlibs) -TESTS += it_test_sync_rbox_alt -it_test_sync_rbox_alt_SOURCES = sync-rbox/it_test_sync_rbox_alt.cpp sync-rbox/TestCase.cpp sync-rbox/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_sync_rbox_alt_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_sync_rbox_alt_LDADD = $(storage_shlibs) $(gtest_shlibs) + TESTS += it_test_sync_rbox_duplicate_uid @@ -176,16 +155,12 @@ it_test_sync_rbox_duplicate_uid_SOURCES = sync-rbox/it_test_sync_rbox_duplicate_ it_test_sync_rbox_duplicate_uid_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) it_test_sync_rbox_duplicate_uid_LDADD = $(storage_shlibs) $(gtest_shlibs) + -TESTS += it_test_doveadm_rmb -it_test_doveadm_rmb_SOURCES = doveadm-rmb/it_test_doveadm_rmb.cpp doveadm-rmb/TestCase.cpp doveadm-rmb/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_doveadm_rmb_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_doveadm_rmb_LDADD = $(storage_shlibs) $(gtest_shlibs) $(doveadm_rmb_shlibs) - -TESTS += it_test_backup -it_test_backup_SOURCES = backup/it_test_backup_rbox.cpp backup/TestCase.cpp backup/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h -it_test_backup_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -it_test_backup_LDADD = $(storage_shlibs) $(gtest_shlibs) $(doveadm_rmb_shlibs) +#TESTS += it_test_backup +#it_test_backup_SOURCES = backup/it_test_backup_rbox.cpp backup/TestCase.cpp backup/TestCase.h test-utils/it_utils.cpp test-utils/it_utils.h +#it_test_backup_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) +#it_test_backup_LDADD = $(storage_shlibs) $(gtest_shlibs) $(doveadm_rmb_shlibs) endif diff --git a/src/tests/dict-rados/it_test_dict_rados.cpp b/src/tests/dict-rados/it_test_dict_rados.cpp index 6e66fc17..f2c9e4d4 100644 --- a/src/tests/dict-rados/it_test_dict_rados.cpp +++ b/src/tests/dict-rados/it_test_dict_rados.cpp @@ -26,7 +26,6 @@ extern "C" { #include "dict.h" #include "dict-private.h" #include "ioloop.h" - #include "libdict-rados-plugin.h" } @@ -94,6 +93,7 @@ TEST_F(DictTest, iterate) { ASSERT_EQ(dict_transaction_commit(&ctx, &error_r), 1); struct dict_iterate_context *iter = dict_iterate_init_multiple(target, OMAP_ITERATE_KEY, dict_iterate_flags(0)); + i = 0; const char *kr; diff --git a/src/tests/docs/README.md b/src/tests/docs/README.md new file mode 100644 index 00000000..d9e05cea --- /dev/null +++ b/src/tests/docs/README.md @@ -0,0 +1,94 @@ +The document describes possible tools to test the dovecot-ceph-plugin project manually. +====================== +### 1: smtp test to test saving a mail +The directory path in the dovecot container where we should run the related command is /repo. After a success message from the server you can check ceph pools to check added objects. + The command is : + + smtp-source -v -L -s 1 -m 1 -c -F /root/lmtp_test_mail.tld -f test@example.com -t t1 inet:127.0.0.1:1024 + +### 2: imaptest to test different operations such as copying mails +The directory path in the dovecot container where we should run the related command is /usr/local/bin. + The command is : + + ./imaptest user=t%d pass=t port=10143 users=100 clients=25 error_quit secs=30 output=/var/log/imaptest.log + +There are two files as the results of the test one in the output path /var/log/imaptest.log and the other one in /var/log/dovecot.log +#### 2_1: expectation +imaptest.log must include some positive values(not zero) determining the number of different transactions such as copied mail ,... + If you check dovecot.log you can follow the dovecot-ceph-plugin progress. The file must return 0 for all following checks. (remove -c from following commands to see where the problem is and what is wrongly happening). + + - cat /var/log/dovecot.log | grep -c \"Error:\" + - cat /var/log/dovecot.log | grep -c \"failed:\" + - cat /var/log/dovecot.log | grep -c \"Internalerror\" + - cat /var/log/dovecot.log | grep -c \"killed\" + - cat /var/log/dovecot.log | grep -c \"Panic:\" + - cat /var/log/dovecot.log | grep -c \"Fatal:\" + - cat /var/log/imaptest.log | grep -c \"BUG:\" + +you can remove dedicated period for test "secs=30" and monitor each mentioned command result in this way: + + tail -f /var/log/dovecot.log | grep \"Error:\" + +### 3: thunderbird Mail to test each operation on app +You are just needed to do some settings in section of Account Setting of the application as follow image.You can monitor the process and search for possible issues as it is explained in part 2_1(dovecot.log file). Please consider in the current test environment there are 100 users from t1 - t100 and their all password is t. +- ![Screenshot](img/thunderbird_AccountSetting.png) + +### 4: telnet + +Exec into the container and + Connect to the IMAP server using Telnet. Replace mail.example.com with the hostname or IP address of your IMAP server, and 143 with the appropriate port number if necessary. + + telnet localhost 10143 + +Once connected, you need to authenticate. Use the LOGIN command with your username and password. Replace username and password with your actual credentials. + + A LOGIN t1 t + +If the authentication is successful, the server will respond with a success message, and you can proceed with other commands. You can monitor the process and search possible issues as it is explained in part 2_1(dovecot.log file). In the following there are a few examples of commands: +##### 4_1: List all mailboxes: + + A LIST "" "*" + +##### 4_2: Select a mailbox (e.g., INBOX): +sql + + A SELECT INBOX + +##### 4_3: Fetch the number of messages in the mailbox: +java + + A STATUS INBOX (MESSAGES) + +##### 4_4: Fetch the first five message headers/ Fetch all messages: +sql + + A FETCH 1:5 (BODY[HEADER]) + + A FETCH 1:* BODY[] + +##### 4_5: Store the first five messages with new flag(e.g., deleted): + + A STORE 1:5 -FLAGS (\deleted) +##### 4_6: Expunge deleted messages: + + Expunge +##### 4_7: Move/Copy selected messages from current selected mailbox to another mailbox(e.g., from INBOX to SPAM) + + MOVE 1:* + COPY 1:5 +##### 4_8: There is the possibility to use doveadm commands to work on IMAP server + + the tutorial link: https://wiki2.dovecot.org/Tools/Doveadm/Mailbox +##### 4_9: rados is a utility for interacting with a Ceph object storage: + the tutorial link: https://docs.ceph.com/en/latest/man/8/rados/ + an example command to save mails: rados -p mail_storage ls - -n t1_u +##### 4_10: Logout and close the connection: + + LOGOUT + +Note: IMAP commands are case-insensitive, so you can use either uppercase or lowercase letters. + +Remember to replace mail.example.com with the actual IMAP server address and modify the commands according to your specific needs. Additionally, some servers may use SSL/TLS encryption, so you may need to use the STARTTLS command to establish a secure connection before authenticating. + +### 5: integration tests +There are some test files including unit tests to test different operations for example: ./it_test_read_mail_rbox to check different cases of reading a mail. You can run these files on path /repo/src/tests of dovecot container. \ No newline at end of file diff --git a/src/tests/docs/img/thunderbird_AccountSetting.png b/src/tests/docs/img/thunderbird_AccountSetting.png new file mode 100644 index 00000000..3e4d7592 Binary files /dev/null and b/src/tests/docs/img/thunderbird_AccountSetting.png differ diff --git a/src/tests/failed b/src/tests/failed new file mode 100644 index 00000000..e69de29b diff --git a/src/tests/it_test_check_metadata b/src/tests/it_test_check_metadata new file mode 100755 index 00000000..6e0bdbad --- /dev/null +++ b/src/tests/it_test_check_metadata @@ -0,0 +1,228 @@ +#! /bin/sh + +# it_test_check_metadata - temporary wrapper script for .libs/it_test_check_metadata +# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2 +# +# The it_test_check_metadata program cannot be directly executed until all the libtool +# libraries that it depends on are installed. +# +# This wrapper script should never be moved out of the build directory. +# If it is, it will not operate correctly. + +# Sed substitution that helps us do robust quoting. It backslashifies +# metacharacters that are still active within double-quoted strings. +sed_quote_subst='s|\([`"$\\]\)|\\\1|g' + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac +fi +BIN_SH=xpg4; export BIN_SH # for Tru64 +DUALCASE=1; export DUALCASE # for MKS sh + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +relink_command="(cd /repo/src/tests; { test -z \"\${LIBRARY_PATH+set}\" || unset LIBRARY_PATH || { LIBRARY_PATH=; export LIBRARY_PATH; }; }; { test -z \"\${COMPILER_PATH+set}\" || unset COMPILER_PATH || { COMPILER_PATH=; export COMPILER_PATH; }; }; { test -z \"\${GCC_EXEC_PREFIX+set}\" || unset GCC_EXEC_PREFIX || { GCC_EXEC_PREFIX=; export GCC_EXEC_PREFIX; }; }; { test -z \"\${LD_RUN_PATH+set}\" || unset LD_RUN_PATH || { LD_RUN_PATH=; export LD_RUN_PATH; }; }; { test -z \"\${LD_LIBRARY_PATH+set}\" || unset LD_LIBRARY_PATH || { LD_LIBRARY_PATH=; export LD_LIBRARY_PATH; }; }; PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; export PATH; g++ -std=gnu++11 -pthread -fpermissive -std=c++11 -Wmissing-declarations -g -O0 -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-undef -Wno-redundant-decls -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-error=undef -Wno-error=redundant-decls -fpermissive -std=c++11 -fPIC -DPIC -o \$progdir/\$file storage-rbox/it_test_check_metadata-it_test_check_metadata.o storage-rbox/it_test_check_metadata-TestCase.o test-utils/it_test_check_metadata-it_utils.o ../../src/storage-rbox/.libs/libstorage_rbox_plugin.so /usr/local/lib/dovecot/libdovecot-storage.so -L/usr/local/lib/dovecot /usr/local/lib/dovecot/libdovecot.so ../../src/librmb/.libs/librmb.so ./.libs/libgmockgtest.a -lrados -pthread -Wl,-rpath -Wl,/repo/src/storage-rbox/.libs -Wl,-rpath -Wl,/usr/local/lib/dovecot -Wl,-rpath -Wl,/repo/src/librmb/.libs -Wl,-rpath -Wl,/usr/local/lib/dovecot -Wl,-rpath -Wl,/usr/local/lib)" + +# This environment variable determines our operation mode. +if test "$libtool_install_magic" = "%%%MAGIC variable%%%"; then + # install mode needs the following variables: + generated_by_libtool_version='2.4.6' + notinst_deplibs=' ../../src/storage-rbox/libstorage_rbox_plugin.la ../../src/librmb/librmb.la' +else + # When we are sourced in execute mode, $file and $ECHO are already set. + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + file="$0" + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +$1 +_LTECHO_EOF' +} + ECHO="printf %s\\n" + fi + +# Very basic option parsing. These options are (a) specific to +# the libtool wrapper, (b) are identical between the wrapper +# /script/ and the wrapper /executable/ that is used only on +# windows platforms, and (c) all begin with the string --lt- +# (application programs are unlikely to have options that match +# this pattern). +# +# There are only two supported options: --lt-debug and +# --lt-dump-script. There is, deliberately, no --lt-help. +# +# The first argument to this parsing function should be the +# script's ../../libtool value, followed by no. +lt_option_debug= +func_parse_lt_options () +{ + lt_script_arg0=$0 + shift + for lt_opt + do + case "$lt_opt" in + --lt-debug) lt_option_debug=1 ;; + --lt-dump-script) + lt_dump_D=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%/[^/]*$%%'` + test "X$lt_dump_D" = "X$lt_script_arg0" && lt_dump_D=. + lt_dump_F=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%^.*/%%'` + cat "$lt_dump_D/$lt_dump_F" + exit 0 + ;; + --lt-*) + $ECHO "Unrecognized --lt- option: '$lt_opt'" 1>&2 + exit 1 + ;; + esac + done + + # Print the debug banner immediately: + if test -n "$lt_option_debug"; then + echo "it_test_check_metadata:it_test_check_metadata:$LINENO: libtool wrapper (GNU libtool) 2.4.6 Debian-2.4.6-2" 1>&2 + fi +} + +# Used when --lt-debug. Prints its arguments to stdout +# (redirection is the responsibility of the caller) +func_lt_dump_args () +{ + lt_dump_args_N=1; + for lt_arg + do + $ECHO "it_test_check_metadata:it_test_check_metadata:$LINENO: newargv[$lt_dump_args_N]: $lt_arg" + lt_dump_args_N=`expr $lt_dump_args_N + 1` + done +} + +# Core function for launching the target application +func_exec_program_core () +{ + + if test -n "$lt_option_debug"; then + $ECHO "it_test_check_metadata:it_test_check_metadata:$LINENO: newargv[0]: $progdir/$program" 1>&2 + func_lt_dump_args ${1+"$@"} 1>&2 + fi + exec "$progdir/$program" ${1+"$@"} + + $ECHO "$0: cannot exec $program $*" 1>&2 + exit 1 +} + +# A function to encapsulate launching the target application +# Strips options in the --lt-* namespace from $@ and +# launches target application with the remaining arguments. +func_exec_program () +{ + case " $* " in + *\ --lt-*) + for lt_wr_arg + do + case $lt_wr_arg in + --lt-*) ;; + *) set x "$@" "$lt_wr_arg"; shift;; + esac + shift + done ;; + esac + func_exec_program_core ${1+"$@"} +} + + # Parse options + func_parse_lt_options "$0" ${1+"$@"} + + # Find the directory that this script lives in. + thisdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'` + test "x$thisdir" = "x$file" && thisdir=. + + # Follow symbolic links until we get to the real thisdir. + file=`ls -ld "$file" | /usr/bin/sed -n 's/.*-> //p'` + while test -n "$file"; do + destdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'` + + # If there was a directory component, then change thisdir. + if test "x$destdir" != "x$file"; then + case "$destdir" in + [\\/]* | [A-Za-z]:[\\/]*) thisdir="$destdir" ;; + *) thisdir="$thisdir/$destdir" ;; + esac + fi + + file=`$ECHO "$file" | /usr/bin/sed 's%^.*/%%'` + file=`ls -ld "$thisdir/$file" | /usr/bin/sed -n 's/.*-> //p'` + done + + # Usually 'no', except on cygwin/mingw when embedded into + # the cwrapper. + WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=no + if test "$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR" = "yes"; then + # special case for '.' + if test "$thisdir" = "."; then + thisdir=`pwd` + fi + # remove .libs from thisdir + case "$thisdir" in + *[\\/].libs ) thisdir=`$ECHO "$thisdir" | /usr/bin/sed 's%[\\/][^\\/]*$%%'` ;; + .libs ) thisdir=. ;; + esac + fi + + # Try to get the absolute directory name. + absdir=`cd "$thisdir" && pwd` + test -n "$absdir" && thisdir="$absdir" + + program=lt-'it_test_check_metadata' + progdir="$thisdir/.libs" + + if test ! -f "$progdir/$program" || + { file=`ls -1dt "$progdir/$program" "$progdir/../$program" 2>/dev/null | /usr/bin/sed 1q`; \ + test "X$file" != "X$progdir/$program"; }; then + + file="$$-$program" + + if test ! -d "$progdir"; then + mkdir "$progdir" + else + rm -f "$progdir/$file" + fi + + # relink executable if necessary + if test -n "$relink_command"; then + if relink_command_output=`eval $relink_command 2>&1`; then : + else + $ECHO "$relink_command_output" >&2 + rm -f "$progdir/$file" + exit 1 + fi + fi + + mv -f "$progdir/$file" "$progdir/$program" 2>/dev/null || + { rm -f "$progdir/$program"; + mv -f "$progdir/$file" "$progdir/$program"; } + rm -f "$progdir/$file" + fi + + if test -f "$progdir/$program"; then + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + # Run the actual program with our arguments. + func_exec_program ${1+"$@"} + fi + else + # The program doesn't exist. + $ECHO "$0: error: '$progdir/$program' does not exist" 1>&2 + $ECHO "This script is just a wrapper for $program." 1>&2 + $ECHO "See the libtool documentation for more information." 1>&2 + exit 1 + fi +fi diff --git a/src/tests/it_test_storage_rbox_2 b/src/tests/it_test_storage_rbox_2 new file mode 100755 index 00000000..c3380363 --- /dev/null +++ b/src/tests/it_test_storage_rbox_2 @@ -0,0 +1,228 @@ +#! /bin/sh + +# it_test_storage_rbox_2 - temporary wrapper script for .libs/it_test_storage_rbox_2 +# Generated by libtool (GNU libtool) 2.4.2 +# +# The it_test_storage_rbox_2 program cannot be directly executed until all the libtool +# libraries that it depends on are installed. +# +# This wrapper script should never be moved out of the build directory. +# If it is, it will not operate correctly. + +# Sed substitution that helps us do robust quoting. It backslashifies +# metacharacters that are still active within double-quoted strings. +sed_quote_subst='s/\([`"$\\]\)/\\\1/g' + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac +fi +BIN_SH=xpg4; export BIN_SH # for Tru64 +DUALCASE=1; export DUALCASE # for MKS sh + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +relink_command="(cd /repo/src/tests; { test -z \"\${LIBRARY_PATH+set}\" || unset LIBRARY_PATH || { LIBRARY_PATH=; export LIBRARY_PATH; }; }; { test -z \"\${COMPILER_PATH+set}\" || unset COMPILER_PATH || { COMPILER_PATH=; export COMPILER_PATH; }; }; { test -z \"\${GCC_EXEC_PREFIX+set}\" || unset GCC_EXEC_PREFIX || { GCC_EXEC_PREFIX=; export GCC_EXEC_PREFIX; }; }; { test -z \"\${LD_RUN_PATH+set}\" || unset LD_RUN_PATH || { LD_RUN_PATH=; export LD_RUN_PATH; }; }; { test -z \"\${LD_LIBRARY_PATH+set}\" || unset LD_LIBRARY_PATH || { LD_LIBRARY_PATH=; export LD_LIBRARY_PATH; }; }; PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; export PATH; g++ -std=gnu++11 -pthread -fpermissive -std=c++11 -Wmissing-declarations -g -O0 -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-undef -Wno-redundant-decls -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-error=undef -Wno-error=redundant-decls -fpermissive -std=c++11 -fPIC -DPIC -o \$progdir/\$file storage-rbox/it_test_storage_rbox_2-it_test_storage_rbox.o storage-rbox2/it_test_storage_rbox_2-TestCase.o test-utils/it_test_storage_rbox_2-it_utils.o ../../src/storage-rbox/.libs/libstorage_rbox_plugin.so -L/usr/local/lib/dovecot /repo/src/librmb/.libs/librmb.so /usr/local/lib/dovecot/libdovecot-storage.so /usr/local/lib/dovecot/libdovecot.so -ldl ../../src/librmb/.libs/librmb.so -ljansson ./.libs/libgmockgtest.a -lrados -pthread -Wl,-rpath -Wl,/repo/src/storage-rbox/.libs -Wl,-rpath -Wl,/repo/src/librmb/.libs -Wl,-rpath -Wl,/usr/local/lib/dovecot -Wl,-rpath -Wl,/usr/local/lib/dovecot -Wl,-rpath -Wl,/usr/local/lib)" + +# This environment variable determines our operation mode. +if test "$libtool_install_magic" = "%%%MAGIC variable%%%"; then + # install mode needs the following variables: + generated_by_libtool_version='2.4.2' + notinst_deplibs=' ../../src/storage-rbox/libstorage_rbox_plugin.la /repo/src/librmb/librmb.la ../../src/librmb/librmb.la' +else + # When we are sourced in execute mode, $file and $ECHO are already set. + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + file="$0" + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +$1 +_LTECHO_EOF' +} + ECHO="printf %s\\n" + fi + +# Very basic option parsing. These options are (a) specific to +# the libtool wrapper, (b) are identical between the wrapper +# /script/ and the wrapper /executable/ which is used only on +# windows platforms, and (c) all begin with the string --lt- +# (application programs are unlikely to have options which match +# this pattern). +# +# There are only two supported options: --lt-debug and +# --lt-dump-script. There is, deliberately, no --lt-help. +# +# The first argument to this parsing function should be the +# script's ../../libtool value, followed by no. +lt_option_debug= +func_parse_lt_options () +{ + lt_script_arg0=$0 + shift + for lt_opt + do + case "$lt_opt" in + --lt-debug) lt_option_debug=1 ;; + --lt-dump-script) + lt_dump_D=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%/[^/]*$%%'` + test "X$lt_dump_D" = "X$lt_script_arg0" && lt_dump_D=. + lt_dump_F=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%^.*/%%'` + cat "$lt_dump_D/$lt_dump_F" + exit 0 + ;; + --lt-*) + $ECHO "Unrecognized --lt- option: '$lt_opt'" 1>&2 + exit 1 + ;; + esac + done + + # Print the debug banner immediately: + if test -n "$lt_option_debug"; then + echo "it_test_storage_rbox_2:it_test_storage_rbox_2:${LINENO}: libtool wrapper (GNU libtool) 2.4.2" 1>&2 + fi +} + +# Used when --lt-debug. Prints its arguments to stdout +# (redirection is the responsibility of the caller) +func_lt_dump_args () +{ + lt_dump_args_N=1; + for lt_arg + do + $ECHO "it_test_storage_rbox_2:it_test_storage_rbox_2:${LINENO}: newargv[$lt_dump_args_N]: $lt_arg" + lt_dump_args_N=`expr $lt_dump_args_N + 1` + done +} + +# Core function for launching the target application +func_exec_program_core () +{ + + if test -n "$lt_option_debug"; then + $ECHO "it_test_storage_rbox_2:it_test_storage_rbox_2:${LINENO}: newargv[0]: $progdir/$program" 1>&2 + func_lt_dump_args ${1+"$@"} 1>&2 + fi + exec "$progdir/$program" ${1+"$@"} + + $ECHO "$0: cannot exec $program $*" 1>&2 + exit 1 +} + +# A function to encapsulate launching the target application +# Strips options in the --lt-* namespace from $@ and +# launches target application with the remaining arguments. +func_exec_program () +{ + case " $* " in + *\ --lt-*) + for lt_wr_arg + do + case $lt_wr_arg in + --lt-*) ;; + *) set x "$@" "$lt_wr_arg"; shift;; + esac + shift + done ;; + esac + func_exec_program_core ${1+"$@"} +} + + # Parse options + func_parse_lt_options "$0" ${1+"$@"} + + # Find the directory that this script lives in. + thisdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'` + test "x$thisdir" = "x$file" && thisdir=. + + # Follow symbolic links until we get to the real thisdir. + file=`ls -ld "$file" | /usr/bin/sed -n 's/.*-> //p'` + while test -n "$file"; do + destdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'` + + # If there was a directory component, then change thisdir. + if test "x$destdir" != "x$file"; then + case "$destdir" in + [\\/]* | [A-Za-z]:[\\/]*) thisdir="$destdir" ;; + *) thisdir="$thisdir/$destdir" ;; + esac + fi + + file=`$ECHO "$file" | /usr/bin/sed 's%^.*/%%'` + file=`ls -ld "$thisdir/$file" | /usr/bin/sed -n 's/.*-> //p'` + done + + # Usually 'no', except on cygwin/mingw when embedded into + # the cwrapper. + WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=no + if test "$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR" = "yes"; then + # special case for '.' + if test "$thisdir" = "."; then + thisdir=`pwd` + fi + # remove .libs from thisdir + case "$thisdir" in + *[\\/].libs ) thisdir=`$ECHO "$thisdir" | /usr/bin/sed 's%[\\/][^\\/]*$%%'` ;; + .libs ) thisdir=. ;; + esac + fi + + # Try to get the absolute directory name. + absdir=`cd "$thisdir" && pwd` + test -n "$absdir" && thisdir="$absdir" + + program=lt-'it_test_storage_rbox_2' + progdir="$thisdir/.libs" + + if test ! -f "$progdir/$program" || + { file=`ls -1dt "$progdir/$program" "$progdir/../$program" 2>/dev/null | /usr/bin/sed 1q`; \ + test "X$file" != "X$progdir/$program"; }; then + + file="$$-$program" + + if test ! -d "$progdir"; then + mkdir "$progdir" + else + rm -f "$progdir/$file" + fi + + # relink executable if necessary + if test -n "$relink_command"; then + if relink_command_output=`eval $relink_command 2>&1`; then : + else + printf %s\n "$relink_command_output" >&2 + rm -f "$progdir/$file" + exit 1 + fi + fi + + mv -f "$progdir/$file" "$progdir/$program" 2>/dev/null || + { rm -f "$progdir/$program"; + mv -f "$progdir/$file" "$progdir/$program"; } + rm -f "$progdir/$file" + fi + + if test -f "$progdir/$program"; then + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + # Run the actual program with our arguments. + func_exec_program ${1+"$@"} + fi + else + # The program doesn't exist. + $ECHO "$0: error: \`$progdir/$program' does not exist" 1>&2 + $ECHO "This script is just a wrapper for $program." 1>&2 + $ECHO "See the libtool documentation for more information." 1>&2 + exit 1 + fi +fi diff --git a/src/tests/librmb/it_test_librmb.cpp b/src/tests/librmb/it_test_librmb.cpp index 4322776a..f76bf110 100644 --- a/src/tests/librmb/it_test_librmb.cpp +++ b/src/tests/librmb/it_test_librmb.cpp @@ -20,9 +20,15 @@ #include "../../librmb/rados-metadata-storage-default.h" #include "../../librmb/rados-metadata-storage-ima.h" #include "../../librmb/rados-dovecot-ceph-cfg-impl.h" -#include "../../librmb/rados-util.h" -#include "../../librmb/tools/rmb/rmb-commands.h" -#include "../../librmb/rados-save-log.h" +#include "../../librmb/rados-util-impl.h" +#include "../../librmb/tools/rmb/rmb-commands-impl.h" +#include "../../librmb/rados-mail-impl.h" +#include "../../storage-interface/rados-mail.h" +#include "../../storage-engine/storage-backend-factory.h" +#include "../../librmb/rados-save-log-impl.h" +#include "../../storage-interface/rados-save-log.h" +#include "../../librmb/rados-metadata-impl.h" +#include "../../storage-interface/rados-dovecot-ceph-cfg.h" using ::testing::AtLeast; using ::testing::Return; @@ -32,80 +38,75 @@ using ::testing::Return; * */ TEST(librmb, split_write_operation) { - uint64_t max_size = 3; - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - std::cout << "jsjjsjssjs" << std::endl; - size_t buffer_length = obj.get_mail_buffer()->length(); - std::cout << "lenght" << std::endl; - + librados::bufferlist bl; + std::string test_buffer_lenght="it is a test for test buffer end"; + bl.append(test_buffer_lenght.c_str()); + std::cout<<"it is a test for test buffer end"<append(splitable_buffer); + int buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + std::cout <<"buffer_length::"<< buffer_length << std::endl; obj.set_mail_size(buffer_length); + std::cout <<"obj.get_mail_size()::"<< obj.get_mail_size() << std::endl; + std::cout <<"storage.get_max_object_size()::"<< storage.get_max_object_size() << std::endl; + std::cout <<"get_max_write_size_bytes()::"<< storage.get_max_write_size_bytes() << std::endl; obj.set_oid("test_oid"); librados::IoCtx io_ctx; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); - librmb::RadosClusterImpl cluster; - librmb::RadosStorageImpl storage(&cluster); - std::string pool_name("test"); std::string ns("t"); std::cout << "open" << std::endl; int open_connection = storage.open_connection(pool_name); - std::cout << "pok" << std::endl; storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - std::cout << "accccc" << std::endl; - - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - std::cout << "lsdlslslss" << std::endl; + bool ret_storage=storage.save_mail(&obj); // stat the object uint64_t size; time_t save_date; + std::cout<<"stat_mail is invoked"<append("HALLO_WELT_"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - uint64_t max_size = buffer_length; - - obj.set_oid("test_oid"); - librados::IoCtx io_ctx; - - librados::ObjectWriteOperation op; //= new librados::ObjectWriteOperation(); + librmb::RadosMailImpl obj; librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); - + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("HALLO_WELT_"); + size_t buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + obj.set_oid("test_oid"); std::string pool_name("test"); std::string ns("t"); - int open_connection = storage.open_connection(pool_name); storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); + bool ret_storage = storage.save_mail(&obj); // stat the object uint64_t size; @@ -119,10 +120,10 @@ TEST(librmb1, split_write_operation_1) { cluster.deinit(); EXPECT_EQ(buffer_length, size); - EXPECT_EQ(0, ret_storage); + EXPECT_EQ(true, ret_storage); EXPECT_EQ(0, ret_stat); EXPECT_EQ(0, ret_remove); - EXPECT_EQ(0, (int)obj.get_num_active_op()); + delete obj.get_mail_buffer(); } /** * Test Rados Metadata type conversion @@ -130,199 +131,191 @@ TEST(librmb1, split_write_operation_1) { */ TEST(librmb1, convert_types) { std::string value = "4441c5339f4c9d59523000009c60b9f7"; - librmb::RadosMetadata attr(librmb::RBOX_METADATA_GUID, value); + librmb::RadosMetadataImpl attr(storage_interface::RBOX_METADATA_GUID, value); EXPECT_EQ(attr.key, "G"); - EXPECT_STREQ(attr.bl.c_str(), "4441c5339f4c9d59523000009c60b9f7"); + EXPECT_STREQ((*attr.bl).c_str(), "4441c5339f4c9d59523000009c60b9f7"); time_t t = 1503488583; attr.key = ""; - attr.bl.clear(); - librmb::RadosMetadata attr2(librmb::RBOX_METADATA_RECEIVED_TIME, t); + (*attr.bl).clear(); + librmb::RadosMetadataImpl attr2(storage_interface::RBOX_METADATA_RECEIVED_TIME, t); EXPECT_EQ(attr2.key, "R"); - EXPECT_STREQ(attr2.bl.c_str(), "1503488583"); + EXPECT_STREQ((*attr2.bl).c_str(), "1503488583"); - time_t recv_date; - attr2.convert(attr2.bl.c_str(), &recv_date); - EXPECT_EQ(t, recv_date); + // time_t recv_date; + // attr2.convert(attr2.key, recv_date); + // EXPECT_EQ(t, recv_date); size_t st = 100; - librmb::RadosMetadata attr4(librmb::RBOX_METADATA_VIRTUAL_SIZE, st); + librmb::RadosMetadataImpl attr4(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, st); EXPECT_EQ(attr4.key, "V"); - EXPECT_STREQ(attr4.bl.c_str(), "100"); + EXPECT_STREQ((*attr4.bl).c_str(), "100"); attr4.key = ""; - attr4.bl.clear(); + (*attr4.bl).clear(); } /** * Test Storage read_mail * */ TEST(librmb1, read_mail) { - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - uint64_t max_size = buffer_length; - - obj.set_oid("test_oid"); - librados::IoCtx io_ctx; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); + librmb::RadosMailImpl obj; librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); - + obj.set_mail_buffer(storage.alloc_mail_buffer()); + const char *message = + "From: user@domain.org\n" + "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" + "Mime-Version: 1.0\n" + "Content-Type: text/plain; charset=us-ascii\n" + "\n" + "body\n"; + std::stringstream test_buff; + test_buff<append(message,test_buff.str().length()); + int buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + int max_size = buffer_length; + obj.set_oid("test_oid"); std::string pool_name("test"); std::string ns("t"); - int open_connection = storage.open_connection(pool_name); storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - - // wait for op to finish. - storage.wait_for_write_operations_complete(obj.get_completion(), obj.get_write_operation()); + bool ret_storage = storage.save_mail(&obj); // stat the object uint64_t size; time_t save_date; - int ret_stat = storage.stat_mail(*obj.get_oid(), &size, &save_date); - - librados::bufferlist bl; - int copy_mail_ret = storage.read_mail(*obj.get_oid(), &bl); - char *buff = new char[copy_mail_ret + 1]; - memset(buff, 1, size + 1); - memcpy(buff, bl.to_str().c_str(), copy_mail_ret + 1); - EXPECT_EQ(buff[copy_mail_ret], '\0'); + int ret_stat = storage.stat_mail(*obj.get_oid(), &size, &save_date); + int copy_mail_ret = storage.read_mail(*obj.get_oid(),&obj,0); // remove it int ret_remove = storage.delete_mail(*obj.get_oid()); // tear down cluster.deinit(); - EXPECT_EQ(ret_storage, 0); + EXPECT_EQ(ret_storage, true); EXPECT_EQ(ret_stat, 0); EXPECT_EQ(ret_remove, 0); - EXPECT_EQ(copy_mail_ret, 14); - EXPECT_EQ(buff[0], 'a'); - EXPECT_EQ(buff[1], 'b'); - EXPECT_EQ(buff[2], 'c'); - EXPECT_EQ(buff[3], 'd'); - - delete[] buff; + EXPECT_EQ(copy_mail_ret, 0); + delete obj.get_mail_buffer(); } /** * Test Load Metadata * */ TEST(librmb, load_metadata) { - uint64_t max_size = 3; - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); + librmb::RadosMailImpl obj; + librmb::RadosClusterImpl *cluster= new librmb::RadosClusterImpl(); + int ret_cluster= cluster->init(); + if(ret_cluster >= 0){ + std::cout<<"cluster is working"; + } + librmb::RadosStorageImpl storage(cluster); + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("abcdefghijklmn"); + size_t buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); obj.set_oid("test_oid"); - librados::IoCtx io_ctx; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); - librmb::RadosClusterImpl cluster; - librmb::RadosStorageImpl storage(&cluster); std::string pool_name("test"); std::string ns("t"); int open_connection = storage.open_connection(pool_name); storage.set_namespace(ns); EXPECT_EQ(0, open_connection); + bool ret_storage = storage.save_mail(&obj); + EXPECT_EQ(true, ret_storage); + + // librmb::RadosMetadataStorageDefault ms(storage.get_io_ctx_wrapper()); + storage_interface::RadosStorageMetadataModule *ms = new librmb::RadosMetadataStorageDefault(storage.get_io_ctx_wrapper()); + std::string key="key_1"; + std::string val="val_1"; + storage_interface::RadosMetadata *rados_metadata_1=new librmb::RadosMetadataImpl(key,val); + std::list metadata_list; + metadata_list.push_back(rados_metadata_1); + + key="key_2"; + val="val_2"; + storage_interface::RadosMetadata *rados_metadata_2=new librmb::RadosMetadataImpl(key,val); + metadata_list.push_back(rados_metadata_2); + ms->update_metadata(*obj.get_oid(),metadata_list); - librmb::RadosMetadataStorageDefault ms(&storage.get_io_ctx()); - - ceph::bufferlist bl; - bl.append("xyz\0"); - op.setxattr("A", bl); - op.setxattr("B", bl); - - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - - ms.load_metadata(&obj); + std::cout << " load with null" << std::endl; + int i = ms->load_metadata(nullptr); + EXPECT_EQ(-1, i); + i = ms->load_metadata(&obj); + EXPECT_EQ(0, i); std::cout << "load metadata ok" << std::endl; + // stat the object uint64_t size; time_t save_date; int ret_stat = storage.stat_mail(*obj.get_oid(), &size, &save_date); EXPECT_EQ(buffer_length, size); - EXPECT_EQ(0, ret_storage); + EXPECT_EQ(0, ret_stat); - EXPECT_EQ(0, (int)obj.get_num_active_op()); - EXPECT_EQ(2, (int)obj.get_metadata()->size()); - std::cout << " load with null" << std::endl; - int i = ms.load_metadata(nullptr); - EXPECT_EQ(-1, i); - // obj->get_metadata()->size == 2 - i = ms.load_metadata(&obj); - EXPECT_EQ(0, i); + EXPECT_EQ(2, obj.get_metadata()->size()); // remove it int ret_remove = storage.delete_mail(*obj.get_oid()); EXPECT_EQ(0, ret_remove); // tear down - cluster.deinit(); + cluster->deinit(); + delete cluster; + ceph::bufferlist *obj_buff= obj.get_mail_buffer(); + delete obj_buff; + delete rados_metadata_1; + delete rados_metadata_2; + delete ms; } /** * rados object version behavior * */ TEST(librmb, AttributeVersions) { - uint64_t max_size = 3; - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - - obj.set_oid("test_oid2"); - librados::IoCtx io_ctx; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); + librmb::RadosMailImpl obj; librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); - + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("abcdefghijklmn"); + size_t buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + obj.set_oid("test_oid2"); std::string pool_name("test"); std::string ns("t"); int open_connection = storage.open_connection(pool_name); storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - - librmb::RadosMetadataStorageDefault ms(&storage.get_io_ctx()); - - ceph::bufferlist bl; - bl.append("xyz\0"); - op.setxattr("A", bl); - - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - EXPECT_EQ(ret_storage, 0); + bool ret_storage = storage.save_mail(&obj); + EXPECT_EQ(ret_storage, true); + + std::list metadata_list; + std::string key="key_1"; + std::string val="val_1"; + librmb::RadosMetadataImpl *rados_metadata_1=new librmb::RadosMetadataImpl(key,val); + metadata_list.push_back(rados_metadata_1); + key="key_2"; + val="val_2"; + librmb::RadosMetadataImpl *rados_metadata_2=new librmb::RadosMetadataImpl(key,val); + metadata_list.push_back(rados_metadata_2); + uint64_t version = storage.get_io_ctx().get_last_version(); + librmb::RadosMetadataStorageDefault ms(storage.get_io_ctx_wrapper()); + ms.update_metadata(*obj.get_oid(),metadata_list); + ms.load_metadata(&obj); + EXPECT_EQ(obj.get_metadata()->size(),2); // stat the object uint64_t size; time_t save_date; int ret_stat = storage.stat_mail(*obj.get_oid(), &size, &save_date); EXPECT_EQ(ret_stat, 0); - uint64_t version = storage.get_io_ctx().get_last_version(); - - // update metadata - librmb::RadosMetadata metadata(librmb::RBOX_METADATA_OLDV1_KEYWORDS, "abc"); - ms.set_metadata(&obj, metadata); - uint64_t version_after_xattr_update = storage.get_io_ctx().get_last_version(); EXPECT_NE(version, version_after_xattr_update); @@ -330,7 +323,7 @@ TEST(librmb, AttributeVersions) { librados::bufferlist omap_bl; omap_bl.append("xxx"); map.insert(std::pair(*obj.get_oid(), omap_bl)); - storage.get_io_ctx().omap_set(*obj.get_oid(), map); + storage.get_io_ctx_wrapper()->omap_set(*obj.get_oid(), map); uint64_t version_after_omap_set = storage.get_io_ctx().get_last_version(); EXPECT_NE(version_after_xattr_update, version_after_omap_set); @@ -338,16 +331,16 @@ TEST(librmb, AttributeVersions) { storage.delete_mail(*obj.get_oid()); // tear down cluster.deinit(); + delete obj.get_mail_buffer(); + delete rados_metadata_1; + delete rados_metadata_2; } // standard call order for metadata updates // 1. save_metadata // 2. set_metadata (update uid) TEST(librmb, json_ima) { - librados::IoCtx io_ctx; - uint64_t max_size = 3; - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -357,67 +350,52 @@ TEST(librmb, json_ima) { int open_connection = storage.open_connection(pool_name); storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - - librmb::RadosDovecotCephCfgImpl cfg(&storage.get_io_ctx()); - - // cfg.update_updatable_attributes(""); - librmb::RadosMetadataStorageIma ms(&storage.get_io_ctx(), &cfg); - - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - obj.set_oid("test_ima"); + + storage_interface::RadosMail *obj=new librmb::RadosMailImpl(); + obj->set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj->get_mail_buffer())->append("abcdefghijklmn"); + size_t buffer_length =((librados::bufferlist*)obj->get_mail_buffer())->length(); + obj->set_mail_size(buffer_length); + obj->set_oid("test_ima"); + + bool ret_storage = storage.save_mail(obj); + EXPECT_EQ(ret_storage, true); + + storage_interface::RadosDovecotCephCfg *cfg = new librmb::RadosDovecotCephCfgImpl(storage.get_io_ctx_wrapper()); + librmb::RadosMetadataStorageIma ms(storage.get_io_ctx_wrapper(),cfg); unsigned int flags = 0x18; long recv_time = 12345677; // all attributes are not updateable. - librmb::RadosMetadata attr(librmb::RBOX_METADATA_GUID, "guid"); - librmb::RadosMetadata attr2(librmb::RBOX_METADATA_OLDV1_FLAGS, flags); - librmb::RadosMetadata attr3(librmb::RBOX_METADATA_RECEIVED_TIME, recv_time); - librmb::RadosMetadata attr4(librmb::RBOX_METADATA_VERSION, "0.1"); - - obj.add_metadata(attr); - obj.add_metadata(attr2); - obj.add_metadata(attr3); - obj.add_metadata(attr4); + storage_interface::RadosMetadata *attr = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_GUID, "guid"); + storage_interface::RadosMetadata *attr2 = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_OLDV1_FLAGS, flags); + storage_interface::RadosMetadata *attr3 = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_RECEIVED_TIME, recv_time); + storage_interface::RadosMetadata *attr4 = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_VERSION, "0.1"); - ms.save_metadata(&op, &obj); - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - EXPECT_EQ(ret_storage, 0); - // wait for op to finish. - storage.wait_for_write_operations_complete(obj.get_completion(), obj.get_write_operation()); + obj->add_metadata(attr); + obj->add_metadata(attr2); + obj->add_metadata(attr3); + obj->add_metadata(attr4); + ms.set_metadata(obj); // check std::map attr_list; - storage.get_io_ctx().getxattrs(*obj.get_oid(), attr_list); - EXPECT_EQ(1, attr_list.size()); - - unsigned int uid = 10; - librmb::RadosMetadata attr_uid(librmb::RBOX_METADATA_MAIL_UID, uid); - - ms.set_metadata(&obj, attr_uid); - - // check again - attr_list.clear(); - storage.get_io_ctx().getxattrs(*obj.get_oid(), attr_list); + storage.get_io_ctx().getxattrs(*obj->get_oid(), attr_list); EXPECT_EQ(1, attr_list.size()); - storage.delete_mail(&obj); + storage.delete_mail(*obj->get_oid()); // tear down cluster.deinit(); + delete obj->get_mail_buffer(); + delete obj; + delete cfg; } // standard call order for metadata updates // 0. pre-condition: setting flags as updateable // 1. save_metadata // 2. set_metadata (update uid) TEST(librmb, json_ima_2) { - librados::IoCtx io_ctx; uint64_t max_size = 3; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -428,57 +406,60 @@ TEST(librmb, json_ima_2) { storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - librmb::RadosDovecotCephCfgImpl cfg(&storage.get_io_ctx()); - - cfg.set_update_attributes("true"); - cfg.update_updatable_attributes("F"); - librmb::RadosMetadataStorageIma ms(&storage.get_io_ctx(), &cfg); - - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - obj.set_oid("test_ima"); + storage_interface::RadosMail *obj = new librmb::RadosMailImpl(); + obj->set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj->get_mail_buffer())->append("abcdefghijklmn"); + size_t buffer_length = ((librados::bufferlist*)obj->get_mail_buffer())->length(); + obj->set_mail_size(buffer_length); + obj->set_oid("test_ima"); + bool ret_storage = storage.save_mail(obj); + EXPECT_EQ(ret_storage, true); + + storage_interface::RadosDovecotCephCfg *cfg=new librmb::RadosDovecotCephCfgImpl (storage.get_io_ctx_wrapper()); + cfg->set_update_attributes("true"); + cfg->update_updatable_attributes("F"); + librmb::RadosMetadataStorageIma ms(storage.get_io_ctx_wrapper(), cfg); unsigned int flags = 0x18; long recv_time = 12345677; // all attributes are not updateable. - librmb::RadosMetadata attr(librmb::RBOX_METADATA_GUID, "guid"); - librmb::RadosMetadata attr2(librmb::RBOX_METADATA_OLDV1_FLAGS, flags); - librmb::RadosMetadata attr3(librmb::RBOX_METADATA_RECEIVED_TIME, recv_time); - librmb::RadosMetadata attr4(librmb::RBOX_METADATA_VERSION, "0.1"); - - obj.add_metadata(attr); - obj.add_metadata(attr2); - obj.add_metadata(attr3); - obj.add_metadata(attr4); - - ms.save_metadata(&op, &obj); - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - EXPECT_EQ(ret_storage, 0); - - // wait for op to finish. - storage.wait_for_write_operations_complete(obj.get_completion(), obj.get_write_operation()); - + storage_interface::RadosMetadata *attr = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_GUID, "guid"); + storage_interface::RadosMetadata *attr2 = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_OLDV1_FLAGS, flags); + storage_interface::RadosMetadata *attr3 = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_RECEIVED_TIME, recv_time); + storage_interface::RadosMetadata *attr4 = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_VERSION, "0.1"); + + obj->add_metadata(attr); + obj->add_metadata(attr2); + obj->add_metadata(attr3); + obj->add_metadata(attr4); + ms.set_metadata(obj); + // check there should be ima and F (Flags) std::map attr_list; - storage.get_io_ctx().getxattrs(*obj.get_oid(), attr_list); + storage.get_io_ctx().getxattrs(*obj->get_oid(), attr_list); EXPECT_EQ(2, attr_list.size()); unsigned int uid = 10; - librmb::RadosMetadata attr_uid(librmb::RBOX_METADATA_MAIL_UID, uid); + storage_interface::RadosMetadata *attr_uid = + new librmb::RadosMetadataImpl (storage_interface::RBOX_METADATA_MAIL_UID, uid); - ms.set_metadata(&obj, attr_uid); + obj->add_metadata(attr_uid); + ms.set_metadata(obj); // check again attr_list.clear(); - storage.get_io_ctx().getxattrs(*obj.get_oid(), attr_list); + storage.get_io_ctx().getxattrs(*obj->get_oid(), attr_list); EXPECT_EQ(2, attr_list.size()); - storage.delete_mail(&obj); + storage.delete_mail(*obj->get_oid()); // tear down cluster.deinit(); + delete obj->get_mail_buffer(); + delete cfg; + delete obj; } // standard call order for metadata updates @@ -486,10 +467,6 @@ TEST(librmb, json_ima_2) { // 1. save_metadata with keywords // 2. set_metadata (update uid) TEST(librmb, json_ima_3) { - librados::IoCtx io_ctx; - uint64_t max_size = 3; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -500,79 +477,79 @@ TEST(librmb, json_ima_3) { storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - librmb::RadosDovecotCephCfgImpl cfg(&storage.get_io_ctx()); - cfg.set_update_attributes("true"); - cfg.update_updatable_attributes("FK"); - librmb::RadosMetadataStorageIma ms(&storage.get_io_ctx(), &cfg); + storage_interface::RadosMail *obj= new librmb::RadosMailImpl(); + obj->set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj->get_mail_buffer())->append("abcdefghijklmn"); + size_t buffer_length = ((librados::bufferlist*)obj->get_mail_buffer())->length(); + obj->set_mail_size(buffer_length); + obj->set_oid("test_ima"); + bool ret_storage = storage.save_mail(obj); + EXPECT_EQ(ret_storage, true); - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - obj.set_oid("test_ima"); unsigned int flags = 0x18; long recv_time = 12345677; // all attributes are not updateable. - librmb::RadosMetadata attr(librmb::RBOX_METADATA_GUID, "guid"); - librmb::RadosMetadata attr2(librmb::RBOX_METADATA_OLDV1_FLAGS, flags); - librmb::RadosMetadata attr3(librmb::RBOX_METADATA_RECEIVED_TIME, recv_time); - librmb::RadosMetadata attr4(librmb::RBOX_METADATA_VERSION, "0.1"); - - obj.add_metadata(attr); - obj.add_metadata(attr2); - obj.add_metadata(attr3); - obj.add_metadata(attr4); - + storage_interface::RadosMetadata *attr = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_GUID, "guid"); + storage_interface::RadosMetadata *attr2 = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_OLDV1_FLAGS, flags); + storage_interface::RadosMetadata *attr3 = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_RECEIVED_TIME, recv_time); + storage_interface::RadosMetadata *attr4 = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_VERSION, "0.1"); + + obj->add_metadata(attr); + obj->add_metadata(attr2); + obj->add_metadata(attr3); + obj->add_metadata(attr4); + + storage_interface::RadosDovecotCephCfg *cfg=new librmb::RadosDovecotCephCfgImpl (storage.get_io_ctx_wrapper()); + cfg->set_update_attributes("true"); + cfg->update_updatable_attributes("FK"); + librmb::RadosMetadataStorageIma ms(storage.get_io_ctx_wrapper(), cfg); + ms.set_metadata(obj); + std::map attr_list; + storage.get_io_ctx().getxattrs(*obj->get_oid(), attr_list); + EXPECT_EQ(2, attr_list.size()); + for (int i = 0; i < 10; i++) { std::string keyword = std::to_string(i); std::string ext_key = "k_" + keyword; - librmb::RadosMetadata ext_metadata(ext_key, keyword); - obj.add_extended_metadata(ext_metadata); + storage_interface::RadosMetadata *ext_metadata = + new librmb::RadosMetadataImpl(ext_key, keyword); + obj->add_extended_metadata(ext_metadata); } - ms.save_metadata(&op, &obj); - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - EXPECT_EQ(ret_storage, 0); - - // wait for op to finish. - storage.wait_for_write_operations_complete(obj.get_completion(), obj.get_write_operation()); - - // check there should be ima and F (Flags) - std::map attr_list; - storage.get_io_ctx().getxattrs(*obj.get_oid(), attr_list); - EXPECT_EQ(2, attr_list.size()); - unsigned int uid = 10; - librmb::RadosMetadata attr_uid(librmb::RBOX_METADATA_MAIL_UID, uid); - - ms.set_metadata(&obj, attr_uid); + storage_interface::RadosMetadata *attr_uid = + new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_MAIL_UID, uid); + obj->add_metadata(attr_uid); + ms.set_metadata(obj); // check again attr_list.clear(); - storage.get_io_ctx().getxattrs(*obj.get_oid(), attr_list); + storage.get_io_ctx().getxattrs(*obj->get_oid(), attr_list); EXPECT_EQ(2, attr_list.size()); - obj.get_metadata()->clear(); - obj.get_extended_metadata()->clear(); + obj->get_metadata()->clear(); + obj->get_extended_metadata()->clear(); std::cout << "loading metatadata" << std::endl; - ms.load_metadata(&obj); + ms.load_metadata(obj); - EXPECT_EQ(10, obj.get_extended_metadata()->size()); + EXPECT_EQ(10, obj->get_extended_metadata()->size()); - storage.delete_mail(&obj); + storage.delete_mail(*obj->get_oid()); // tear down cluster.deinit(); + delete obj->get_mail_buffer(); + delete cfg; + delete obj; } /** * Load metadata with default metadata reader */ TEST(librmb, test_default_metadata_load_attributes) { - librados::IoCtx io_ctx; uint64_t max_size = 3; - - librados::ObjectWriteOperation op; // = new librados::ObjectWriteOperation(); librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -583,54 +560,55 @@ TEST(librmb, test_default_metadata_load_attributes) { storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - librmb::RadosDovecotCephCfgImpl cfg(&storage.get_io_ctx()); - cfg.set_update_attributes("true"); - cfg.update_updatable_attributes("FK"); - librmb::RadosMetadataStorageDefault ms(&storage.get_io_ctx()); + storage_interface::RadosMail *obj=new librmb::RadosMailImpl(); + obj->set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj->get_mail_buffer())->append("abcdefghijklmn"); + size_t buffer_length = ((librados::bufferlist*)obj->get_mail_buffer())->length(); + obj->set_mail_size(buffer_length); + obj->set_oid("test_ima"); + + bool ret_storage = storage.save_mail(obj); + EXPECT_EQ(ret_storage, true); - librmb::RadosMail obj; - librados::bufferlist buffer; - obj.set_mail_buffer(&buffer); - obj.get_mail_buffer()->append("abcdefghijklmn"); - size_t buffer_length = obj.get_mail_buffer()->length(); - obj.set_mail_size(buffer_length); - obj.set_oid("test_ima"); unsigned int flags = 0x18; long recv_time = 12345677; // all attributes are not updateable. - librmb::RadosMetadata attr(librmb::RBOX_METADATA_GUID, "guid"); - librmb::RadosMetadata attr2(librmb::RBOX_METADATA_OLDV1_FLAGS, flags); - librmb::RadosMetadata attr3(librmb::RBOX_METADATA_RECEIVED_TIME, recv_time); - librmb::RadosMetadata attr4(librmb::RBOX_METADATA_VERSION, "0.1"); + storage_interface::RadosMetadata *attr = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_GUID, "guid"); + storage_interface::RadosMetadata *attr2 = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_OLDV1_FLAGS, flags); + storage_interface::RadosMetadata *attr3 = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_RECEIVED_TIME, recv_time); + storage_interface::RadosMetadata *attr4 = new librmb::RadosMetadataImpl(storage_interface::RBOX_METADATA_VERSION, "0.1"); - obj.add_metadata(attr); - obj.add_metadata(attr2); - obj.add_metadata(attr3); - obj.add_metadata(attr4); + obj->add_metadata(attr); + obj->add_metadata(attr2); + obj->add_metadata(attr3); + obj->add_metadata(attr4); for (int i = 0; i < 10; i++) { std::string keyword = std::to_string(i); std::string ext_key = "k_" + keyword; - librmb::RadosMetadata ext_metadata(ext_key, keyword); - obj.add_extended_metadata(ext_metadata); + storage_interface::RadosMetadata *ext_metadata = new librmb::RadosMetadataImpl(ext_key, keyword); + obj->add_extended_metadata(ext_metadata); } + + storage_interface::RadosDovecotCephCfg *cfg=new librmb::RadosDovecotCephCfgImpl (storage.get_io_ctx_wrapper()); + cfg->set_update_attributes("true"); + cfg->update_updatable_attributes("FK"); + librmb::RadosMetadataStorageDefault ms(storage.get_io_ctx_wrapper()); - ms.save_metadata(&op, &obj); - int ret_storage = storage.split_buffer_and_exec_op(&obj, &op, max_size); - EXPECT_EQ(ret_storage, 0); - - // wait for op to finish. - storage.wait_for_write_operations_complete(obj.get_completion(), obj.get_write_operation()); + ms.set_metadata(obj); - librmb::RadosMail obj2; + librmb::RadosMailImpl obj2; obj2.set_oid("test_ima"); int a = ms.load_metadata(&obj2); EXPECT_EQ(true, a >= 0); - storage.delete_mail(&obj); + storage.delete_mail(*obj->get_oid()); // tear down cluster.deinit(); + delete obj->get_mail_buffer(); + delete cfg; + delete obj; } /** * Test LoadMetadata default reader @@ -648,12 +626,12 @@ TEST(librmb, test_default_metadata_load_attributes_obj_no_longer_exist) { storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - librmb::RadosDovecotCephCfgImpl cfg(&storage.get_io_ctx()); - cfg.set_update_attributes("true"); - cfg.update_updatable_attributes("FK"); - librmb::RadosMetadataStorageDefault ms(&storage.get_io_ctx()); + storage_interface::RadosDovecotCephCfg *cfg=new librmb::RadosDovecotCephCfgImpl (storage.get_io_ctx_wrapper()); + cfg->set_update_attributes("true"); + cfg->update_updatable_attributes("FK"); + librmb::RadosMetadataStorageDefault ms(storage.get_io_ctx_wrapper()); - librmb::RadosMail obj2; + librmb::RadosMailImpl obj2; obj2.set_oid("test_ima1"); int a = ms.load_metadata(&obj2); @@ -661,6 +639,7 @@ TEST(librmb, test_default_metadata_load_attributes_obj_no_longer_exist) { // tear down cluster.deinit(); + delete cfg; } /** * Test Metadata reader with ima reader @@ -678,12 +657,12 @@ TEST(librmb, test_default_metadata_load_attributes_obj_no_longer_exist_ima) { storage.set_namespace(ns); EXPECT_EQ(0, open_connection); - librmb::RadosDovecotCephCfgImpl cfg(&storage.get_io_ctx()); - cfg.set_update_attributes("true"); - cfg.update_updatable_attributes("FK"); - librmb::RadosMetadataStorageIma ms(&storage.get_io_ctx(), &cfg); + storage_interface::RadosDovecotCephCfg *cfg=new librmb::RadosDovecotCephCfgImpl (storage.get_io_ctx_wrapper()); + cfg->set_update_attributes("true"); + cfg->update_updatable_attributes("FK"); + librmb::RadosMetadataStorageIma ms(storage.get_io_ctx_wrapper(), cfg); - librmb::RadosMail obj2; + librmb::RadosMailImpl obj2; obj2.set_oid("test_ima1"); int a = ms.load_metadata(&obj2); @@ -691,13 +670,12 @@ TEST(librmb, test_default_metadata_load_attributes_obj_no_longer_exist_ima) { // tear down cluster.deinit(); + delete cfg; } /** * Test osd increment */ TEST(librmb, increment_add_to_non_existing_key) { - librados::IoCtx io_ctx; - librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -709,31 +687,37 @@ TEST(librmb, increment_add_to_non_existing_key) { EXPECT_EQ(0, open_connection); std::string key = "my-key"; + librmb::RadosMailImpl obj; + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length =((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj.set_oid("test_oid"); - librmb::RadosMail obj2; - obj2.set_oid("myobject"); - - ceph::bufferlist mail_buf; - storage.save_mail(*obj2.get_oid(), mail_buf); + EXPECT_EQ(true,storage.save_mail(&obj)); long val = 10; // value to add - int ret = librmb::RadosUtils::osd_add(&storage.get_io_ctx(), *obj2.get_oid(), key, val); + librmb::RadosUtilsImpl rados_utils; + int ret = rados_utils.osd_add(storage.get_io_ctx_wrapper(), *obj.get_oid(), key, val); ASSERT_EQ(0, ret); // get the value! std::set keys; std::map omap; keys.insert(key); - ASSERT_EQ(0, storage.get_io_ctx().omap_get_vals_by_keys(*obj2.get_oid(), keys, &omap)); + ASSERT_EQ(0, storage.get_io_ctx().omap_get_vals_by_keys(*obj.get_oid(), keys, &omap)); std::map::iterator it = omap.find(key); ASSERT_NE(omap.end(), it); ceph::bufferlist bl = (*it).second; EXPECT_EQ(bl.to_str(), "10"); - storage.delete_mail(&obj2); + storage.delete_mail(*obj.get_oid()); // tear down cluster.deinit(); + delete obj.get_mail_buffer(); } /** * Test osd increment @@ -753,13 +737,14 @@ TEST(librmb, increment_add_to_non_existing_object) { std::string key = "my-key"; - librmb::RadosMail obj2; + librmb::RadosMailImpl obj2; obj2.set_oid("myobject"); long val = 10; // value to add - int ret = librmb::RadosUtils::osd_add(&storage.get_io_ctx(), *obj2.get_oid(), key, val); + librmb::RadosUtilsImpl rados_utils; + int ret = rados_utils.osd_add(storage.get_io_ctx_wrapper(), *obj2.get_oid(), key, val); ASSERT_EQ(0, ret); - ret = librmb::RadosUtils::osd_add(&storage.get_io_ctx(), *obj2.get_oid(), key, val); + ret = rados_utils.osd_add(storage.get_io_ctx_wrapper(), *obj2.get_oid(), key, val); ASSERT_EQ(0, ret); // get the value! std::set keys; @@ -774,7 +759,7 @@ TEST(librmb, increment_add_to_non_existing_object) { ceph::bufferlist bl = (*it).second; EXPECT_EQ(bl.to_str(), "20"); - storage.delete_mail(&obj2); + storage.delete_mail(*obj2.get_oid()); // tear down cluster.deinit(); } @@ -782,8 +767,6 @@ TEST(librmb, increment_add_to_non_existing_object) { * Test osd increment */ TEST(librmb, increment_add_to_existing_key) { - librados::IoCtx io_ctx; - librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -795,16 +778,20 @@ TEST(librmb, increment_add_to_existing_key) { EXPECT_EQ(0, open_connection); std::string key = "my-key"; - - librmb::RadosMail obj2; - obj2.set_oid("myobject"); - - ceph::bufferlist mail_buf; - storage.save_mail(*obj2.get_oid(), mail_buf); + librmb::RadosMailImpl obj2; + obj2.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj2.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length = ((librados::bufferlist*)obj2.get_mail_buffer())->length(); + obj2.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj2.set_oid("test_oid"); + EXPECT_EQ(true,storage.save_mail(&obj2)); long val = 10; // value to add - int ret = librmb::RadosUtils::osd_add(&storage.get_io_ctx(), *obj2.get_oid(), key, val); + librmb::RadosUtilsImpl rados_utils; + int ret = rados_utils.osd_add(storage.get_io_ctx_wrapper(), *obj2.get_oid(), key, val); ASSERT_EQ(0, ret); - ret = librmb::RadosUtils::osd_add(&storage.get_io_ctx(), *obj2.get_oid(), key, val); + ret = rados_utils.osd_add(storage.get_io_ctx_wrapper(), *obj2.get_oid(), key, val); ASSERT_EQ(0, ret); // get the value! std::set keys; @@ -819,17 +806,16 @@ TEST(librmb, increment_add_to_existing_key) { ceph::bufferlist bl = (*it).second; EXPECT_EQ(bl.to_str(), "20"); - storage.delete_mail(&obj2); + storage.delete_mail(*obj2.get_oid()); // tear down cluster.deinit(); + delete obj2.get_mail_buffer(); } /** * Test osd decrement */ TEST(librmb, increment_sub_from_existing_key) { - librados::IoCtx io_ctx; - librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -841,18 +827,22 @@ TEST(librmb, increment_sub_from_existing_key) { EXPECT_EQ(0, open_connection); std::string key = "my-key"; - - librmb::RadosMail obj2; - obj2.set_oid("myobject"); - - ceph::bufferlist mail_buf; - storage.save_mail(*obj2.get_oid(), mail_buf); + librmb::RadosMailImpl obj2; + obj2.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj2.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length = ((librados::bufferlist*)obj2.get_mail_buffer())->length(); + obj2.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj2.set_oid("test_oid"); + EXPECT_EQ(true,storage.save_mail(&obj2)); long val = 10; // value to add - int ret = librmb::RadosUtils::osd_add(&storage.get_io_ctx(), *obj2.get_oid(), key, val); + librmb::RadosUtilsImpl rados_utils; + int ret = rados_utils.osd_add(storage.get_io_ctx_wrapper(), *obj2.get_oid(), key, val); ASSERT_EQ(0, ret); long sub_val = 5; // value to add - ret = librmb::RadosUtils::osd_sub(&storage.get_io_ctx(), *obj2.get_oid(), key, sub_val); + ret = rados_utils.osd_sub(storage.get_io_ctx_wrapper(), *obj2.get_oid(), key, sub_val); // get the value! ASSERT_EQ(0, ret); std::set keys; @@ -867,16 +857,15 @@ TEST(librmb, increment_sub_from_existing_key) { ceph::bufferlist bl = (*it).second; EXPECT_EQ(bl.to_str(), "5"); - storage.delete_mail(&obj2); + storage.delete_mail(*obj2.get_oid()); // tear down cluster.deinit(); + delete obj2.get_mail_buffer(); } /** * RmbCommands load objects */ TEST(librmb, rmb_load_objects) { - librados::IoCtx io_ctx; - librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -884,57 +873,66 @@ TEST(librmb, rmb_load_objects) { std::string ns("t1"); int open_connection = storage.open_connection(pool_name); + EXPECT_EQ(0, open_connection); - librmb::RadosCephConfig ceph_cfg(&storage.get_io_ctx()); - EXPECT_EQ(0, ceph_cfg.save_cfg()); + storage_interface::RadosCephConfig *ceph_cfg= + storage_engine::StorageBackendFactory::create_ceph_config_io(storage_engine::CEPH,storage.get_io_ctx_wrapper()); + EXPECT_EQ(0, ceph_cfg->save_cfg()); std::map opts; opts["pool"] = pool_name; opts["namespace"] = ns; opts["print_cfg"] = "true"; - opts["cfg_obj"] = ceph_cfg.get_cfg_object_name(); + opts["cfg_obj"] = ceph_cfg->get_cfg_object_name(); - librmb::RmbCommands rmb_commands(&storage, &cluster, &opts); + librmb::RmbCommandsImpl rmb_commands(&storage, &cluster, &opts); /* update config rmb_commands.configuration(false, ceph_cfg); */ // load metadata info std::string uid; - librmb::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); EXPECT_NE(nullptr, ms); storage.set_namespace(ns); - librmb::RadosMail obj2; - obj2.set_oid("myobject"); - - ceph::bufferlist mail_buf; - storage.save_mail(*obj2.get_oid(), mail_buf); - - std::list mail_objects; + librmb::RadosMailImpl obj2; + obj2.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj2.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length = ((librados::bufferlist*)obj2.get_mail_buffer())->length(); + obj2.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj2.set_oid("test_oid"); + storage.save_mail(&obj2); + std::list mail_objects; std::string sort_string = "uid"; - + // std::set find_mails_set =storage.find_mails(nullptr); + EXPECT_EQ(0, mail_objects.size()); EXPECT_EQ(0, rmb_commands.load_objects(ms, mail_objects, sort_string)); + // EXPECT_EQ(4,find_mails_set.size()); EXPECT_EQ(1, mail_objects.size()); - for (std::list::iterator it = mail_objects.begin(); it != mail_objects.end(); ++it) { - librmb::RadosMail *obj = *it; + for (std::list::iterator it = mail_objects.begin(); it != mail_objects.end(); ++it) { + storage_interface::RadosMail *obj = *it; delete obj; } - storage.delete_mail(&obj2); - storage.delete_mail(ceph_cfg.get_cfg_object_name()); + storage.delete_mail(*obj2.get_oid()); + std::cout<<"*obj2.get_oid()::"<<*obj2.get_oid()<get_cfg_object_name()); + std::cout<<"ceph_cfg.get_cfg_object_name()"<get_cfg_object_name()<save_cfg()); std::map opts; opts["pool"] = pool_name; opts["namespace"] = ns; opts["print_cfg"] = "true"; - opts["cfg_obj"] = ceph_cfg.get_cfg_object_name(); + opts["cfg_obj"] = ceph_cfg->get_cfg_object_name(); - librmb::RmbCommands rmb_commands(&storage, &cluster, &opts); + librmb::RmbCommandsImpl rmb_commands(&storage, &cluster, &opts); // load metadata info std::string uid; - librmb::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); EXPECT_NE(nullptr, ms); storage.set_namespace(ns); - librmb::RadosMail obj2; - librados::bufferlist *buffer = new librados::bufferlist(); - obj2.set_mail_buffer(buffer); - obj2.set_oid("myobject_valid"); - obj2.get_mail_buffer()->append("hallo_welt"); // make sure obj is not empty. - obj2.set_mail_size(obj2.get_mail_buffer()->length()); - librados::ObjectWriteOperation write_op; // = new librados::ObjectWriteOperation(); + storage_interface::RadosMail *obj2 = new librmb::RadosMailImpl(); + obj2->set_mail_buffer(storage.alloc_mail_buffer()); + obj2->set_oid("myobject_valid"); + ((librados::bufferlist*)obj2->get_mail_buffer())->append("hallo_welt"); // make sure obj is not empty. + obj2->set_mail_size(((librados::bufferlist*)obj2->get_mail_buffer())->length()); { std::string key = "M"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "I"; std::string val = "v0.1"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "G"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "R"; std::string val = "1234567"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "S"; std::string val = "1234561117"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "P"; std::string val = "1"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "O"; std::string val = "0"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "Z"; std::string val = "200"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "V"; std::string val = "250"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "U"; std::string val = "1"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "A"; std::string val = ""; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "F"; std::string val = "01"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "B"; std::string val = "DRAFTS"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } - // convert metadata to xattr. and add to write_op - ms->save_metadata(&write_op, &obj2); + librmb::RadosMetadataImpl *xattr=new librmb::RadosMetadataImpl(); + ms->set_metadata(obj2); + // save complete mail. - EXPECT_EQ(true, storage.save_mail(&write_op, &obj2)); - std::list list; - list.push_back(&obj2); - - EXPECT_EQ(true, !storage.wait_for_rados_operations(list)); - - std::list mail_objects; + EXPECT_EQ(true, storage.save_mail(obj2)); + std::list mail_objects; std::string sort_string = "uid"; - EXPECT_EQ(0, rmb_commands.load_objects(ms, mail_objects, sort_string)); // there needs to be one mail EXPECT_EQ(1, mail_objects.size()); - - storage.delete_mail(&obj2); - storage.delete_mail(ceph_cfg.get_cfg_object_name()); - for (std::list::iterator it = mail_objects.begin(); it != mail_objects.end(); ++it) { - librmb::RadosMail *obj = *it; - storage.delete_mail(obj); + storage.delete_mail(*obj2->get_oid()); + storage.delete_mail(ceph_cfg->get_cfg_object_name()); + for (std::list::iterator it = mail_objects.begin(); it != mail_objects.end(); ++it) { + storage_interface::RadosMail *obj = *it; + storage.delete_mail(*obj->get_oid()); delete obj; } - delete ms; - + delete xattr; mail_objects.clear(); // tear down cluster.deinit(); + delete obj2->get_mail_buffer(); + delete ceph_cfg; + delete obj2; } /** * Test RmbCommands load objects */ TEST(librmb, rmb_load_objects_invalid_metadata) { - librados::IoCtx io_ctx; - librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); @@ -1092,136 +1083,135 @@ TEST(librmb, rmb_load_objects_invalid_metadata) { int open_connection = storage.open_connection(pool_name); EXPECT_EQ(0, open_connection); - librmb::RadosCephConfig ceph_cfg(&storage.get_io_ctx()); - EXPECT_EQ(0, ceph_cfg.save_cfg()); + storage_interface::RadosCephConfig *ceph_cfg= + storage_engine::StorageBackendFactory::create_ceph_config_io(storage_engine::CEPH,storage.get_io_ctx_wrapper()); + EXPECT_EQ(0, ceph_cfg->save_cfg()); std::map opts; opts["pool"] = pool_name; opts["namespace"] = ns; opts["print_cfg"] = "true"; - opts["cfg_obj"] = ceph_cfg.get_cfg_object_name(); + opts["cfg_obj"] = ceph_cfg->get_cfg_object_name(); - librmb::RmbCommands rmb_commands(&storage, &cluster, &opts); + librmb::RmbCommandsImpl rmb_commands(&storage, &cluster, &opts); // load metadata info std::string uid; - librmb::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); + storage_interface::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); EXPECT_NE(nullptr, ms); storage.set_namespace(ns); - librmb::RadosMail obj2; - librados::bufferlist *buffer = new librados::bufferlist(); - obj2.set_mail_buffer(buffer); - obj2.set_oid("myobject_invalid"); - obj2.get_mail_buffer()->append("hallo_welt"); // make sure obj is not empty. - obj2.set_mail_size(obj2.get_mail_buffer()->length()); - librados::ObjectWriteOperation write_op; // = new librados::ObjectWriteOperation(); + storage_interface::RadosMail *obj2 = new librmb::RadosMailImpl(); + obj2->set_mail_buffer(storage.alloc_mail_buffer()); + obj2->set_oid("myobject_invalid"); + ((librados::bufferlist*)obj2->get_mail_buffer())->append("hallo_welt"); // make sure obj is not empty. + obj2->set_mail_size(((librados::bufferlist*)obj2->get_mail_buffer())->length()); { std::string key = "M"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "I"; std::string val = "v0.1"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "G"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "R"; std::string val = "abnahsijsksisis"; // <-- This should be numeric - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "S"; std::string val = "1234561117"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "P"; std::string val = "1"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "O"; std::string val = "0"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "Z"; std::string val = "200"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "V"; std::string val = "250"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "U"; std::string val = "1"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "A"; std::string val = ""; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "F"; std::string val = "01"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } { std::string key = "B"; std::string val = "DRAFTS"; - librmb::RadosMetadata m(key, val); - obj2.add_metadata(m); + storage_interface::RadosMetadata *m = new librmb::RadosMetadataImpl(key, val); + obj2->add_metadata(m); } - // convert metadata to xattr. and add to write_op - ms->save_metadata(&write_op, &obj2); + librmb::RadosMetadataImpl *xattr=new librmb::RadosMetadataImpl(); + ms->set_metadata(obj2); // save complete mail. - EXPECT_EQ(true, storage.save_mail(&write_op, &obj2)); - std::list list; - list.push_back(&obj2); + EXPECT_EQ(true, storage.save_mail(obj2)); - EXPECT_EQ(true, !storage.wait_for_rados_operations(list)); - - std::list mail_objects; + std::list mail_objects; std::string sort_string = "uid"; EXPECT_EQ(0, rmb_commands.load_objects(ms, mail_objects, sort_string)); // no mail EXPECT_EQ(1, mail_objects.size()); - for (std::list::iterator it = mail_objects.begin(); it != mail_objects.end(); ++it) { - librmb::RadosMail *obj = *it; + for (std::list::iterator it = mail_objects.begin(); it != mail_objects.end(); ++it) { + storage_interface::RadosMail *obj = *it; delete obj; } - storage.delete_mail(&obj2); - storage.delete_mail(ceph_cfg.get_cfg_object_name()); + storage.delete_mail(*obj2->get_oid()); + storage.delete_mail(ceph_cfg->get_cfg_object_name()); delete ms; + delete xattr; mail_objects.clear(); // tear down cluster.deinit(); + delete obj2->get_mail_buffer(); + delete ceph_cfg; + delete obj2; } /** * Test RmbCommands @@ -1229,7 +1219,6 @@ TEST(librmb, rmb_load_objects_invalid_metadata) { TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file) { librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); - std::string pool_name("rmb_tool_tests"); std::string ns("t1"); @@ -1238,35 +1227,64 @@ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file) { storage.set_namespace(ns); std::string test_file_name = "test1.log"; - librmb::RadosSaveLog log_file(test_file_name); - EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save")); - EXPECT_EQ(true, log_file.close()); - librados::bufferlist bl; - EXPECT_EQ(0, storage.save_mail("abc", bl)); - - std::map> moved_items; - EXPECT_EQ(1, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); + storage_interface::RadosSaveLog *log_file = new librmb::RadosSaveLogImpl(test_file_name); + EXPECT_EQ(true, log_file->open()); + storage_interface::RadosSaveLogEntry *log_entry = new librmb::RadosSaveLogEntryImpl("abc", "t1", "rmb_tool_tests", "save"); + log_file->append(log_entry); + EXPECT_EQ(true, log_file->close()); + + librmb::RadosMailImpl obj; + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length =((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj.set_oid("abc"); + storage.save_mail(&obj); + + std::map> moved_items; + storage_interface::RmbCommands *rmb_cmds=storage_engine::StorageBackendFactory::create_rmb_commands_default( + storage_engine::CEPH); + EXPECT_EQ(1, rmb_cmds->delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); std::remove(test_file_name.c_str()); - cluster.deinit(); + delete rmb_cmds; + + delete obj.get_mail_buffer(); + for(std::map>::iterator it=moved_items.begin(); it != moved_items.end(); ++it){ + for(std::list::iterator iter = it->second.begin();iter != it->second.end(); ++iter){ + delete *iter; + *iter=nullptr; + } + } + delete log_file; + delete log_entry; } -/** - * Test RmbCommands - */ +// /** +// * Test RmbCommands +// */ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_file_not_found) { std::string pool_name("rmb_tool_tests"); std::string ns("t1"); std::string test_file_name = "test1.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc", "t1", "rmb_tool_tests", "save")); EXPECT_EQ(true, log_file.close()); - std::map> moved_items; - - EXPECT_EQ(0, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); + std::map> moved_items; + storage_interface::RmbCommands *rmb_cmds=storage_engine::StorageBackendFactory::create_rmb_commands_default( + storage_engine::CEPH); + EXPECT_EQ(0, rmb_cmds->delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); std::remove(test_file_name.c_str()); + delete rmb_cmds; + for(std::map>::iterator it=moved_items.begin(); it != moved_items.end(); ++it){ + for(std::list::iterator iter = it->second.begin();iter != it->second.end(); ++iter){ + delete *iter; + *iter=nullptr; + } + } } /** * Test RmbCommands @@ -1276,14 +1294,22 @@ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_file) { std::string ns("t1"); std::string test_file_name = "test1.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc", "t1", "rmb_tool_tests", "save")); EXPECT_EQ(true, log_file.close()); - std::map> moved_items; - - EXPECT_EQ(-1, librmb::RmbCommands::delete_with_save_log("test12.log", "ceph", "client.admin", &moved_items)); + std::map> moved_items; + storage_interface::RmbCommands *rmb_cmds=storage_engine::StorageBackendFactory::create_rmb_commands_default( + storage_engine::CEPH); + EXPECT_EQ(-1,rmb_cmds->delete_with_save_log("test12.log", "ceph", "client.admin", &moved_items)); std::remove(test_file_name.c_str()); + delete rmb_cmds; + for(std::map>::iterator it=moved_items.begin(); it != moved_items.end(); ++it){ + for(std::list::iterator iter = it->second.begin();iter != it->second.end(); ++iter){ + delete *iter; + *iter=nullptr; + } + } } /** * Test RmbCommands @@ -1291,7 +1317,6 @@ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_file) { TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_entry) { librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); - std::string pool_name("rmb_tool_tests"); std::string ns("t1"); @@ -1300,20 +1325,27 @@ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_entry) { storage.set_namespace(ns); std::string test_file_name = "test1.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); log_file.append( - librmb::RadosSaveLogEntry("abc2", "t1", "2,2,2rmb_tool_tests", "save")); // -> stop processing (invalid entry)! - log_file.append(librmb::RadosSaveLogEntry("abc2", "t1", "rmb_tool_tests", "save")); + &librmb::RadosSaveLogEntryImpl("abc2", "t1", "2,2,2rmb_tool_tests", "save")); // -> stop processing (invalid entry)! + log_file.append( + &librmb::RadosSaveLogEntryImpl("abc2", "t1", "rmb_tool_tests", "save")); EXPECT_EQ(true, log_file.close()); - librados::bufferlist bl; - - EXPECT_EQ(0, storage.save_mail("abc2", bl)); - cluster.deinit(); - std::map> moved_items; - - EXPECT_EQ(0, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph", "client.admin", + librmb::RadosMailImpl obj; + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj.set_oid("abc2"); + EXPECT_EQ(true,storage.save_mail(&obj)); + std::map> moved_items; + storage_interface::RmbCommands *rmb_cmds=storage_engine::StorageBackendFactory::create_rmb_commands_default( + storage_engine::CEPH); + EXPECT_EQ(0, rmb_cmds->delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); // -> due to invalid entry in object list std::remove(test_file_name.c_str()); @@ -1322,6 +1354,14 @@ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_entry) { storage.set_namespace(ns); EXPECT_EQ(storage.delete_mail("abc2"), 0); // check that save log processing does stop at invalid line! cluster.deinit(); + delete obj.get_mail_buffer(); + delete rmb_cmds; + for(std::map>::iterator it=moved_items.begin(); it != moved_items.end(); ++it){ + for(std::list::iterator iter = it->second.begin();iter != it->second.end(); ++iter){ + delete *iter; + *iter=nullptr; + } + } } /** * Test RmbCommands @@ -1329,47 +1369,63 @@ TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_entry) { TEST(librmb, move_object_delete_with_save_log) { librmb::RadosClusterImpl cluster; librmb::RadosStorageImpl storage(&cluster); - std::string pool_name("rmb_tool_tests"); std::string ns("t1"); int open_connection = storage.open_connection(pool_name); EXPECT_EQ(0, open_connection); storage.set_namespace(ns); - + std::string test_file_name = "test1.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc3", "t1", "rmb_tool_tests", - "mv:t1:abc3:t1;M=123:B=INBOX:U=1:G=0246da2269ac1f5b3e1700009c60b9f7")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc3", "t1", "rmb_tool_tests", "mv:t1:abc3:t1;M=123:B=INBOX:U=1:G=0246da2269ac1f5b3e1700009c60b9f7")); EXPECT_EQ(true, log_file.close()); - librados::bufferlist bl; - EXPECT_EQ(0, storage.save_mail("abc3", bl)); - cluster.deinit(); - - std::map> moved_items; - EXPECT_EQ(1, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); + + librmb::RadosMailImpl obj; + obj.set_mail_buffer(storage.alloc_mail_buffer()); + ((librados::bufferlist*)obj.get_mail_buffer())->append("abcdefghijklmn"); + int buffer_length = ((librados::bufferlist*)obj.get_mail_buffer())->length(); + obj.set_mail_size(buffer_length); + int max_size = buffer_length; + + obj.set_oid("abc3"); + EXPECT_EQ(true,storage.save_mail(&obj)); + + std::map> moved_items; + storage_interface::RmbCommands *rmb_cmds=storage_engine::StorageBackendFactory::create_rmb_commands_default( + storage_engine::CEPH); + EXPECT_EQ(1, rmb_cmds->delete_with_save_log("test1.log", "ceph", "client.admin", &moved_items)); + std::remove(test_file_name.c_str()); EXPECT_EQ(1, moved_items.size()); - std::list list = moved_items["t1"]; + std::list list = moved_items["t1"]; EXPECT_EQ(1, list.size()); - librmb::RadosSaveLogEntry entry = list.front(); - - EXPECT_EQ(entry.src_oid, "abc3"); - EXPECT_EQ(entry.src_ns, "t1"); - EXPECT_EQ(entry.src_user, "t1"); - - std::string key_guid(1, static_cast(librmb::RBOX_METADATA_GUID)); - std::list::iterator it_guid = - std::find_if(entry.metadata.begin(), entry.metadata.end(), - [key_guid](librmb::RadosMetadata const &m) { return m.key == key_guid; }); - EXPECT_EQ("0246da2269ac1f5b3e1700009c60b9f7", (*it_guid).bl.to_str()); + storage_interface::RadosSaveLogEntry* entry = list.front(); + + EXPECT_EQ(entry->get_src_oid(), "abc3"); + EXPECT_EQ(entry->get_src_ns(), "t1"); + EXPECT_EQ(entry->get_src_user(), "t1"); + std::string key_guid(1, static_cast(storage_interface::RBOX_METADATA_GUID)); + std::list::iterator it_guid = + std::find_if(entry->get_metadata().begin(), entry->get_metadata().end(), + [key_guid](storage_interface::RadosMetadata* const m) { return m->get_key() == key_guid; }); + std::cout<< ((ceph::bufferlist*)((*it_guid)->get_buffer()))->to_str() <get_buffer()))->to_str()); - std::remove(test_file_name.c_str()); open_connection = storage.open_connection(pool_name); EXPECT_EQ(0, open_connection); storage.set_namespace(ns); - EXPECT_EQ(storage.delete_mail("abc3"), 0); // move does not delete the object + EXPECT_EQ(storage.delete_mail("abc3"), 0); cluster.deinit(); + delete obj.get_mail_buffer(); + delete rmb_cmds; + for(std::map>::iterator it=moved_items.begin(); it != moved_items.end(); ++it){ + for(std::list::iterator iter = it->second.begin();iter != it->second.end(); ++iter){ + delete *iter; + *iter=nullptr; + } + } + } TEST(librmb, mock_obj) {} int main(int argc, char **argv) { diff --git a/src/tests/librmb/test_librmb_utils.cpp b/src/tests/librmb/test_librmb_utils.cpp index 2f780c4a..ed0a4a52 100644 --- a/src/tests/librmb/test_librmb_utils.cpp +++ b/src/tests/librmb/test_librmb_utils.cpp @@ -13,109 +13,113 @@ #include #include "../../librmb/rados-cluster-impl.h" -#include "../../librmb/rados-ceph-json-config.h" #include "../../librmb/rados-storage-impl.h" +#include "../../librmb/rados-ceph-json-config-impl.h" #include "mock_test.h" #include "gtest/gtest.h" #include "gmock/gmock.h" -#include "rados-util.h" -#include "rados-types.h" -#include "rados-save-log.h" -#include "rados-mail.h" +#include "../../librmb/rados-util-impl.h" +#include "../../storage-interface/rados-types.h" +#include "../../storage-interface/rados-mail.h" +#include "../../storage-engine/storage-backend-factory.h" #include #include +#include "../../librmb/rados-metadata-impl.h" +#include "../../storage-interface/rados-save-log.h" +#include "../../librmb/rados-util-impl.h" using ::testing::AtLeast; using ::testing::Return; TEST(librmb, get_metadata_1) { - enum librmb::rbox_metadata_key key = librmb::rbox_metadata_key::RBOX_METADATA_GUID; - librmb::RadosMetadata m(key, "abcdefg"); - librmb::RadosMail mail; - mail.add_metadata(m); + enum storage_interface::rbox_metadata_key key = storage_interface::rbox_metadata_key::RBOX_METADATA_GUID; + librmb::RadosMetadataImpl m(key, "abcdefg"); + storage_interface::RadosMail *mail= + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + mail->add_metadata(&m); char *val = NULL; - - librmb::RadosUtils::get_metadata(key, mail.get_metadata(), &val); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(key, mail->get_metadata(), &val); std::cout << val << std::endl; EXPECT_STREQ(val, "abcdefg\0"); } TEST(librmb, convert_enum) { - enum librmb::rbox_metadata_key key = librmb::rbox_metadata_key::RBOX_METADATA_GUID; + enum storage_interface::rbox_metadata_key key = storage_interface::rbox_metadata_key::RBOX_METADATA_GUID; - std::string metadata_key(librmb::rbox_metadata_key_to_char(key)); + std::string metadata_key(storage_interface::rbox_metadata_key_to_char(key)); EXPECT_EQ("G", metadata_key); - key = librmb::RBOX_METADATA_MAILBOX_GUID; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_MAILBOX_GUID; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("M", metadata_key); - key = librmb::RBOX_METADATA_GUID; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_GUID; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("G", metadata_key); - key = librmb::RBOX_METADATA_POP3_UIDL; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_POP3_UIDL; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("P", metadata_key); - key = librmb::RBOX_METADATA_POP3_ORDER; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_POP3_ORDER; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("O", metadata_key); - key = librmb::RBOX_METADATA_RECEIVED_TIME; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_RECEIVED_TIME; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("R", metadata_key); - key = librmb::RBOX_METADATA_PHYSICAL_SIZE; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_PHYSICAL_SIZE; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("Z", metadata_key); - key = librmb::RBOX_METADATA_VIRTUAL_SIZE; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_VIRTUAL_SIZE; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("V", metadata_key); - key = librmb::RBOX_METADATA_EXT_REF; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_EXT_REF; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("X", metadata_key); - key = librmb::RBOX_METADATA_ORIG_MAILBOX; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_ORIG_MAILBOX; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("B", metadata_key); - key = librmb::RBOX_METADATA_MAIL_UID; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_MAIL_UID; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("U", metadata_key); - key = librmb::RBOX_METADATA_VERSION; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_VERSION; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("I", metadata_key); - key = librmb::RBOX_METADATA_FROM_ENVELOPE; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_FROM_ENVELOPE; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("A", metadata_key); - key = librmb::RBOX_METADATA_PVT_FLAGS; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_PVT_FLAGS; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("C", metadata_key); - key = librmb::RBOX_METADATA_OLDV1_EXPUNGED; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_OLDV1_EXPUNGED; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("E", metadata_key); - key = librmb::RBOX_METADATA_OLDV1_FLAGS; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_OLDV1_FLAGS; + metadata_key =storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("F", metadata_key); - key = librmb::RBOX_METADATA_OLDV1_KEYWORDS; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_OLDV1_KEYWORDS; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("K", metadata_key); - key = librmb::RBOX_METADATA_OLDV1_SAVE_TIME; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ("S", metadata_key); - key = librmb::RBOX_METADATA_OLDV1_SPACE; - metadata_key = librmb::rbox_metadata_key_to_char(key); + key = storage_interface::RBOX_METADATA_OLDV1_SPACE; + metadata_key = storage_interface::rbox_metadata_key_to_char(key); EXPECT_EQ(" ", metadata_key); } @@ -123,72 +127,79 @@ TEST(librmb, utils_convert_str_to_time) { time_t test_time; // %Y-%m-%d %H:%M:%S const std::string test_string = "2017-10-10 12:12:12"; - EXPECT_TRUE(librmb::RadosUtils::convert_str_to_time_t(test_string, &test_time)); + librmb::RadosUtilsImpl rados_util; + EXPECT_TRUE(rados_util.convert_str_to_time_t(test_string, &test_time)); // EXPECT_EQ(test_time, 1507630332); EXPECT_NE(0, test_time); - EXPECT_FALSE(librmb::RadosUtils::convert_str_to_time_t("2017-2-2 ", &test_time)); - EXPECT_FALSE(librmb::RadosUtils::convert_str_to_time_t("", &test_time)); + EXPECT_FALSE(rados_util.convert_str_to_time_t("2017-2-2 ", &test_time)); + EXPECT_FALSE(rados_util.convert_str_to_time_t("", &test_time)); } TEST(librmb, utils_is_numeric) { - EXPECT_TRUE(librmb::RadosUtils::is_numeric("12345678")); - EXPECT_FALSE(librmb::RadosUtils::is_numeric("abchdjdkd")); - EXPECT_FALSE(librmb::RadosUtils::is_numeric("1234Abvsusi§EEE")); + librmb::RadosUtilsImpl rados_util; + EXPECT_TRUE(rados_util.is_numeric("12345678")); + EXPECT_FALSE(rados_util.is_numeric("abchdjdkd")); + EXPECT_FALSE(rados_util.is_numeric("1234Abvsusi§EEE")); } TEST(librmb, utils_is_date_attribute) { - enum librmb::rbox_metadata_key key = librmb::RBOX_METADATA_RECEIVED_TIME; - EXPECT_TRUE(librmb::RadosUtils::is_date_attribute(key)); - enum librmb::rbox_metadata_key key2 = librmb::RBOX_METADATA_OLDV1_SAVE_TIME; - EXPECT_TRUE(librmb::RadosUtils::is_date_attribute(key2)); - - enum librmb::rbox_metadata_key key3 = librmb::RBOX_METADATA_VERSION; - EXPECT_FALSE(librmb::RadosUtils::is_date_attribute(key3)); + librmb::RadosUtilsImpl rados_util; + enum storage_interface::rbox_metadata_key key = storage_interface::RBOX_METADATA_RECEIVED_TIME; + EXPECT_TRUE(rados_util.is_date_attribute(key)); + enum storage_interface::rbox_metadata_key key2 = storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME; + EXPECT_TRUE(rados_util.is_date_attribute(key2)); + + enum storage_interface::rbox_metadata_key key3 = storage_interface::RBOX_METADATA_VERSION; + EXPECT_FALSE(rados_util.is_date_attribute(key3)); } TEST(librmb, utils_convert_string_to_date) { + librmb::RadosUtilsImpl rados_util; std::string date_str = "2017-10-10 12:12:12"; std::string str; - librmb::RadosUtils::convert_string_to_date(date_str, &str); + rados_util.convert_string_to_date(date_str, &str); // EXPECT_EQ(str, "1507630332"); EXPECT_NE("0", "1507630332"); std::string test_str = "asjsjsjsj09202920"; std::string str2; - librmb::RadosUtils::convert_string_to_date(test_str, &str2); + rados_util.convert_string_to_date(test_str, &str2); EXPECT_EQ(str2, ""); } TEST(librmb, utils_convert_convert_time_t_to_str) { + librmb::RadosUtilsImpl rados_util; time_t test_time = 1507630332; std::string test_str; - EXPECT_EQ(0, librmb::RadosUtils::convert_time_t_to_str(test_time, &test_str)); + EXPECT_EQ(0, rados_util.convert_time_t_to_str(test_time, &test_str)); // EXPECT_EQ(test_str, "2017-10-10 12:12:12"); EXPECT_NE("", test_str); } TEST(librmb, config_mutable_metadata) { - librmb::RadosCephJsonConfig config; + librmb::RadosCephJsonConfigImpl config; std::string str = "MGP"; config.update_mail_attribute(str.c_str()); - EXPECT_TRUE(config.is_mail_attribute(librmb::RBOX_METADATA_MAILBOX_GUID)); - EXPECT_TRUE(config.is_mail_attribute(librmb::RBOX_METADATA_GUID)); - EXPECT_TRUE(config.is_mail_attribute(librmb::RBOX_METADATA_POP3_UIDL)); - EXPECT_FALSE(config.is_mail_attribute(librmb::RBOX_METADATA_ORIG_MAILBOX)); + EXPECT_TRUE(config.is_mail_attribute(storage_interface::RBOX_METADATA_MAILBOX_GUID)); + EXPECT_TRUE(config.is_mail_attribute(storage_interface::RBOX_METADATA_GUID)); + EXPECT_TRUE(config.is_mail_attribute(storage_interface::RBOX_METADATA_POP3_UIDL)); + EXPECT_FALSE(config.is_mail_attribute(storage_interface::RBOX_METADATA_ORIG_MAILBOX)); // use defaults. - librmb::RadosCephJsonConfig config2; + librmb::RadosCephJsonConfigImpl config2; config2.update_mail_attribute(NULL); - EXPECT_TRUE(config2.is_mail_attribute(librmb::RBOX_METADATA_POP3_UIDL)); + EXPECT_TRUE(config2.is_mail_attribute(storage_interface::RBOX_METADATA_POP3_UIDL)); } TEST(librmb, convert_flags) { + librmb::RadosUtilsImpl rados_util; uint8_t flags = 0x3f; std::string s; - librmb::RadosUtils::flags_to_string(flags, &s); + rados_util.flags_to_string(flags, &s); uint8_t flags_; - librmb::RadosUtils::string_to_flags(s, &flags_); + rados_util.string_to_flags(s, &flags_); EXPECT_EQ(flags, flags_); } TEST(librmb, find_and_replace) { + librmb::RadosUtilsImpl rados_util; const std::string str_red_house = "i have a red house and a red car"; const std::string str_banana = "i love banana"; const std::string str_underscore = "some_words_separated_by_underscore"; @@ -198,47 +209,47 @@ TEST(librmb, find_and_replace) { std::string text; text = "i have a blue house and a blue car"; - librmb::RadosUtils::find_and_replace(&text, "blue", "red"); + rados_util.find_and_replace(&text, "blue", "red"); EXPECT_EQ(str_red_house, text); text = "i love apple"; - librmb::RadosUtils::find_and_replace(&text, "apple", "banana"); + rados_util.find_and_replace(&text, "apple", "banana"); EXPECT_EQ(str_banana, text); text = "some-words-separated-by-hyphen"; - librmb::RadosUtils::find_and_replace(&text, "-", "_"); - librmb::RadosUtils::find_and_replace(&text, "hyphen", "underscore"); + rados_util.find_and_replace(&text, "-", "_"); + rados_util.find_and_replace(&text, "hyphen", "underscore"); EXPECT_EQ(str_underscore, text); text = "this string has an is missing"; - librmb::RadosUtils::find_and_replace(&text, "is", ""); + rados_util.find_and_replace(&text, "is", ""); EXPECT_EQ(str_missing, text); text = "hello;world;"; - librmb::RadosUtils::find_and_replace(&text, ";", " "); + rados_util.find_and_replace(&text, ";", " "); EXPECT_EQ(str_hello, text); } TEST(librmb, append_to_new_log_file) { std::string test_file_name = "test.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc", "ns_1", "mail_storage", "save")); EXPECT_EQ(true, log_file.close()); int line_count = 0; /** check content **/ std::ifstream read(test_file_name); while (true) { - librmb::RadosSaveLogEntry entry; + librmb::RadosSaveLogEntryImpl entry; read >> entry; if (read.eof()) { break; } - EXPECT_EQ(entry.oid, "abc"); - EXPECT_EQ(entry.ns, "ns_1"); - EXPECT_EQ(entry.pool, "mail_storage"); - EXPECT_EQ(entry.op, "save"); + EXPECT_EQ(entry.get_oid(), "abc"); + EXPECT_EQ(entry.get_ns(), "ns_1"); + EXPECT_EQ(entry.get_pool(), "mail_storage"); + EXPECT_EQ(entry.get_op(), "save"); line_count++; } EXPECT_EQ(1, line_count); @@ -248,29 +259,29 @@ TEST(librmb, append_to_new_log_file) { TEST(librmb, append_to_existing_file_log_file) { std::string test_file_name = "test.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc", "ns_1", "mail_storage", "save")); EXPECT_EQ(true, log_file.close()); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc", "ns_1", "mail_storage", "save")); EXPECT_EQ(true, log_file.close()); int line_count = 0; /** check content **/ std::ifstream read(test_file_name); while (true) { - librmb::RadosSaveLogEntry entry; + librmb::RadosSaveLogEntryImpl entry; read >> entry; if (read.eof()) { break; } - EXPECT_EQ(entry.oid, "abc"); - EXPECT_EQ(entry.ns, "ns_1"); - EXPECT_EQ(entry.pool, "mail_storage"); - EXPECT_EQ(entry.op, "save"); + EXPECT_EQ(entry.get_oid(), "abc"); + EXPECT_EQ(entry.get_ns(), "ns_1"); + EXPECT_EQ(entry.get_pool(), "mail_storage"); + EXPECT_EQ(entry.get_op(), "save"); line_count++; } EXPECT_EQ(2, line_count); @@ -280,10 +291,10 @@ TEST(librmb, append_to_existing_file_log_file) { __attribute__((noreturn)) static void *write_to_save_file(void *threadid) { std::string test_file_name = "test1.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); for (int i = 0; i < 5; i++) { - log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save")); + log_file.append(&librmb::RadosSaveLogEntryImpl("abc", "ns_1", "mail_storage", "save")); } EXPECT_EQ(true, log_file.close()); std::cout << " exiting thread " << threadid << std::endl; @@ -322,16 +333,16 @@ TEST(librmb, append_to_existing_file_multi_threading) { /** check content **/ std::ifstream read(test_file_name); while (true) { - librmb::RadosSaveLogEntry entry; + librmb::RadosSaveLogEntryImpl entry; read >> entry; if (read.eof()) { break; } - EXPECT_EQ(entry.oid, "abc"); - EXPECT_EQ(entry.ns, "ns_1"); - EXPECT_EQ(entry.pool, "mail_storage"); - EXPECT_EQ(entry.op, "save"); + EXPECT_EQ(entry.get_oid(), "abc"); + EXPECT_EQ(entry.get_ns(), "ns_1"); + EXPECT_EQ(entry.get_pool(), "mail_storage"); + EXPECT_EQ(entry.get_op(), "save"); line_count++; } EXPECT_EQ(25, line_count); @@ -343,38 +354,52 @@ TEST(librmb, append_to_existing_file_multi_threading) { } TEST(librmb, test_mvn_option) { - std::list metadata; - librmb::RadosMetadata guid(librmb::RBOX_METADATA_MAILBOX_GUID, "ABCDEFG"); - librmb::RadosMetadata mb_name(librmb::RBOX_METADATA_ORIG_MAILBOX, "INBOX"); + std::list metadata; + storage_interface::RadosMetadata *guid= + storage_engine::StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, storage_interface::RBOX_METADATA_MAILBOX_GUID, "ABCDEFG"); + + storage_interface::RadosMetadata *mb_name= + storage_engine::StorageBackendFactory::create_metadata_string( + storage_engine::CEPH, storage_interface::RBOX_METADATA_ORIG_MAILBOX, "INBOX"); + uint uid_ = 1; - librmb::RadosMetadata uid(librmb::RBOX_METADATA_MAIL_UID, uid_); - metadata.push_back(&guid); - metadata.push_back(&mb_name); - metadata.push_back(&uid); + storage_interface::RadosMetadata *uid= + storage_engine::StorageBackendFactory::create_metadata_uint( + storage_engine::CEPH, storage_interface::RBOX_METADATA_MAIL_UID, uid_); + + metadata.push_back(guid); + metadata.push_back(mb_name); + metadata.push_back(uid); std::string test_file_name = "test_2.log"; - librmb::RadosSaveLog log_file(test_file_name); + librmb::RadosSaveLogImpl log_file(test_file_name); EXPECT_EQ(true, log_file.open()); - log_file.append(librmb::RadosSaveLogEntry("dest_oid", "ns_dest", "mail_storage", - librmb::RadosSaveLogEntry::op_mv("ns_src", "src_oid", "user", metadata))); + storage_interface::RadosSaveLogEntry *save_log_entry= + storage_engine::StorageBackendFactory::create_save_log_entry_default(storage_engine::CEPH); + log_file.append(&librmb::RadosSaveLogEntryImpl("dest_oid", "ns_dest", "mail_storage", + save_log_entry->op_mv("ns_src", "src_oid", "user", metadata))); std::ifstream read(test_file_name); while (true) { - librmb::RadosSaveLogEntry entry; + librmb::RadosSaveLogEntryImpl entry; read >> entry; if (read.eof()) { break; } - EXPECT_EQ(entry.oid, "dest_oid"); - EXPECT_EQ(entry.ns, "ns_dest"); - EXPECT_EQ(entry.pool, "mail_storage"); - EXPECT_EQ(entry.op, "mv:ns_src:src_oid:user;M=ABCDEFG:B=INBOX:U=1\0"); - EXPECT_EQ(entry.metadata.size(), 3); + EXPECT_EQ(entry.get_oid(), "dest_oid"); + EXPECT_EQ(entry.get_ns(), "ns_dest"); + EXPECT_EQ(entry.get_pool(), "mail_storage"); + EXPECT_EQ(entry.get_op(), "mv:ns_src:src_oid:user;M=ABCDEFG:B=INBOX:U=1\0"); + EXPECT_EQ(entry.get_metadata().size(), 3); } - + delete guid; + delete mb_name; + delete uid; read.close(); // EXPECT_EQ(1, 0); std::remove(test_file_name.c_str()); + delete save_log_entry; } /*TEST(librmb, test_if) { diff --git a/src/tests/mocks/mock_test.h b/src/tests/mocks/mock_test.h index 0e3f5f0a..29b19608 100644 --- a/src/tests/mocks/mock_test.h +++ b/src/tests/mocks/mock_test.h @@ -12,117 +12,124 @@ #include #include #include -#include "rados-types.h" -#include "../../librmb/rados-cluster.h" -#include "../../librmb/rados-dictionary.h" +#include "../../storage-interface/rados-types.h" +#include "../../storage-interface/rados-cluster.h" +#include "../../storage-interface/rados-dictionary.h" #include "../../librmb/rados-dovecot-config.h" -#include "../../librmb/rados-storage.h" -#include "../../librmb/rados-dovecot-ceph-cfg.h" -#include "../../librmb/rados-metadata-storage-impl.h" +#include "../../storage-interface/rados-storage.h" +#include "../../storage-interface/rados-dovecot-ceph-cfg.h" #include "../../librmb/rados-metadata-storage-impl.h" +#include "../../storage-interface/rados-mail.h" +#include "../../storage-interface/rbox-io-ctx.h" #include "gmock/gmock.h" +#include "../../storage-interface/rados-metadata.h" namespace librmbtest { -using librmb::RadosMail; -using librmb::RadosMetadata; -using librmb::RadosMetadataStorage; -using librmb::RadosStorage; -using librmb::RadosStorageMetadataModule; +using storage_interface::RadosMail; +using storage_interface::RadosMetadata; +using storage_interface::RadosMetadataStorage; +using storage_interface::RadosStorage; +using storage_interface::RadosStorageMetadataModule; +using storage_interface::RboxIoCtx; + +class RboxIoCtxMock : public RboxIoCtx{ + public: + MOCK_METHOD1(set_io_ctx,void(librados::IoCtx& io_ctx_)); + MOCK_METHOD0(get_io_ctx,librados::IoCtx& ()); + MOCK_METHOD0(get_recovery_io_ctx,librados::IoCtx& ()); + MOCK_METHOD5(exec,int(const std::string& oid, const char *cls, const char *method,ceph::bufferlist& inbl, ceph::bufferlist& outbl)); + MOCK_METHOD4(aio_stat,int(const std::string& oid,librados::AioCompletion *aio_complete,uint64_t *psize,time_t *pmtime)); + MOCK_METHOD3(omap_get_vals_by_keys,int(const std::string& oid,const std::set& keys,std::map *vals)); + MOCK_METHOD2(omap_rm_keys,int(const std::string& oid,const std::set& keys)); + MOCK_METHOD2(omap_set,int(const std::map& map,const std::string& oid)); + MOCK_METHOD2(omap_set,void(const std::string& oid,const std::map& map)); + MOCK_METHOD2(getxattrs,int(const std::string& oid,std::map& attrset)); + MOCK_METHOD3(setxattr,int(const std::string& oid,const char *name, librados::bufferlist& bl)); + MOCK_METHOD0(nobjects_end,const librados::NObjectIterator& ()); + MOCK_METHOD0(nobjects_begin,librados::NObjectIterator()); + MOCK_METHOD1(nobjects_begin,librados::NObjectIterator(const librados::bufferlist& filter)); + MOCK_METHOD1(set_namespace,void(const std::string& nspace)); + MOCK_METHOD3(stat,int(const std::string& oid, uint64_t *psize, time_t *pmtime)); + MOCK_METHOD4(aio_operate,int(const std::string& oid, librados::AioCompletion *aio_completion, + librados::ObjectReadOperation *read_op, librados::bufferlist *pbl)); + MOCK_METHOD5(aio_operate,int(const std::string& oid, librados::AioCompletion *aio_completion, + librados::ObjectReadOperation *read_op, int flags,librados::bufferlist *pbl)); + MOCK_METHOD3(aio_operate,int(const std::string& oid, librados::AioCompletion *aio_complete, librados::ObjectWriteOperation *write_op)); + MOCK_METHOD1(remove,int(const std::string& oid)); + MOCK_METHOD2(write_full,int(const std::string& oid, librados::bufferlist& bl)); + MOCK_METHOD4(read,int(const std::string& oid, librados::bufferlist& bl, size_t len, uint64_t off)); + MOCK_METHOD2(operate,int(const std::string& oid, librados::ObjectWriteOperation* write_op_xattr)); + MOCK_METHOD3(append,bool(const std::string& oid, librados::bufferlist& bufferlist, int length)); + MOCK_METHOD3(operate,int(const std::string& oid,librados::ObjectReadOperation* read_op,librados::bufferlist* buffer)); + MOCK_METHOD0(get_last_version,uint64_t()); + MOCK_METHOD1(set_remove_completion,void(librados::AioCompletion &aio_commepletion)); + MOCK_METHOD0(get_remove_completion,librados::AioCompletion&()); + MOCK_METHOD1(set_push_back_completion,void(librados::AioCompletion &aio_commepletion)); + MOCK_METHOD0(get_push_back_completion,librados::AioCompletion&()); +}; class RadosStorageMock : public RadosStorage { public: - MOCK_METHOD0(get_io_ctx, librados::IoCtx &()); - MOCK_METHOD0(get_recovery_io_ctx, librados::IoCtx &()); - + MOCK_METHOD0(get_io_ctx_wrapper,storage_interface::RboxIoCtx* ()); MOCK_METHOD3(stat_mail, int(const std::string &oid, uint64_t *psize, time_t *pmtime)); MOCK_METHOD1(set_namespace, void(const std::string &nspace)); MOCK_METHOD0(get_namespace, std::string()); - MOCK_METHOD0(get_pool_name, std::string()); - MOCK_METHOD0(get_max_write_size, int()); MOCK_METHOD0(get_max_write_size_bytes, int()); MOCK_METHOD0(get_max_object_size, int()); - - - MOCK_METHOD3(split_buffer_and_exec_op, int(RadosMail *current_object, librados::ObjectWriteOperation *write_op_xattr, - const uint64_t &max_write)); - - MOCK_METHOD1(delete_mail, int(RadosMail *mail)); MOCK_METHOD1(delete_mail, int(const std::string &oid)); - MOCK_METHOD4(aio_operate, int(librados::IoCtx *io_ctx_, const std::string &oid, librados::AioCompletion *c, - librados::ObjectWriteOperation *op)); - MOCK_METHOD1(find_mails, librados::NObjectIterator(const RadosMetadata *attr)); + MOCK_METHOD1(find_mails, std::set(const storage_interface::RadosMetadata *attr)); MOCK_METHOD1(open_connection, int(const std::string &poolname)); MOCK_METHOD2(open_connection, int(const std::string &poolname, const std::string &index_pool)); - - MOCK_METHOD4(find_mails_async, std::set(const RadosMetadata *attr, std::string &pool_name,int num_threads, void (*ptr)(std::string&))); + MOCK_METHOD4(find_mails_async, std::set(const storage_interface::RadosMetadata *attr, std::string &pool_name,int num_threads, void (*ptr)(std::string&))); MOCK_METHOD4(open_connection, int(const std::string &poolname, const std::string &index_pool, const std::string &clustername, const std::string &rados_username)); MOCK_METHOD3(open_connection, int(const std::string &poolname, const std::string &clustername, const std::string &rados_username)); + MOCK_METHOD0(close_connection, void()); - MOCK_METHOD2(wait_for_write_operations_complete, - bool(librados::AioCompletion *completion, librados::ObjectWriteOperation *write_operation)); - MOCK_METHOD1(wait_for_rados_operations, bool(const std::list &object_list)); - MOCK_METHOD1(set_ceph_wait_method, void(enum librmb::rbox_ceph_aio_wait_method wait_method)); - MOCK_METHOD2(read_mail, int(const std::string &oid, librados::bufferlist *buffer)); + MOCK_METHOD1(set_ceph_wait_method, void(enum storage_interface::rbox_ceph_aio_wait_method wait_method)); + MOCK_METHOD3(read_mail, int(const std::string &oid, storage_interface::RadosMail* mail,int try_counter)); MOCK_METHOD6(move, int(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update, bool delete_source)); + std::list &to_update, bool delete_source)); MOCK_METHOD5(copy, int(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, - std::list &to_update)); - MOCK_METHOD2(save_mail, int(const std::string &oid, librados::bufferlist &bufferlist)); - MOCK_METHOD1(save_mail, bool(RadosMail *mail)); - MOCK_METHOD2(save_mail, bool(librados::ObjectWriteOperation *write_op, RadosMail *mail)); - MOCK_METHOD0(alloc_rados_mail, librmb::RadosMail *()); - - MOCK_METHOD1(free_rados_mail, void(librmb::RadosMail *mail)); - + std::list &to_update)); + MOCK_METHOD1(save_mail, bool(storage_interface::RadosMail *mail)); + MOCK_METHOD0(alloc_rados_mail, storage_interface::RadosMail *()); + MOCK_METHOD1(free_rados_mail, void(storage_interface::RadosMail *mail)); MOCK_METHOD0(create_anker, int()); - MOCK_METHOD0(ceph_index_size,uint64_t()); - MOCK_METHOD1(ceph_index_append,int(const std::string &oid)); MOCK_METHOD1(ceph_index_append,int(const std::set &oids)); MOCK_METHOD1(ceph_index_overwrite,int(const std::set &oids)); MOCK_METHOD0(ceph_index_read,std::set()); MOCK_METHOD0(ceph_index_delete,int()); + MOCK_METHOD0(alloc_mail_buffer,void*()); + MOCK_METHOD2(get_mail_buffer,const char*(void *buffer,int *mail_buff_size)); + MOCK_METHOD1(free_mail_buffer,void(void *mail_buffer)); + MOCK_METHOD3(append_to_buffer,void(void *buff,const unsigned char * chunk, size_t size)); }; class RadosStorageMetadataMock : public RadosStorageMetadataModule { public: - MOCK_METHOD1(set_io_ctx, void(librados::IoCtx *io_ctx)); - MOCK_METHOD1(load_metadata, int(RadosMail *mail)); - MOCK_METHOD2(set_metadata, int(RadosMail *mail, RadosMetadata &xattr)); - MOCK_METHOD3(set_metadata, int(RadosMail *mail, RadosMetadata &xattr, librados::ObjectWriteOperation *write_op)); - - MOCK_METHOD2(update_metadata, bool(const std::string &oid, std::list &to_update)); - // MOCK_METHOD2(save_metadata, void(librados::ObjectWriteOperation *write_op, RadosMailObject *mail)); - void save_metadata(librados::ObjectWriteOperation *write_op, RadosMail *mail) { - // delete write_op to avoid memory leak in case mocks are used - // if you need to change this, design your test so that storage is not a mock! - /*if (write_op != nullptr) { - delete write_op; - write_op = nullptr; - }*/ - } - MOCK_METHOD2(update_keyword_metadata, int(const std::string &oid, librmb::RadosMetadata *metadata)); + MOCK_METHOD1(set_io_ctx, void(storage_interface::RboxIoCtx *io_ctx_wrapper)); + MOCK_METHOD1(load_metadata, int(storage_interface::RadosMail *mail)); + MOCK_METHOD1(set_metadata, int(storage_interface::RadosMail *mail)); + MOCK_METHOD2(update_metadata, bool(const std::string &oid, std::list &to_update)); + MOCK_METHOD2(update_keyword_metadata, int(const std::string &oid, storage_interface::RadosMetadata *metadata)); MOCK_METHOD2(remove_keyword_metadata, int(const std::string &oid, std::string &key)); - MOCK_METHOD3(load_keyword_metadata, int(const std::string &oid, std::set &keys, - std::map *metadata)); }; class RadosMetadataStorageProducerMock : public RadosMetadataStorage { public: - MOCK_METHOD2(create_metadata_storage, - RadosStorageMetadataModule *(librados::IoCtx *io_ctx_, librmb::RadosDovecotCephCfg *cfg_)); MOCK_METHOD0(get_storage, RadosStorageMetadataModule *()); }; -using librmb::RadosDictionary; +using storage_interface::RadosDictionary; class RadosDictionaryMock : public RadosDictionary { public: @@ -131,14 +138,17 @@ class RadosDictionaryMock : public RadosDictionary { MOCK_METHOD0(get_private_oid, const std::string()); MOCK_METHOD0(get_oid, const std::string &()); MOCK_METHOD0(get_username, const std::string &()); - MOCK_METHOD0(get_io_ctx, librados::IoCtx &()); - MOCK_METHOD1(remove_completion, void(librados::AioCompletion *c)); - MOCK_METHOD1(push_back_completion, void(librados::AioCompletion *c)); + MOCK_METHOD0(get_io_ctx_wrapper,storage_interface::RboxIoCtx*()); + MOCK_METHOD0(get_poolname,const std::string& ()); + MOCK_METHOD0(get_shared_io_ctx_wrapper,storage_interface::RboxIoCtx*()); + MOCK_METHOD0(get_private_io_ctx_wrapper,storage_interface::RboxIoCtx*()); + MOCK_METHOD1(remove_completion, void(storage_interface::RboxIoCtx *remove_completion_wrapper)); + MOCK_METHOD1(push_back_completion, void(storage_interface::RboxIoCtx *push_back_completion_wrapper_)); MOCK_METHOD0(wait_for_completions, void()); MOCK_METHOD2(get, int(const std::string &key, std::string *value_r)); }; -using librmb::RadosCluster; +using storage_interface::RadosCluster; class RadosClusterMock : public RadosCluster { public: @@ -147,9 +157,9 @@ class RadosClusterMock : public RadosCluster { MOCK_METHOD0(deinit, void()); MOCK_METHOD1(pool_create, int(const std::string &pool)); - MOCK_METHOD2(recovery_index_io_ctx, int(const std::string &pool,librados::IoCtx *io_ctx)); + MOCK_METHOD2(recovery_index_io_ctx, int(const std::string &pool,storage_interface::RboxIoCtx *io_ctx_wrapper)); - MOCK_METHOD2(io_ctx_create, int(const std::string &pool, librados::IoCtx *io_ctx)); + MOCK_METHOD2(io_ctx_create,int (const std::string &pool,storage_interface::RboxIoCtx *io_ctx_wrapper)); MOCK_METHOD2(get_config_option, int(const char *option, std::string *value)); MOCK_METHOD0(is_connected, bool()); MOCK_METHOD2(set_config_option, void(const char *option, const char *value)); @@ -157,7 +167,7 @@ class RadosClusterMock : public RadosCluster { MOCK_METHOD1(list_pgs_osd_for_pool, std::map> (std::string &pool_name)); }; -using librmb::RadosDovecotCephCfg; +using storage_interface::RadosDovecotCephCfg; class RadosDovecotCephCfgMock : public RadosDovecotCephCfg { public: // dovecot configuration @@ -166,8 +176,8 @@ class RadosDovecotCephCfgMock : public RadosDovecotCephCfg { MOCK_METHOD0(get_rados_save_log_file, const std::string &()); // dovecot configuration - MOCK_METHOD1(is_mail_attribute, bool(enum librmb::rbox_metadata_key key)); - MOCK_METHOD1(is_updateable_attribute, bool(enum librmb::rbox_metadata_key key)); + MOCK_METHOD1(is_mail_attribute, bool(enum storage_interface::rbox_metadata_key key)); + MOCK_METHOD1(is_updateable_attribute, bool(enum storage_interface::rbox_metadata_key key)); MOCK_METHOD1(set_update_attributes, void(const std::string &update_attributes_)); MOCK_METHOD0(is_ceph_posix_bugfix_enabled, bool()); MOCK_METHOD0(is_ceph_aio_wait_for_safe_and_cb, bool()); @@ -178,7 +188,9 @@ class RadosDovecotCephCfgMock : public RadosDovecotCephCfg { MOCK_METHOD0(get_object_search_method,int()); MOCK_METHOD0(get_object_search_threads,int()); - + MOCK_METHOD0(get_log_level,const std::string &()); + MOCK_METHOD0(get_log_file_path,const std::string &()); + MOCK_METHOD1(update_mail_attributes, void(const char *value)); MOCK_METHOD1(update_updatable_attributes, void(const char *value)); MOCK_METHOD1(update_pool_name_metadata, void(const char *value)); @@ -201,7 +213,7 @@ class RadosDovecotCephCfgMock : public RadosDovecotCephCfg { MOCK_METHOD1(set_rbox_cfg_object_name, void(const std::string &value)); // ceph configuration - MOCK_METHOD1(set_io_ctx, void(librados::IoCtx *io_ctx)); + MOCK_METHOD1(set_io_ctx_wrapper,void(storage_interface::RboxIoCtx *io_ctx_wrapper)); MOCK_METHOD0(load_rados_config, int()); MOCK_METHOD0(save_default_rados_config, int()); @@ -216,13 +228,14 @@ class RadosDovecotCephCfgMock : public RadosDovecotCephCfg { MOCK_METHOD1(update_mail_attributes, void(const std::string &mail_attributes)); MOCK_METHOD1(update_updatable_attributes, void(const std::string &updateable_attributes)); - MOCK_METHOD2(save_object, int(const std::string &oid, librados::bufferlist &buffer)); - MOCK_METHOD2(read_object, int(const std::string &oid, librados::bufferlist *buffer)); + MOCK_METHOD2(save_object, int(const std::string &oid, std::istream &stream_buffer)); + MOCK_METHOD2(read_object, int(const std::string &oid, std::ostream &stream_buffer)); MOCK_METHOD1(set_io_ctx_namespace, void(const std::string &namespace_)); MOCK_METHOD0(get_metadata_storage_module, std::string &()); MOCK_METHOD0(get_metadata_storage_attribute, std::string &()); MOCK_METHOD0(is_rbox_check_empty_mailboxes, bool()); + MOCK_METHOD0(get_rados_ceph_cfg,storage_interface::RadosCephConfig *()); }; } // namespace librmbtest diff --git a/src/tests/rmb/test_rmb.cpp b/src/tests/rmb/test_rmb.cpp index df11a0d2..5c6c7010 100644 --- a/src/tests/rmb/test_rmb.cpp +++ b/src/tests/rmb/test_rmb.cpp @@ -12,16 +12,22 @@ #include "gtest/gtest.h" #include "gmock/gmock.h" #include -#include "../../librmb/rados-util.h" - +#include "../../storage-interface/rados-util.h" +#include"../../librmb/rbox-io-ctx-impl.h" #include "../../librmb/rados-cluster-impl.h" -#include "../../librmb/rados-mail.h" +#include "../../librmb/rados-mail-impl.h" #include "../../librmb/rados-storage-impl.h" -#include "../../librmb/tools/rmb/ls_cmd_parser.h" -#include "../../librmb/tools/rmb/mailbox_tools.h" -#include "../../librmb/tools/rmb/rmb-commands.h" +#include "../../storage-interface/tools/rmb/rmb-commands.h" +#include "../../storage-interface/tools/rmb/ls_cmd_parser.h" +#include "../../librmb/tools/rmb/rmb-commands-impl.h" +#include "../../librmb/tools/rmb/ls_cmd_parser_impl.h" +#include "../../librmb/tools/rmb/mailbox_tools_impl.h" +#include "../../librmb/tools/rmb/rados-mail-box-impl.h" #include "mock_test.h" -#include "../../librmb/rados-types.h" +#include "../../storage-interface/rados-types.h" +#include "../../storage-interface/rados-mail.h" +#include "../../librmb/rados-metadata-impl.h" + using ::testing::Return; using ::testing::_; using ::testing::ReturnRef; @@ -35,7 +41,7 @@ TEST(rmb, test_cmd_parser) { std::string key3 = "R"; // 917378644 std::string ls = "M=abc;U=1;R<2013-12-04 15:03"; - librmb::CmdLineParser parser(ls); + librmb::CmdLineParserImpl parser(ls); EXPECT_TRUE(parser.parse_ls_string()); EXPECT_EQ(3, (int)parser.get_predicates().size()); @@ -44,17 +50,17 @@ TEST(rmb, test_cmd_parser) { EXPECT_TRUE(parser.contains_key(key2)); EXPECT_TRUE(parser.contains_key(key3)); - librmb::rbox_metadata_key k = static_cast('M'); - EXPECT_EQ(k, librmb::RBOX_METADATA_MAILBOX_GUID); - librmb::Predicate *p = parser.get_predicate(key); + storage_interface::rbox_metadata_key k = static_cast('M'); + EXPECT_EQ(k, storage_interface::RBOX_METADATA_MAILBOX_GUID); + storage_interface::Predicate *p = parser.get_predicate(key); std::string value = "abc"; EXPECT_TRUE(p->eval(value)); - librmb::Predicate *p2 = parser.get_predicate(key2); + storage_interface::Predicate *p2 = parser.get_predicate(key2); value = "1"; EXPECT_TRUE(p2->eval(value)); - librmb::Predicate *p3 = parser.get_predicate(key3); + storage_interface::Predicate *p3 = parser.get_predicate(key3); value = "1503393219"; EXPECT_FALSE(p3->eval(value)); @@ -64,7 +70,7 @@ TEST(rmb, test_cmd_parser) { * Test date predicate */ TEST(rmb1, date_arg) { - librmb::Predicate *p = new librmb::Predicate(); + storage_interface::Predicate *p = new librmb::PredicateImpl(); std::string date = "2013-12-04 15:03:00"; time_t t = -1; @@ -81,7 +87,6 @@ TEST(rmb1, date_arg) { time_t t3 = 1086165760; p->convert_time_t_to_str(t3, &val); std::cout << val << std::endl; - delete p; } @@ -90,25 +95,25 @@ TEST(rmb1, date_arg) { */ TEST(rmb1, save_mail) { std::string mbox_guid = "abc"; - librmb::RadosMailBox mbox(mbox_guid, 1, mbox_guid); + librmb::RadosMailBoxImpl mbox(mbox_guid, 1, mbox_guid); std::string base_path = "test"; - librmb::MailboxTools tools(&mbox, base_path); + librmb::MailboxToolsImpl tools(&mbox, base_path); int init = tools.init_mailbox_dir(); EXPECT_EQ(0, init); - librmb::RadosMail mail; + librmb::RadosMailImpl mail; librados::bufferlist bl; std::string attr_val = "1"; bl.append(attr_val.c_str(), attr_val.length() + 1); - (*mail.get_metadata())["U"] = bl; + (*mail.get_metadata())["U"] = (void*)&bl; std::string mail_guid = "defg"; - librados::bufferlist buffer; - mail.set_mail_buffer(&buffer); - mail.get_mail_buffer()->append("hallo welt\nbababababa\n"); + void *buffer=(void*)new librados::bufferlist(); + mail.set_mail_buffer(buffer); + ((librados::bufferlist*)mail.get_mail_buffer())->append("hallo welt\nbababababa\n"); mail.set_oid(mail_guid); - mail.set_mail_size(mail.get_mail_buffer()->length() - 1); + mail.set_mail_size(((librados::bufferlist*)mail.get_mail_buffer())->length() - 1); int save = tools.save_mail(&mail); EXPECT_EQ(0, save); @@ -116,23 +121,24 @@ TEST(rmb1, save_mail) { int ret_rm_dir = tools.delete_mailbox_dir(); EXPECT_EQ(0, ret); EXPECT_EQ(0, ret_rm_dir); + delete buffer; } /** * Test mailbox Tools constructor */ TEST(rmb1, path_tests) { std::string mbox_guid = "abc"; - librmb::RadosMailBox mbox(mbox_guid, 1, mbox_guid); + librmb::RadosMailBoxImpl mbox(mbox_guid, 1, mbox_guid); std::string base_path = "test"; - librmb::MailboxTools tools(&mbox, base_path); + librmb::MailboxToolsImpl tools(&mbox, base_path); EXPECT_EQ("test/abc", tools.get_mailbox_path()); std::string test_path = "test/"; - librmb::MailboxTools tools2(&mbox, test_path); + librmb::MailboxToolsImpl tools2(&mbox, test_path); EXPECT_EQ("test/abc", tools2.get_mailbox_path()); std::string test_path2 = ""; - librmb::MailboxTools tools3(&mbox, test_path2); + librmb::MailboxToolsImpl tools3(&mbox, test_path2); EXPECT_EQ("abc", tools3.get_mailbox_path()); } /** @@ -145,14 +151,14 @@ TEST(rmb1, rmb_commands_no_objects_found) { librmbtest::RadosStorageMetadataMock ms_module_mock; std::map opts; - librmb::RmbCommands rmb_cmd(&storage_mock, &cluster_mock, &opts); - std::list mails; + librmb::RmbCommandsImpl rmb_cmd(&storage_mock, &cluster_mock, &opts); + std::list mails; std::string search_string = "uid"; - const librados::NObjectIterator iter = librados::NObjectIterator::__EndObjectIterator; - librados::IoCtx test_ioctx; + std::set mail_list; + librmb::RboxIoCtxImpl test_ioctx; - EXPECT_CALL(storage_mock, find_mails(nullptr)).WillRepeatedly(Return(iter)); - EXPECT_CALL(storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); + EXPECT_CALL(storage_mock, find_mails(nullptr)).WillRepeatedly(Return(mail_list)); + EXPECT_CALL(storage_mock,get_io_ctx_wrapper()).WillRepeatedly(Return(&test_ioctx)); EXPECT_CALL(storage_mock, stat_mail(_, _, _)).WillRepeatedly(Return(0)); int ret = rmb_cmd.load_objects(&ms_module_mock, mails, search_string); EXPECT_EQ(ret, 0); @@ -166,89 +172,89 @@ TEST(rmb1, rmb_command_filter_result) { librmbtest::RadosClusterMock cluster_mock; std::map opts; opts["ls"] = "-"; - librmb::CmdLineParser parser(opts["ls"]); - librmb::RmbCommands rmb_cmd(&storage_mock, &cluster_mock, &opts); - std::list mails; - librmb::RadosMail obj1; + librmb::CmdLineParserImpl parser(opts["ls"]); + librmb::RmbCommandsImpl rmb_cmd(&storage_mock, &cluster_mock, &opts); + std::list mails; + librmb::RadosMailImpl obj1; obj1.set_oid("oid_1"); obj1.set_mail_size(200); obj1.set_rados_save_date(time(NULL)); { std::string key = "M"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "I"; std::string val = "v0.1"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "G"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "R"; std::string val = "1234567"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "S"; std::string val = "1234561117"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "P"; std::string val = "1"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "O"; std::string val = "0"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "Z"; std::string val = "200"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "V"; std::string val = "250"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "U"; std::string val = "1"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "A"; std::string val = ""; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "F"; std::string val = "01"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "B"; std::string val = "DRAFTS"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } mails.push_back(&obj1); @@ -264,89 +270,89 @@ TEST(rmb1, rmb_command_filter_result2) { librmbtest::RadosClusterMock cluster_mock; std::map opts; opts["ls"] = "-"; - librmb::CmdLineParser parser(opts["ls"]); - librmb::RmbCommands rmb_cmd(&storage_mock, &cluster_mock, &opts); - std::list mails; - librmb::RadosMail obj1; + librmb::CmdLineParserImpl parser(opts["ls"]); + librmb::RmbCommandsImpl rmb_cmd(&storage_mock, &cluster_mock, &opts); + std::list mails; + librmb::RadosMailImpl obj1; obj1.set_oid("oid_1"); obj1.set_mail_size(200); obj1.set_rados_save_date(time(NULL)); { std::string key = "M"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "I"; std::string val = "v0.1"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "G"; std::string val = "8eed840764b05359f12718004d2485ee"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "R"; std::string val = "aafsadfasdfasdf"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "S"; std::string val = "adfhasdfhsfdkahsdf"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "P"; std::string val = "1"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "O"; std::string val = "0"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "Z"; std::string val = ""; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "V"; std::string val = "250"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "U"; std::string val = "1"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "A"; std::string val = ""; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "F"; std::string val = "01"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } { std::string key = "B"; std::string val = "DRAFTS"; - librmb::RadosMetadata m(key, val); + librmb::RadosMetadataImpl *m = new librmb::RadosMetadataImpl(key, val); obj1.add_metadata(m); } mails.push_back(&obj1); diff --git a/src/tests/storage-mock-rbox/test_repair_rbox.cpp b/src/tests/storage-mock-rbox/test_repair_rbox.cpp index 39ac0e01..4aa7b0ce 100644 --- a/src/tests/storage-mock-rbox/test_repair_rbox.cpp +++ b/src/tests/storage-mock-rbox/test_repair_rbox.cpp @@ -43,7 +43,7 @@ extern "C" { #include "rbox-save.h" #include "../mocks/mock_test.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" using ::testing::_; using ::testing::AtLeast; @@ -101,9 +101,10 @@ TEST_F(StorageTest, ParseLsByPoolOutput) { 10.1c 604 0 0 0 1114372 0 0 1329 active+clean 62m 3641'18755 3657:50579 [2,3,1]p2 [2,3,1]p2 2022-09-13T17:37:36.331891+0000 2022-09-13T17:37:36.331891+0000\n\ 10.1d 607 0 0 0 191148409 0 0 1271 active+clean 26h 3641'19526 3656:51525 [2,1,3]p2 [2,1,3]p2 2022-09-12T16:25:02.581960+0000 2022-09-12T16:25:02.581960+0000\n\ 10.1e 593 0 0 0 848897 0 0 1413 active+clean 20h 3641'19318 3656:51479 [1,3,4]p1 [1,3,4]p1 2022-09-12T22:13:27.455449+0000 2022-09-12T22:13:27.455449+0000\n\ - 10.1f 589 0 0 0 827645 0 0 1288 active+clean 13h 3641'20726 3656:53355 [2,4,1]p2 [2,4,1]p2 2022-09-13T05:09:00.823199+0000 2022-09-07T23:42:54.081441+0000\n"; - - std::vector list = librmb::RadosUtils::extractPgs(lsByPoolOutPut); + 10.1f 589 0 0 0 827645 0 0 1288 active+clean 13h 3641'20726 3656:53355 [2,4,1]p2 [2,4,1]p2 2022-09-13T05:09:00.823199+0000 2022-09-07T23:42:54.081441+0000\n\ + last row....."; + librmb::RadosUtilsImpl rados_utils; + std::vector list = rados_utils.extractPgs(lsByPoolOutPut); /*for (auto const &token: list) { std::cout << token << std::endl; @@ -149,11 +150,11 @@ TEST_F(StorageTest, scanForPg) { "mon command outbl: PG OBJECTS DEGRADED MISPLACED UNFOUND BYTES OMAP_BYTES* OMAP_KEYS* LOG STATE SINCE VERSION REPORTED UP ACTING SCRUB_STAMP DEEP_SCRUB_STAMP\n"; const std::string row ="10.0 568 0 0 0 841115 0 0 1165 active+clean 26h 3641'18906 3656:51938 [3,1,4]p3 [3,1,4]p3 2022-09-12T15:41:31.269234+0000 2022-09-12T15:41:31.269234+0000\n"; - - std::vector list = librmb::RadosUtils::split(row,' '); + librmb::RadosUtilsImpl rados_utils; + std::vector list = rados_utils.split(row,' '); EXPECT_EQ(17,list.size()); - std::vector list2= librmb::RadosUtils::split(header,' '); + std::vector list2= rados_utils.split(header,' '); EXPECT_EQ(20,list2.size()); } @@ -193,8 +194,8 @@ TEST_F(StorageTest, extractPrimaryOsd) { 10.1d 607 0 0 0 191148409 0 0 1271 active+clean 26h 3641'19526 3656:51525 [2,1,3]p2 [2,1,3]p2 2022-09-12T16:25:02.581960+0000 2022-09-12T16:25:02.581960+0000\n\ 10.1e 593 0 0 0 848897 0 0 1413 active+clean 20h 3641'19318 3656:51479 [1,3,4]p1 [1,3,4]p1 2022-09-12T22:13:27.455449+0000 2022-09-12T22:13:27.455449+0000\n\ 10.1f 589 0 0 0 827645 0 0 1288 active+clean 13h 3641'20726 3656:53355 [2,4,1]p2 [2,4,1]p2 2022-09-13T05:09:00.823199+0000 2022-09-07T23:42:54.081441+0000\n"; - - std::map> list = librmb::RadosUtils::extractPgAndPrimaryOsd(lsByPoolOutPut); + librmb::RadosUtilsImpl rados_utils; + std::map> list = rados_utils.extractPgAndPrimaryOsd(lsByPoolOutPut); for (const auto& x : list) { @@ -209,17 +210,17 @@ TEST_F(StorageTest, extractPrimaryOsd) { } TEST_F(StorageTest,create_read_index) { - + librmb::RadosUtilsImpl rados_utils; std::set test_mails; test_mails.insert("1"); test_mails.insert("2"); test_mails.insert("3"); test_mails.insert("4"); test_mails.insert("5"); - std::string test_string = librmb::RadosUtils::convert_to_ceph_index(test_mails); + std::string test_string = rados_utils.convert_to_ceph_index(test_mails); EXPECT_EQ("1,2,3,4,5,",test_string); - std::set test_mails_read = librmb::RadosUtils::ceph_index_to_set(test_string); + std::set test_mails_read = rados_utils.ceph_index_to_set(test_string); EXPECT_EQ(5,test_mails_read.size()); auto my_vect = std::vector(test_mails_read.begin(), test_mails_read.end()); // O[n] @@ -228,16 +229,12 @@ TEST_F(StorageTest,create_read_index) { EXPECT_EQ("3",my_vect[2]); EXPECT_EQ("4",my_vect[3]); EXPECT_EQ("5",my_vect[4]); - - - std::string test_string1 = librmb::RadosUtils::convert_to_ceph_index("abd"); - EXPECT_EQ("abd,",test_string1); + std::string test_string1 = rados_utils.convert_to_ceph_index("abd"); + EXPECT_EQ("abd,",test_string1); } - - int main(int argc, char **argv) { ::testing::InitGoogleMock(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp b/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp index be3176cb..9a8203e6 100644 --- a/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp +++ b/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp @@ -12,7 +12,7 @@ #include "../storage-mock-rbox/TestCase.h" #include "gtest/gtest.h" #include "gmock/gmock.h" - +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" // turn off warnings for Dovecot :-( #pragma GCC diagnostic ignored "-Wundef" // turn off warnings for Dovecot :-( @@ -35,22 +35,28 @@ extern "C" { #include "libdict-rados-plugin.h" } - +#include #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" #include "rbox-storage.hpp" #include "rbox-save.h" - +#include "rbox-mail.h" #include "../mocks/mock_test.h" #include "rados-dovecot-ceph-cfg-impl.h" #include "../../storage-rbox/istream-bufferlist.h" #include "../../storage-rbox/ostream-bufferlist.h" +#include "../../storage-engine/storage-backend-factory.h" +#include "../../storage-interface/rados-mail.h" using ::testing::_; using ::testing::AtLeast; using ::testing::Matcher; using ::testing::Return; using ::testing::ReturnRef; +using ::testing::DoAll; +using ::testing::Assign; +using ::testing::Lt; +using ::testing::NiceMock; #pragma GCC diagnostic pop #if DOVECOT_PREREQ(2, 3) @@ -65,742 +71,334 @@ using ::testing::ReturnRef; TEST_F(StorageTest, init) {} -/** - * make sure is deprecated uid is always false initialy - **/ -TEST_F(StorageTest, create_rados_mail) { +// test storage class, cluster not initialized +TEST_F(StorageTest, connect_failed_test) { - for(int i = 0 ; i < 10000 ;i++) { - librmb::RadosMail *mail = new librmb::RadosMail(); - ASSERT_EQ(false, mail->is_deprecated_uid()); - delete mail; - } -} -/** - * Error test: - * - open_connection to rados will fail with -1 . - */ -TEST_F(StorageTest, mail_save_to_inbox_storage_mock_no_rados_available) { - struct mail_namespace *ns = mail_namespace_find_inbox(s_test_mail_user->namespaces); - ASSERT_NE(ns, nullptr); - struct mailbox *box = mailbox_alloc(ns->list, "INBOX", (mailbox_flags)0); - ASSERT_NE(box, nullptr); - ASSERT_GE(mailbox_open(box), 0); - - const char *message = - "From: user@domain.org\n" - "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" - "Mime-Version: 1.0\n" - "Content-Type: text/plain; charset=us-ascii\n" - "\n" - "body\n"; - - struct istream *input = i_stream_create_from_data(message, strlen(message)); -#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); -#else - char reason[256]; - memset(reason, '\0', sizeof(reason)); - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); -#endif + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + /* configure behavior of the mock */ + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(-1)); - struct mail_save_context *save_ctx = mailbox_save_alloc(trans); - - // set the Mock storage - struct rbox_storage *storage = (struct rbox_storage *)box->storage; - delete storage->s; - - librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - // first call to open_connection will fail! - EXPECT_CALL(*storage_mock, open_connection("mail_storage", "ceph", "client.admin")) - .Times(AtLeast(1)) - .WillOnce(Return(-1)); - - librmb::RadosMail *test_obj = new librmb::RadosMail(); - - EXPECT_CALL(*storage_mock, alloc_rados_mail()).WillOnce(Return(test_obj)); - // storage->ns_mgr->set_storage(storage_mock); - storage->s = storage_mock; - - bool save_failed = FALSE; - - if (mailbox_save_begin(&save_ctx, input) < 0) { - i_error("Saving failed: %s", mailbox_get_last_internal_error(box, NULL)); - mailbox_save_finish(&save_ctx); - mailbox_transaction_rollback(&trans); - SUCCEED() << "saving failed: " << mailbox_get_last_internal_error(box, NULL); - } else { - ssize_t ret; - do { - if (mailbox_save_continue(save_ctx) < 0) { - save_failed = TRUE; - ret = -1; - SUCCEED() << "mailbox_save_continue() failed"; - break; - } - } while ((ret = i_stream_read(input)) > 0); - EXPECT_EQ(ret, -1); - - if (input->stream_errno != 0) { - FAIL() << "read(msg input) failed: " << i_stream_get_error(input); - } else if (save_failed) { - SUCCEED() << "Saving failed: " << mailbox_get_last_internal_error(box, NULL); - } else if (mailbox_save_finish(&save_ctx) < 0) { - FAIL() << "Saving should fail, due to connection to rados is not available."; - } else if (mailbox_transaction_commit(&trans) < 0) { - FAIL() << "Save transaction commit failed: " << mailbox_get_last_internal_error(box, NULL); - } else { - ret = 0; - } - - EXPECT_NE(save_ctx, nullptr); - if (save_ctx != nullptr) - mailbox_save_cancel(&save_ctx); - - EXPECT_NE(trans, nullptr); - if (trans != nullptr) - mailbox_transaction_rollback(&trans); - - EXPECT_FALSE(input->eof); - EXPECT_GE(ret, -1); - } - i_stream_unref(&input); - mailbox_free(&box); + librmb::RadosStorageImpl under_test(cluster_mock); - delete test_obj; -} + std::string pool_name("test"); - -/** - * Error test: - * - * - save mail to rados fails. - * - */ -TEST_F(StorageTest, save_mail_fail_test) { - struct mail_namespace *ns = mail_namespace_find_inbox(s_test_mail_user->namespaces); - ASSERT_NE(ns, nullptr); - struct mailbox *box = mailbox_alloc(ns->list, "INBOX", (mailbox_flags)0); - ASSERT_NE(box, nullptr); - ASSERT_GE(mailbox_open(box), 0); - - const char *message = - "From: user@domain.org\n" - "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" - "Mime-Version: 1.0\n" - "Content-Type: text/plain; charset=us-ascii\n" - "\n" - "body\n"; - - struct istream *input = i_stream_create_from_data(message, strlen(message)); - -#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); -#else - char reason[256]; - memset(reason, '\0', sizeof(reason)); - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); -#endif - struct mail_save_context *save_ctx = mailbox_save_alloc(trans); - - // set the Mock storage - struct rbox_storage *storage = (struct rbox_storage *)box->storage; - delete storage->s; - librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - librados::IoCtx test_ioctx; - EXPECT_CALL(*storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); - - EXPECT_CALL(*storage_mock, open_connection("mail_storage", "ceph", "client.admin")) - .Times(AtLeast(1)) - .WillRepeatedly(Return(0)); - - EXPECT_CALL(*storage_mock, get_max_object_size()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(65000)); - - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(65000)); - - - EXPECT_CALL(*storage_mock, aio_operate(_,_,_,_)).Times(AtLeast(1)).WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, wait_for_write_operations_complete(_,_)).WillRepeatedly(Return(true));//failed = false - - librmb::RadosMail *test_obj = new librmb::RadosMail(); - test_obj->set_mail_buffer(nullptr); - librmb::RadosMail *test_obj2 = new librmb::RadosMail(); - test_obj2->set_mail_buffer(nullptr); - EXPECT_CALL(*storage_mock, alloc_rados_mail()).Times(2).WillOnce(Return(test_obj)).WillOnce(Return(test_obj2)); - EXPECT_CALL(*storage_mock, free_rados_mail(_)).Times(2); - - delete storage->ms; - librmbtest::RadosMetadataStorageProducerMock *ms_p_mock = new librmbtest::RadosMetadataStorageProducerMock(); - storage->ms = ms_p_mock; - - librmbtest::RadosStorageMetadataMock ms_mock; - EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _)).WillRepeatedly(Return(0)); - - delete storage->config; - librmbtest::RadosDovecotCephCfgMock *cfg_mock = new librmbtest::RadosDovecotCephCfgMock(); - EXPECT_CALL(*cfg_mock, is_config_valid()).WillRepeatedly(Return(true)); - std::string user = "client.admin"; - std::string cluster = "ceph"; - std::string pool = "mail_storage"; - std::string suffix = "_u"; - EXPECT_CALL(*cfg_mock, get_index_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_object_search_method()).WillRepeatedly(Return(0)); - EXPECT_CALL(*cfg_mock, get_rados_username()).WillRepeatedly(ReturnRef(user)); - EXPECT_CALL(*cfg_mock, get_rados_cluster_name()).WillRepeatedly(ReturnRef(cluster)); - EXPECT_CALL(*cfg_mock, get_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_user_suffix()).WillRepeatedly(ReturnRef(suffix)); - EXPECT_CALL(*cfg_mock, get_write_method()).WillRepeatedly(Return(1)); - EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); - - storage->ns_mgr->set_config(cfg_mock); - - storage->config = cfg_mock; - storage->s = storage_mock; - - bool save_failed = FALSE; - - if (mailbox_save_begin(&save_ctx, input) < 0) { - i_error("Saving failed: %s", mailbox_get_last_internal_error(box, NULL)); - mailbox_transaction_rollback(&trans); - FAIL() << "saving failed: " << mailbox_get_last_internal_error(box, NULL); - } else { - ssize_t ret; - do { - if (mailbox_save_continue(save_ctx) < 0) { - save_failed = TRUE; - ret = -1; - FAIL() << "mailbox_save_continue() failed"; - break; - } - } while ((ret = i_stream_read(input)) > 0); - EXPECT_EQ(ret, -1); - - if (input->stream_errno != 0) { - FAIL() << "read(msg input) failed: " << i_stream_get_error(input); - } else if (save_failed) { - FAIL() << "Saving failed: " << mailbox_get_last_internal_error(box, NULL); - } else if (mailbox_save_finish(&save_ctx) < 0) { - SUCCEED() << "Saving should fail, due to connection to rados is not available."; - } else if (mailbox_transaction_commit(&trans) < 0) { - FAIL() << "Save transaction commit failed: " << mailbox_get_last_internal_error(box, NULL); - } else { - ret = 0; - } - - EXPECT_EQ(save_ctx, nullptr); - if (save_ctx != nullptr) - mailbox_save_cancel(&save_ctx); - - EXPECT_NE(trans, nullptr); - if (trans != nullptr) - mailbox_transaction_rollback(&trans); - - EXPECT_TRUE(input->eof); - EXPECT_GE(ret, -1); - } - i_stream_unref(&input); - mailbox_free(&box); - - if (test_obj->get_mail_buffer() != nullptr) { - delete test_obj->get_mail_buffer(); - } - delete test_obj; - if (test_obj2->get_mail_buffer() != nullptr) { - delete test_obj2->get_mail_buffer(); - } - delete test_obj2; -} -/** - * Error test: - * - * - save mail to rados fails in mailbox_transaction_commit - * - */ -TEST_F(StorageTest, write_op_fails) { - struct mail_namespace *ns = mail_namespace_find_inbox(s_test_mail_user->namespaces); - ASSERT_NE(ns, nullptr); - struct mailbox *box = mailbox_alloc(ns->list, "INBOX", (mailbox_flags)0); - ASSERT_NE(box, nullptr); - i_debug("preparing to open"); - ASSERT_GE(mailbox_open(box), 0); - i_debug("mailbox open"); - const char *message = - "From: user@domain.org\n" - "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" - "Mime-Version: 1.0\n" - "Content-Type: text/plain; charset=us-ascii\n" - "\n" - "body\n"; - - struct istream *input = i_stream_create_from_data(message, strlen(message)); - -#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); -#else - char reason[256]; - memset(reason, '\0', sizeof(reason)); - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); -#endif - struct mail_save_context *save_ctx = mailbox_save_alloc(trans); - // set the Mock storage - struct rbox_storage *storage = (struct rbox_storage *)box->storage; - delete storage->s; - - librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - librados::IoCtx test_ioctx; - EXPECT_CALL(*storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); - - EXPECT_CALL(*storage_mock, open_connection("mail_storage", "ceph", "client.admin")) - .Times(AtLeast(1)) - .WillRepeatedly(Return(0)); - - - EXPECT_CALL(*storage_mock, get_max_object_size()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(65000)); - - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(65000)); - - - EXPECT_CALL(*storage_mock, aio_operate(_,_,_,_)).Times(AtLeast(1)).WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, wait_for_write_operations_complete(_,_)).WillRepeatedly(Return(false));//failed = false - - - EXPECT_CALL(*storage_mock, read_mail(_, _)).WillRepeatedly(Return(-2)); - - librmb::RadosMail *test_obj = new librmb::RadosMail(); - test_obj->set_mail_buffer(nullptr); - librmb::RadosMail *test_obj2 = new librmb::RadosMail(); - test_obj2->set_mail_buffer(nullptr); - EXPECT_CALL(*storage_mock, alloc_rados_mail()).Times(2).WillOnce(Return(test_obj)).WillOnce(Return(test_obj2)); - EXPECT_CALL(*storage_mock, set_ceph_wait_method(_)).Times(1); - - EXPECT_CALL(*storage_mock, free_rados_mail(_)).Times(2); - delete storage->config; - librmbtest::RadosDovecotCephCfgMock *cfg_mock = new librmbtest::RadosDovecotCephCfgMock(); - EXPECT_CALL(*cfg_mock, is_config_valid()).WillRepeatedly(Return(true)); - EXPECT_CALL(*cfg_mock, is_write_chunks()).WillRepeatedly(Return(false)); - EXPECT_CALL(*cfg_mock, is_ceph_posix_bugfix_enabled()).WillRepeatedly(Return(false)); - EXPECT_CALL(*cfg_mock, is_ceph_aio_wait_for_safe_and_cb()).WillOnce(Return(false)); - EXPECT_CALL(*cfg_mock, load_rados_config()).WillOnce(Return(0)); - EXPECT_CALL(*cfg_mock, is_mail_attribute(_)).WillRepeatedly(Return(true)); - EXPECT_CALL(*cfg_mock, is_user_mapping()).WillRepeatedly(Return(false)); - - std::string user = "client.admin"; - std::string cluster = "ceph"; - std::string pool = "mail_storage"; - std::string suffix = "_u"; - EXPECT_CALL(*cfg_mock, get_index_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_object_search_method()).WillRepeatedly(Return(0)); - EXPECT_CALL(*cfg_mock, get_rados_username()).WillRepeatedly(ReturnRef(user)); - EXPECT_CALL(*cfg_mock, get_rados_cluster_name()).WillRepeatedly(ReturnRef(cluster)); - EXPECT_CALL(*cfg_mock, get_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_user_suffix()).WillRepeatedly(ReturnRef(suffix)); - EXPECT_CALL(*cfg_mock, get_write_method()).WillRepeatedly(Return(1)); - EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); + int open_connection = under_test.open_connection(pool_name); - storage->ns_mgr->set_config(cfg_mock); - - storage->config = cfg_mock; - storage->s = storage_mock; - - delete storage->ms; - librmbtest::RadosMetadataStorageProducerMock *ms_p_mock = new librmbtest::RadosMetadataStorageProducerMock(); - storage->ms = ms_p_mock; - - librmbtest::RadosStorageMetadataMock ms_mock; - EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _)).WillRepeatedly(Return(0)); - EXPECT_CALL(*ms_p_mock, create_metadata_storage(_,_)).Times(1); - - bool save_failed = FALSE; - - if (mailbox_save_begin(&save_ctx, input) < 0) { - i_error("Saving failed: %s", mailbox_get_last_internal_error(box, NULL)); - mailbox_transaction_rollback(&trans); - FAIL() << "saving failed: " << mailbox_get_last_internal_error(box, NULL); - } else { - ssize_t ret; - do { - if (mailbox_save_continue(save_ctx) < 0) { - save_failed = TRUE; - ret = -1; - FAIL() << "mailbox_save_continue() failed"; - break; - } - } while ((ret = i_stream_read(input)) > 0); - EXPECT_EQ(ret, -1); - - if (input->stream_errno != 0) { - FAIL() << "read(msg input) failed: " << i_stream_get_error(input); - } else if (save_failed) { - FAIL() << "Saving failed: " << mailbox_get_last_internal_error(box, NULL); - } else if (mailbox_save_finish(&save_ctx) < 0) { - SUCCEED() << "Saving should fail"; - mailbox_transaction_rollback(&trans); - - } else if (mailbox_transaction_commit(&trans) < 0) { - i_debug("failed at correct place"); - i_info("mailbox_transaction_commit <0 "); - FAIL() << "transcation should not succeed"; - } else { - ret = 0; - i_info("ELSE RET = 0"); - } - EXPECT_EQ(save_ctx, nullptr); - if (save_ctx != nullptr){ - i_info("save_ctx != nullptr"); - mailbox_save_cancel(&save_ctx); - } - - EXPECT_EQ(trans, nullptr); - if (trans != nullptr){ - i_info("transcation rollback"); - mailbox_transaction_rollback(&trans); - } - - - EXPECT_TRUE(input->eof); - i_info("input eof %ld",ret); - EXPECT_GE(ret, -1); - - } - i_info("unref"); - i_stream_unref(&input); - i_info("pre mailbox freed"); - mailbox_free(&box); - i_info("mailbox freed"); - if (test_obj->get_mail_buffer() != nullptr) { - delete test_obj->get_mail_buffer(); - } - delete test_obj; - if (test_obj2->get_mail_buffer() != nullptr) { - delete test_obj2->get_mail_buffer(); - } - delete test_obj2; - i_info("done"); - SUCCEED() << "should be ok here"; + EXPECT_EQ(-1, open_connection); + delete cluster_mock; + cluster_mock=nullptr; } -/** - * Error test: - * - * - eval copy from input to output stream - */ -TEST_F(StorageTest, copy_input_to_output_stream) { - librados::bufferlist *buffer = new librados::bufferlist(); - // librados::bufferlist buffer_out; - librmb::RadosMail mail; - buffer->append("\r\t\0\nJAN"); - unsigned long physical_size = buffer->length(); - struct istream *input; // = *stream_r; - struct ostream *output; - - librados::bufferlist buffer2; - mail.set_mail_buffer(&buffer2); - output = o_stream_create_bufferlist(&mail, nullptr, false); - input = i_stream_create_from_bufferlist(buffer, physical_size); - - do { - if (o_stream_send_istream(output, input) < 0) { - EXPECT_EQ(1, -1); - } - - } while (i_stream_read(input) > 0); - - EXPECT_EQ(buffer->to_str(), mail.get_mail_buffer()->to_str()); - o_stream_unref(&output); - i_stream_unref(&input); -} +// test storage class, cluster not initialized +TEST_F(StorageTest, connect_io_ctx_cant_be_created_test) { -/** - * Error test: - * - * - copy mail fails due to storage.copy call returns -1 - */ -TEST_F(StorageTest, mock_copy_failed_due_to_rados_err) { - struct mailbox_transaction_context *desttrans; - struct mail_save_context *save_ctx; - struct mail *mail; - struct mail_search_context *search_ctx; - struct mail_search_args *search_args; - struct mail_search_arg *sarg; - - const char *message = - "From: user@domain.org\n" - "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" - "Mime-Version: 1.0\n" - "Content-Type: text/plain; charset=us-ascii\n" - "\n" - "body\n"; - - const char *mailbox = "INBOX"; - - librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - - EXPECT_CALL(*storage_mock, get_max_object_size()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(65000)); - - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(65000)); - EXPECT_CALL(*storage_mock, set_ceph_wait_method(_)).Times(1); - - EXPECT_CALL(*storage_mock, free_rados_mail(_)).Times(2); - - librados::IoCtx test_ioctx; - EXPECT_CALL(*storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); - + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + /* configure behavior of the mock */ + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(-1)); + librmb::RadosStorageImpl under_test(cluster_mock); - librmb::RadosMail *test_obj_save = new librmb::RadosMail(); - librmb::RadosMail *test_obj_save2 = new librmb::RadosMail(); - test_obj_save->set_mail_buffer(nullptr); - test_obj_save2->set_mail_buffer(nullptr); + std::string pool_name("test"); - EXPECT_CALL(*storage_mock, alloc_rados_mail()) - .Times(2) - .WillOnce(Return(test_obj_save)) - .WillOnce(Return(test_obj_save2)); - - EXPECT_CALL(*storage_mock, aio_operate(_,_,_,_)).Times(AtLeast(1)).WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, wait_for_write_operations_complete(_,_)).WillRepeatedly(Return(false));//failed = false - - // testdata - testutils::ItUtils::add_mail(message, mailbox, StorageTest::s_test_mail_user->namespaces, storage_mock); - - if (test_obj_save->get_mail_buffer() != nullptr) { - delete test_obj_save->get_mail_buffer(); - } - delete test_obj_save; - if (test_obj_save2->get_mail_buffer() != nullptr) { - delete test_obj_save2->get_mail_buffer(); - } - delete test_obj_save2; - - search_args = mail_search_build_init(); - sarg = mail_search_build_add(search_args, SEARCH_ALL); - ASSERT_NE(sarg, nullptr); - - struct mail_namespace *ns = mail_namespace_find_inbox(s_test_mail_user->namespaces); - ASSERT_NE(ns, nullptr); - - struct mailbox *box = mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_SAVEONLY); - - // set the Mock storage - struct rbox_storage *storage = (struct rbox_storage *)box->storage; - delete storage->s; - - librmbtest::RadosStorageMock *storage_mock_copy = new librmbtest::RadosStorageMock(); - librmb::RadosMail *test_object = new librmb::RadosMail(); - librmb::RadosMail *test_object2 = new librmb::RadosMail(); - test_object->set_mail_buffer(nullptr); - test_object2->set_mail_buffer(nullptr); - - librmb::RadosMetadata recv_date = librmb::RadosMetadata(librmb::RBOX_METADATA_RECEIVED_TIME, time(NULL)); - test_object->add_metadata(recv_date); - librmb::RadosMetadata guid = librmb::RadosMetadata(librmb::RBOX_METADATA_GUID, "67ffff24efc0e559194f00009c60b9f7"); - test_object->add_metadata(guid); - - EXPECT_CALL(*storage_mock_copy, alloc_rados_mail()) - .Times(2) - .WillOnce(Return(test_object)) - .WillOnce(Return(test_object2)); - - - EXPECT_CALL(*storage_mock_copy, copy(_, _, _, _, _)).WillRepeatedly(Return(-1)); - EXPECT_CALL(*storage_mock_copy, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); - - storage->s = storage_mock_copy; - delete storage->config; - librmbtest::RadosDovecotCephCfgMock *cfg_mock = new librmbtest::RadosDovecotCephCfgMock(); - EXPECT_CALL(*cfg_mock, is_config_valid()).WillRepeatedly(Return(true)); - std::string user = "client.admin"; - std::string cluster = "ceph"; - std::string pool = "mail_storage"; - std::string suffix = "_u"; - - delete storage->ms; - librmbtest::RadosMetadataStorageProducerMock *ms_p_mock = new librmbtest::RadosMetadataStorageProducerMock(); - storage->ms = ms_p_mock; - - librmbtest::RadosStorageMetadataMock ms_mock; - EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _)).WillRepeatedly(Return(0)); - EXPECT_CALL(*cfg_mock, get_index_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_object_search_method()).WillRepeatedly(Return(0)); - EXPECT_CALL(*cfg_mock, is_config_valid()).WillRepeatedly(Return(true)); - EXPECT_CALL(*cfg_mock, is_write_chunks()).WillRepeatedly(Return(false)); - EXPECT_CALL(*cfg_mock, is_ceph_posix_bugfix_enabled()).WillRepeatedly(Return(false)); - EXPECT_CALL(*cfg_mock, is_ceph_aio_wait_for_safe_and_cb()).WillRepeatedly(Return(false)); - EXPECT_CALL(*cfg_mock, load_rados_config()).WillRepeatedly(Return(0)); - EXPECT_CALL(*cfg_mock, is_mail_attribute(_)).WillRepeatedly(Return(true)); - EXPECT_CALL(*cfg_mock, is_user_mapping()).WillRepeatedly(Return(false)); - - EXPECT_CALL(*cfg_mock, get_rados_username()).WillRepeatedly(ReturnRef(user)); - EXPECT_CALL(*cfg_mock, get_rados_cluster_name()).WillRepeatedly(ReturnRef(cluster)); - EXPECT_CALL(*cfg_mock, get_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_user_suffix()).WillRepeatedly(ReturnRef(suffix)); - EXPECT_CALL(*cfg_mock, get_write_method()).WillRepeatedly(Return(1)); - EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); + int open_connection = under_test.open_connection(pool_name); - storage->ns_mgr->set_config(cfg_mock); + EXPECT_EQ(-1, open_connection); - storage->config = cfg_mock; - - if (mailbox_open(box) < 0) { - i_error("######################### Opening mailbox %s failed: %s", mailbox, mailbox_get_last_internal_error(box, NULL)); - FAIL() << " Forcing a resync on mailbox INBOX Failed"; - } + delete cluster_mock; + cluster_mock=nullptr; +} -#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - desttrans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); -#else - char reason[256]; - memset(reason, '\0', sizeof(reason)); - desttrans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); -#endif +// test storage class, cluster not initialized +TEST_F(StorageTest, connect_io_ctx_recovery_index_io_ctx_failed) { - search_ctx = mailbox_search_init(desttrans, search_args, NULL, static_cast(0), NULL); - mail_search_args_unref(&search_args); - int ret2 = 0; - while (mailbox_search_next(search_ctx, &mail)) { - save_ctx = mailbox_save_alloc(desttrans); // src save context - mailbox_save_copy_flags(save_ctx, mail); + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + /* configure behavior of the mock */ + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(-1)); + librmb::RadosStorageImpl under_test(cluster_mock); - ret2 = mailbox_copy(&save_ctx, mail); - i_info("mailbox copy done : %d",ret2); - - break; // only move one mail. - } - i_info("before deinig"); - if (mailbox_search_deinit(&search_ctx) < 0) { - i_debug("#########################search deint failed!"); - FAIL() << "deinit failed"; - } - i_info("before commit"); - if (mailbox_transaction_commit(&desttrans) < 0) { - i_debug("#########################transaction commit <0"); - FAIL() << "tnx commit failed"; - } - i_info("closing mailbox"); - // mail should be marked as expunged!!! - mailbox_free(&box); + std::string pool_name("test"); + int open_connection = under_test.open_connection(pool_name); + EXPECT_EQ(-1, open_connection); - if (test_object->get_mail_buffer() != nullptr) { - delete test_object->get_mail_buffer(); - } - delete test_object; - if (test_object2->get_mail_buffer() != nullptr) { - delete test_object2->get_mail_buffer(); - } - delete test_object2; - i_info("#########################done."); + delete cluster_mock; + cluster_mock=nullptr; +} +TEST_F(StorageTest,false_io_ctx_created){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl under_test(cluster_mock); + + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(-1)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(0); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(0); + + std::string pool_name("test"); + /*create_connection is invoked by open_connection method*/ + int open_connection = under_test.open_connection(pool_name); + EXPECT_EQ(-1, open_connection); + delete cluster_mock; + cluster_mock=nullptr; } +TEST_F(StorageTest,first_cluster_connectio){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl under_test(cluster_mock); -/** - * Error test: - * - * - save mail failed due to quota / cancel save - * - */ -TEST_F(StorageTest, save_mail_cancel) { - struct mail_namespace *ns = mail_namespace_find_inbox(s_test_mail_user->namespaces); - ASSERT_NE(ns, nullptr); - struct mailbox *box = mailbox_alloc(ns->list, "INBOX", (mailbox_flags)0); - ASSERT_NE(box, nullptr); - ASSERT_GE(mailbox_open(box), 0); - - const char *message = - "From: user@domain.org\n" - "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" - "Mime-Version: 1.0\n" - "Content-Type: text/plain; charset=us-ascii\n" - "\n" - "body\n"; - -#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); -#else - char reason[256]; - memset(reason, '\0', sizeof(reason)); - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); -#endif - struct mail_save_context *save_ctx = mailbox_save_alloc(trans); - - // set the Mock storage - struct rbox_storage *storage = (struct rbox_storage *)box->storage; - delete storage->s; - librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - librados::IoCtx test_ioctx; - EXPECT_CALL(*storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); - - librmb::RadosMail *test_obj = new librmb::RadosMail(); - test_obj->set_mail_buffer(nullptr); - librmb::RadosMail *test_obj2 = new librmb::RadosMail(); - test_obj2->set_mail_buffer(nullptr); - EXPECT_CALL(*storage_mock, alloc_rados_mail()).Times(0); - EXPECT_CALL(*storage_mock, free_rados_mail(_)).Times(0); - - delete storage->ms; - librmbtest::RadosMetadataStorageProducerMock *ms_p_mock = new librmbtest::RadosMetadataStorageProducerMock(); - storage->ms = ms_p_mock; - - librmbtest::RadosStorageMetadataMock ms_mock; - EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _)).WillRepeatedly(Return(0)); - - delete storage->config; - librmbtest::RadosDovecotCephCfgMock *cfg_mock = new librmbtest::RadosDovecotCephCfgMock(); - EXPECT_CALL(*cfg_mock, is_config_valid()).WillRepeatedly(Return(true)); - std::string user = "client.admin"; - std::string cluster = "ceph"; - std::string pool = "mail_storage"; - std::string suffix = "_u"; - EXPECT_CALL(*cfg_mock, get_index_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_object_search_method()).WillRepeatedly(Return(0)); - EXPECT_CALL(*cfg_mock, get_rados_username()).WillRepeatedly(ReturnRef(user)); - EXPECT_CALL(*cfg_mock, get_rados_cluster_name()).WillRepeatedly(ReturnRef(cluster)); - EXPECT_CALL(*cfg_mock, get_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_user_suffix()).WillRepeatedly(ReturnRef(suffix)); - EXPECT_CALL(*cfg_mock, get_write_method()).WillRepeatedly(Return(1)); - EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); - - storage->ns_mgr->set_config(cfg_mock); + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + + std::string pool_name("test"); + /*create_connection is invoked by open_connection method*/ + int open_connection = under_test.open_connection(pool_name); + EXPECT_EQ(0, open_connection); - storage->config = cfg_mock; - storage->s = storage_mock; + EXPECT_CALL(*cluster_mock, is_connected()).Times(1).WillOnce(Return(false)); + + std::string buffer_text="simple_test_one_chunck"; + storage_interface::RadosMail *rados_mail= + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + void *buffer=(void*)new librados::bufferlist(); + rados_mail->set_mail_buffer(buffer); + ((librados::bufferlist*)rados_mail->get_mail_buffer())->append(buffer_text); + int buffer_length = ((librados::bufferlist*)rados_mail->get_mail_buffer())->length(); + rados_mail->set_mail_size(buffer_length); + rados_mail->set_oid("test_mail_id"); + + /*div==1*/ + bool ret_storage = under_test.save_mail(rados_mail); + EXPECT_EQ(false,ret_storage); + delete cluster_mock; + cluster_mock=nullptr; + delete buffer; + delete rados_mail; + rados_mail=nullptr; +} +TEST_F(StorageTest,second_cluster_connectio){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl under_test(cluster_mock); + + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + + librmbtest::RboxIoCtxMock *io_ctx_mock=new librmbtest::RboxIoCtxMock(); + under_test.set_io_ctx_wrapper(io_ctx_mock); + librados::IoCtx io_ctx; + librmb::RadosMetadataStorageDefault rados_metadata_storage (io_ctx_mock); + rados_metadata_storage.set_io_ctx(io_ctx_mock); + EXPECT_CALL(*io_ctx_mock,append(_,_,_)).Times(0); + EXPECT_CALL(*io_ctx_mock,operate(_,_)).Times(0).WillOnce(Return(0)); + ON_CALL(*io_ctx_mock,get_io_ctx()).WillByDefault(ReturnRef(io_ctx)); + + std::string pool_name("test"); + /*create_connection is invoked by open_connection method*/ + int open_connection = under_test.open_connection(pool_name); + EXPECT_EQ(0, open_connection); + EXPECT_CALL(*cluster_mock, is_connected()).Times(2).WillOnce(Return(true)) + .WillOnce(Return(false)); + + std::string buffer_text="simple_test_one_chunck"; + librmb::RadosMailImpl rados_mail; + void *buffer=(void*)new librados::bufferlist(); + rados_mail.set_mail_buffer(buffer); + ((librados::bufferlist*)rados_mail.get_mail_buffer())->append(buffer_text); + int buffer_length =((librados::bufferlist*)rados_mail.get_mail_buffer())->length(); + rados_mail.set_mail_size(buffer_length); + rados_mail.set_oid("test_mail_id"); + + /*div==1*/ - mailbox_save_cancel(&save_ctx); - i_info("save cancel donw"); - mailbox_transaction_rollback(&trans) ; - i_info("transcatgion collback"); - - mailbox_free(&box); + bool ret_storage = under_test.save_mail(&rados_mail); + EXPECT_EQ(false,ret_storage); + delete cluster_mock; + cluster_mock=nullptr; + delete buffer; +} +TEST_F(StorageTest,true_cluster_connection){ + + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl under_test(cluster_mock); + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + + librmbtest::RboxIoCtxMock *io_ctx_mock=new librmbtest::RboxIoCtxMock(); + under_test.set_io_ctx_wrapper(io_ctx_mock); + librados::IoCtx io_ctx; + librmb::RadosMetadataStorageDefault rados_metadata_storage (io_ctx_mock); + rados_metadata_storage.set_io_ctx(io_ctx_mock); + EXPECT_CALL(*io_ctx_mock,append(_,_,_)).Times(0); + EXPECT_CALL(*io_ctx_mock,operate(_,_)).Times(1).WillOnce(Return(0)); + ON_CALL(*io_ctx_mock,get_io_ctx()).WillByDefault(ReturnRef(io_ctx)); + + std::string pool_name("test"); + /*create_connection is invoked by open_connection method*/ + int open_connection = under_test.open_connection(pool_name); + EXPECT_EQ(0, open_connection); + + EXPECT_CALL(*cluster_mock, is_connected()).Times(2).WillOnce(Return(true)) + .WillOnce(Return(true)); + + - if (test_obj->get_mail_buffer() != nullptr) { - delete test_obj->get_mail_buffer(); - } - delete test_obj; - if (test_obj2->get_mail_buffer() != nullptr) { - delete test_obj2->get_mail_buffer(); + std::string buffer_text="simple_test_one_chunck"; + storage_interface::RadosMail *rados_mail= + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + void *buffer=(void*)new librados::bufferlist(); + rados_mail->set_mail_buffer(buffer); + ((librados::bufferlist*)rados_mail->get_mail_buffer())->append(buffer_text); + int buffer_length = ((librados::bufferlist*)rados_mail->get_mail_buffer())->length(); + rados_mail->set_mail_size(buffer_length); + rados_mail->set_oid("test_mail_id"); + /*div==1*/ + bool ret_storage = under_test.save_mail(rados_mail); + EXPECT_EQ(true,ret_storage); + delete cluster_mock; + cluster_mock=nullptr; + delete buffer; + delete rados_mail; + rados_mail=nullptr; +} +TEST_F(StorageTest,split_buffer){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl under_test(cluster_mock); + + librmbtest::RboxIoCtxMock* io_ctx_mock=new librmbtest::RboxIoCtxMock(); + under_test.set_io_ctx_wrapper(io_ctx_mock); + librados::IoCtx io_ctx; + librmb::RadosMetadataStorageDefault rados_metadata_storage (io_ctx_mock); + rados_metadata_storage.set_io_ctx(io_ctx_mock); + EXPECT_CALL(*io_ctx_mock,append(_,_,_)).Times(4).WillRepeatedly(Return(true)); + EXPECT_CALL(*io_ctx_mock,operate(_,_)).Times(0).WillOnce(Return(0)); + ON_CALL(*io_ctx_mock,get_io_ctx()).WillByDefault(ReturnRef(io_ctx)); + + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + + std::string pool_name("test"); + int open_connection = under_test.open_connection(pool_name); + EXPECT_EQ(0, open_connection); + EXPECT_CALL(*cluster_mock, is_connected()).Times(5).WillRepeatedly(Return(true)); + + std::string buffer_text=""; + for(int i=0;i < under_test.get_max_write_size_bytes();i++){ + buffer_text.append("Sara"); } - delete test_obj2; + librmb::RadosMailImpl rados_mail; + void *buffer=(void*)new librados::bufferlist(); + rados_mail.set_mail_buffer(buffer); + ((librados::bufferlist*)rados_mail.get_mail_buffer())->append(buffer_text); + int buffer_length =((librados::bufferlist*)rados_mail.get_mail_buffer())->length(); + rados_mail.set_mail_size(buffer_length); + rados_mail.set_oid("test_mail_id"); + + /*div==4*/ + bool ret_storage = under_test.save_mail(&rados_mail); + EXPECT_EQ(true,ret_storage); + delete cluster_mock; + cluster_mock=nullptr; + delete buffer; +} +TEST_F(StorageTest,true_first_read){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl storage(cluster_mock); + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + EXPECT_CALL(*cluster_mock, is_connected()).Times(1).WillOnce(Return(true)); + + std::string pool_name("test"); + int open_connection = storage.open_connection(pool_name); + EXPECT_EQ(0, open_connection); + + librmb::RadosMailImpl mail; + const std::string& oid="read_test"; + mail.set_oid(oid); + mail.set_mail_buffer(new std::stringstream); + + librmbtest::RboxIoCtxMock* io_ctx_mock=new librmbtest::RboxIoCtxMock(); + storage.set_io_ctx_wrapper(io_ctx_mock); + librados::IoCtx io_ctx; + ON_CALL(*io_ctx_mock,get_io_ctx()).WillByDefault(ReturnRef(io_ctx)); + EXPECT_CALL(*io_ctx_mock,operate(_,_,_)).Times(1).WillOnce(Return(0)); + int ret=storage.read_mail(oid,&mail,0); + EXPECT_EQ(0,ret); + delete cluster_mock; + cluster_mock=nullptr; +} +TEST_F(StorageTest,true_repeated_read){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl storage(cluster_mock); + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + EXPECT_CALL(*cluster_mock, is_connected()).Times(4).WillRepeatedly(Return(true)); + + std::string pool_name("test"); + int open_connection = storage.open_connection(pool_name); + EXPECT_EQ(0, open_connection); + + librmb::RadosMailImpl mail; + const std::string& oid="read_test"; + mail.set_oid(oid); + mail.set_mail_buffer(new std::stringstream); + + librmbtest::RboxIoCtxMock* io_ctx_mock=new librmbtest::RboxIoCtxMock(); + storage.set_io_ctx_wrapper(io_ctx_mock); + librados::IoCtx io_ctx; + ON_CALL(*io_ctx_mock,get_io_ctx()).WillByDefault(ReturnRef(io_ctx)); + EXPECT_CALL(*io_ctx_mock,operate(_,_,_)).Times(4).WillOnce(Return(-ETIMEDOUT)) + .WillOnce(Return(-ETIMEDOUT)) + .WillOnce(Return(-ETIMEDOUT)) + .WillOnce(Return(0)); + int ret=storage.read_mail(oid,&mail,0); + EXPECT_EQ(0,ret); + delete cluster_mock; + cluster_mock=nullptr; +} +TEST_F(StorageTest,false_read){ + librmbtest::RadosClusterMock *cluster_mock = new librmbtest::RadosClusterMock(); + librmb::RadosStorageImpl storage(cluster_mock); + EXPECT_CALL(*cluster_mock, init()).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, io_ctx_create(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, recovery_index_io_ctx(_ , _)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*cluster_mock, get_config_option(_ , _)).Times(2).WillRepeatedly(Return(0)); + EXPECT_CALL(*cluster_mock, is_connected()).Times(11).WillRepeatedly(Return(true)); + + std::string pool_name("test"); + int open_connection = storage.open_connection(pool_name); + EXPECT_EQ(0, open_connection); + + librmb::RadosMailImpl mail; + const std::string& oid="read_test"; + mail.set_oid(oid); + mail.set_mail_buffer(new std::stringstream); + + librmbtest::RboxIoCtxMock* io_ctx_mock=new librmbtest::RboxIoCtxMock(); + storage.set_io_ctx_wrapper(io_ctx_mock); + librados::IoCtx io_ctx; + ON_CALL(*io_ctx_mock,get_io_ctx()).WillByDefault(ReturnRef(io_ctx)); + EXPECT_CALL(*io_ctx_mock,operate(_,_,_)).Times(11).WillRepeatedly(Return(-ETIMEDOUT)); + int ret=storage.read_mail(oid,&mail,0); + EXPECT_NE(0,ret); + delete cluster_mock; + cluster_mock=nullptr; } - TEST_F(StorageTest, deinit) {} int main(int argc, char **argv) { ::testing::InitGoogleMock(&argc, argv); return RUN_ALL_TESTS(); -} +} \ No newline at end of file diff --git a/src/tests/storage-mock-rbox/test_storage_mock_rbox_bugs.cpp b/src/tests/storage-mock-rbox/test_storage_mock_rbox_bugs.cpp index 9f65a03e..2295b140 100644 --- a/src/tests/storage-mock-rbox/test_storage_mock_rbox_bugs.cpp +++ b/src/tests/storage-mock-rbox/test_storage_mock_rbox_bugs.cpp @@ -12,6 +12,7 @@ #include "../storage-mock-rbox/TestCase.h" #include "gtest/gtest.h" #include "gmock/gmock.h" +#include"../../librmb/rbox-io-ctx-impl.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" // turn off warnings for Dovecot :-( @@ -46,6 +47,8 @@ extern "C" { #include "rados-dovecot-ceph-cfg-impl.h" #include "../../storage-rbox/istream-bufferlist.h" #include "../../storage-rbox/ostream-bufferlist.h" +#include "../../storage-engine/storage-backend-factory.h" +#include "../../storage-interface/rados-mail.h" using ::testing::_; using ::testing::AtLeast; using ::testing::Matcher; @@ -99,19 +102,16 @@ TEST_F(StorageTest, save_mail_rados_connection_failed) { struct rbox_storage *storage = (struct rbox_storage *)box->storage; delete storage->s; librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - - EXPECT_CALL(*storage_mock, aio_operate(_,_,_,_)).Times(AtLeast(1)).WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, wait_for_write_operations_complete(_,_)).WillRepeatedly(Return(false));//failed = false - - librados::IoCtx test_ioctx; - EXPECT_CALL(*storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(test_ioctx)); + + librmb::RboxIoCtxImpl *test_ioctx = new librmb::RboxIoCtxImpl(); + EXPECT_CALL(*storage_mock, get_io_ctx_wrapper()).WillRepeatedly(Return(test_ioctx)); EXPECT_CALL(*storage_mock, set_ceph_wait_method(_)).Times(1); EXPECT_CALL(*storage_mock, set_namespace(_)).Times(1); EXPECT_CALL(*storage_mock, get_namespace()).Times(0); - EXPECT_CALL(*storage_mock, delete_mail(Matcher(_))).Times(0); + EXPECT_CALL(*storage_mock, delete_mail(_)).Times(1); EXPECT_CALL(*storage_mock, close_connection()).Times(0); @@ -119,32 +119,33 @@ TEST_F(StorageTest, save_mail_rados_connection_failed) { .Times(AtLeast(1)) .WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, get_max_object_size()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(100000)); - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()) + EXPECT_CALL(*storage_mock, save_mail(_)) .Times(AtLeast(1)) - .WillRepeatedly(Return(10)); - + .WillRepeatedly(Return(false)); + storage_interface::RadosMail *test_obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + test_obj->set_mail_buffer((void*) new librados::bufferlist()); + ceph::bufferlist *obj_buff=(ceph::bufferlist*)test_obj->get_mail_buffer(); + obj_buff->append(message,strlen(message)-1); + + storage_interface::RadosMail *test_obj2 = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + test_obj2->set_mail_buffer((void*) new librados::bufferlist()); + ceph::bufferlist *obj2_buff=(ceph::bufferlist*)test_obj2->get_mail_buffer(); + obj2_buff->append(message,strlen(message)-1); - librmb::RadosMail *test_obj = new librmb::RadosMail(); - test_obj->set_mail_buffer(nullptr); - librmb::RadosMail *test_obj2 = new librmb::RadosMail(); - test_obj2->set_mail_buffer(nullptr); EXPECT_CALL(*storage_mock, alloc_rados_mail()).Times(2).WillOnce(Return(test_obj)).WillOnce(Return(test_obj2)); EXPECT_CALL(*storage_mock, free_rados_mail(_)).Times(2); delete storage->ms; librmbtest::RadosMetadataStorageProducerMock *ms_p_mock = new librmbtest::RadosMetadataStorageProducerMock(); storage->ms = ms_p_mock; - - librmbtest::RadosStorageMetadataMock ms_mock; - EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _,_)).WillRepeatedly(Return(0)); - EXPECT_CALL(*ms_p_mock, create_metadata_storage(_,_)).Times(1); - + + librmbtest::RadosStorageMetadataMock *ms_mock= new librmbtest::RadosStorageMetadataMock(); + EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(ms_mock)); + EXPECT_CALL(*ms_mock, set_metadata(_)).WillRepeatedly(Return(0)); delete storage->config; librmbtest::RadosDovecotCephCfgMock *cfg_mock = new librmbtest::RadosDovecotCephCfgMock(); EXPECT_CALL(*cfg_mock, is_config_valid()).WillRepeatedly(Return(true)); @@ -167,12 +168,12 @@ TEST_F(StorageTest, save_mail_rados_connection_failed) { EXPECT_CALL(*cfg_mock, is_mail_attribute(_)).WillRepeatedly(Return(true)); EXPECT_CALL(*cfg_mock, is_user_mapping()).WillRepeatedly(Return(false)); EXPECT_CALL(*cfg_mock, get_write_method()).WillRepeatedly(Return(1)); - EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); - storage->ns_mgr->set_config(cfg_mock); storage->config = cfg_mock; storage->s = storage_mock; - + storage->ns_mgr = + storage_engine::StorageBackendFactory::create_namespace_manager(storage_engine::CEPH,cfg_mock); + bool save_failed = FALSE; // ------ begin flow @@ -205,17 +206,26 @@ TEST_F(StorageTest, save_mail_rados_connection_failed) { } else { ret = 0; } - - i_info("ok done."); + i_info("ok done. %d",ret); EXPECT_GE(ret, -1); } + i_info("unref stream!"); i_stream_unref(&input); + i_info("mailbox free!"); mailbox_free(&box); - - if (test_obj->get_mail_buffer() != nullptr) { + + delete test_ioctx; + test_ioctx=nullptr; + delete storage->ns_mgr; + storage->ns_mgr = nullptr; + + i_info("next delete test_obj"); + if(test_obj->get_mail_buffer() != nullptr){ delete test_obj->get_mail_buffer(); } delete test_obj; + + i_info("next delete test_obj2"); if (test_obj2->get_mail_buffer() != nullptr) { delete test_obj2->get_mail_buffer(); } @@ -260,18 +270,16 @@ TEST_F(StorageTest, save_mail_success) { librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - EXPECT_CALL(*storage_mock, aio_operate(_,_,_,_)).Times(AtLeast(1)).WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, wait_for_write_operations_complete(_,_)).WillRepeatedly(Return(false));//failed = false - librados::IoCtx io_ctx; + librmb::RboxIoCtxImpl *test_ioctx = new librmb::RboxIoCtxImpl(); - EXPECT_CALL(*storage_mock, get_io_ctx()).WillRepeatedly(ReturnRef(io_ctx)); + EXPECT_CALL(*storage_mock, get_io_ctx_wrapper()).WillRepeatedly(Return(test_ioctx)); EXPECT_CALL(*storage_mock, set_ceph_wait_method(_)).Times(1); EXPECT_CALL(*storage_mock, set_namespace(_)).Times(1); EXPECT_CALL(*storage_mock, get_namespace()).Times(0); - EXPECT_CALL(*storage_mock, delete_mail(Matcher(_))).Times(0); + EXPECT_CALL(*storage_mock, delete_mail(_)).Times(0); EXPECT_CALL(*storage_mock, close_connection()).Times(1); @@ -279,19 +287,23 @@ TEST_F(StorageTest, save_mail_success) { .Times(AtLeast(1)) .WillRepeatedly(Return(0)); - EXPECT_CALL(*storage_mock, get_max_object_size()) + EXPECT_CALL(*storage_mock, save_mail(_)) .Times(AtLeast(1)) - .WillRepeatedly(Return(100000)); + .WillRepeatedly(Return(true)); - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()) - .Times(AtLeast(1)) - .WillRepeatedly(Return(10)); - - librmb::RadosMail *test_obj = new librmb::RadosMail(); - test_obj->set_mail_buffer(nullptr); - librmb::RadosMail *test_obj2 = new librmb::RadosMail(); - test_obj2->set_mail_buffer(nullptr); + storage_interface::RadosMail *test_obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + test_obj->set_mail_buffer((void*) new librados::bufferlist()); + ceph::bufferlist *obj_buff=(ceph::bufferlist*)test_obj->get_mail_buffer(); + obj_buff->append(message,strlen(message)-1); + + storage_interface::RadosMail *test_obj2 = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); + test_obj2->set_mail_buffer((void*) new librados::bufferlist()); + ceph::bufferlist *obj2_buff=(ceph::bufferlist*)test_obj2->get_mail_buffer(); + obj2_buff->append(message,strlen(message)-1); + EXPECT_CALL(*storage_mock, alloc_rados_mail()).Times(2).WillOnce(Return(test_obj)).WillOnce(Return(test_obj2)); EXPECT_CALL(*storage_mock, free_rados_mail(_)).Times(2); @@ -299,10 +311,9 @@ TEST_F(StorageTest, save_mail_success) { librmbtest::RadosMetadataStorageProducerMock *ms_p_mock = new librmbtest::RadosMetadataStorageProducerMock(); storage->ms = ms_p_mock; - librmbtest::RadosStorageMetadataMock ms_mock; - EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _,_)).WillRepeatedly(Return(0)); - EXPECT_CALL(*ms_p_mock, create_metadata_storage(_,_)).Times(1); + librmbtest::RadosStorageMetadataMock *ms_mock = new librmbtest::RadosStorageMetadataMock(); + EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(ms_mock)); + EXPECT_CALL(*ms_mock, set_metadata(_)).WillRepeatedly(Return(0)); delete storage->config; librmbtest::RadosDovecotCephCfgMock *cfg_mock = new librmbtest::RadosDovecotCephCfgMock(); @@ -312,7 +323,6 @@ TEST_F(StorageTest, save_mail_success) { std::string pool = "mail_storage"; std::string suffix = "_u"; EXPECT_CALL(*cfg_mock, get_index_pool_name()).WillRepeatedly(ReturnRef(pool)); - EXPECT_CALL(*cfg_mock, get_object_search_method()).WillRepeatedly(Return(0)); EXPECT_CALL(*cfg_mock, get_rados_username()).WillRepeatedly(ReturnRef(user)); EXPECT_CALL(*cfg_mock, get_rados_cluster_name()).WillRepeatedly(ReturnRef(cluster)); EXPECT_CALL(*cfg_mock, get_pool_name()).WillRepeatedly(ReturnRef(pool)); @@ -325,10 +335,10 @@ TEST_F(StorageTest, save_mail_success) { EXPECT_CALL(*cfg_mock, is_mail_attribute(_)).WillRepeatedly(Return(true)); EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); EXPECT_CALL(*cfg_mock, is_user_mapping()).WillRepeatedly(Return(false)); - EXPECT_CALL(*cfg_mock, get_write_method()).WillRepeatedly(Return(1)); - storage->ns_mgr->set_config(cfg_mock); + storage->ns_mgr = + storage_engine::StorageBackendFactory::create_namespace_manager(storage_engine::CEPH,cfg_mock); storage->config = cfg_mock; storage->s = storage_mock; @@ -364,21 +374,30 @@ TEST_F(StorageTest, save_mail_success) { ret = 0; } - i_info("ok done."); + i_info("ok done. %d",ret); EXPECT_GE(ret, 0); } i_stream_unref(&input); + i_info("stream unref"); mailbox_free(&box); -i_info("mailbox free done"); + i_info("mailbox free done"); + delete test_ioctx; + test_ioctx=nullptr; + delete storage->ns_mgr; + storage->ns_mgr = nullptr; + i_info("next delete test_obj"); + if(test_obj->get_mail_buffer() != nullptr){ + delete test_obj->get_mail_buffer(); + } delete test_obj; + + i_info("next delete test_obj2"); if (test_obj2->get_mail_buffer() != nullptr) { delete test_obj2->get_mail_buffer(); } - i_info("next delete test_obj2"); delete test_obj2; - } int main(int argc, char **argv) { diff --git a/src/tests/storage-mock-rbox/testdata/gzip_invalid_trailer.mail b/src/tests/storage-mock-rbox/testdata/gzip_invalid_trailer.mail new file mode 100644 index 00000000..bbef2efc Binary files /dev/null and b/src/tests/storage-mock-rbox/testdata/gzip_invalid_trailer.mail differ diff --git a/src/tests/storage-mock-rbox/testdata/gzip_valid_trailer.mail b/src/tests/storage-mock-rbox/testdata/gzip_valid_trailer.mail new file mode 100644 index 00000000..9a76d23b Binary files /dev/null and b/src/tests/storage-mock-rbox/testdata/gzip_valid_trailer.mail differ diff --git a/src/tests/storage-rbox/it_test_check_metadata.cpp b/src/tests/storage-rbox/it_test_check_metadata.cpp index f4846aa6..c144c544 100644 --- a/src/tests/storage-rbox/it_test_check_metadata.cpp +++ b/src/tests/storage-rbox/it_test_check_metadata.cpp @@ -43,7 +43,7 @@ extern "C" { #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" #include "rbox-mail.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" using ::testing::AtLeast; using ::testing::Return; @@ -182,7 +182,8 @@ TEST_F(StorageTest, check_metadata) { i_debug("recv date value %ld = %s", date_recv, buff2); char *val3 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, rmail->rados_mail->get_metadata(), &val3); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, rmail->rados_mail->get_metadata(), &val3); ASSERT_TRUE(val3 != NULL); ASSERT_EQ(rmail->rados_mail->get_metadata()->size(), 8); diff --git a/src/tests/storage-rbox/it_test_copy_rbox.cpp b/src/tests/storage-rbox/it_test_copy_rbox.cpp index 0626ce29..13484343 100644 --- a/src/tests/storage-rbox/it_test_copy_rbox.cpp +++ b/src/tests/storage-rbox/it_test_copy_rbox.cpp @@ -40,8 +40,9 @@ extern "C" { #include "../mocks/mock_test.h" #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" -#include "rados-util.h" - +#include "../../librmb/rados-util-impl.h" +#include "../../storage-engine/storage-backend-factory.h" +#include "../../storage-interface/rados-types.h" using ::testing::AtLeast; using ::testing::Return; @@ -66,7 +67,7 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { struct mail_search_context *search_ctx; struct mail_search_args *search_args; struct mail_search_arg *sarg; - + librmb::RadosUtilsImpl rados_utils; const char *message = "From: user@domain.org\n" "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" @@ -127,10 +128,11 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { EXPECT_EQ(ret2, 0); struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); - std::vector objects; + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper()->nobjects_begin()); + std::vector objects; while (iter != librados::NObjectIterator::__EndObjectIterator) { - librmb::RadosMail *obj = new librmb::RadosMail(); + storage_interface::RadosMail *obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); obj->set_oid((*iter).get_oid()); r_storage->ms->get_storage()->load_metadata(obj); objects.push_back(obj); @@ -141,73 +143,73 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { // compare objects ASSERT_EQ(2, (int)objects.size()); - librmb::RadosMail *mail1 = objects[0]; - librmb::RadosMail *mail2 = objects[1]; + storage_interface::RadosMail *mail1 = objects[0]; + storage_interface::RadosMail *mail2 = objects[1]; char *val = NULL; char *val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); - ASSERT_STREQ(val, val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); + ASSERT_NE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VERSION, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); ASSERT_STRNE(val, val2); ASSERT_EQ(2, (int)box->index->map->hdr.messages_count); diff --git a/src/tests/storage-rbox/it_test_copy_rbox_alt.cpp b/src/tests/storage-rbox/it_test_copy_rbox_alt.cpp index 88538096..ef561961 100644 --- a/src/tests/storage-rbox/it_test_copy_rbox_alt.cpp +++ b/src/tests/storage-rbox/it_test_copy_rbox_alt.cpp @@ -42,7 +42,9 @@ extern "C" { #include "../test-utils/it_utils.h" #include "rbox-mail.h" #include "rbox-storage.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" +#include "../../storage-interface/rados-mail.h" +#include "../../storage-engine/storage-backend-factory.h" using ::testing::AtLeast; using ::testing::Return; @@ -68,7 +70,7 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { struct mail_search_context *search_ctx; struct mail_search_args *search_args; struct mail_search_arg *sarg; - + librmb::RadosUtilsImpl rados_utils; const char *message = "From: user@domain.org\n" "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" @@ -130,7 +132,7 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { struct rbox_mailbox *mbox = (struct rbox_mailbox *)box; // MOVE TO ALT std::string oid = *r_mail->rados_mail->get_oid(); - librmb::RadosUtils::move_to_alt(oid, mbox->storage->s, mbox->storage->alt, mbox->storage->ms, false); + rados_utils.move_to_alt(oid, mbox->storage->s, mbox->storage->alt, mbox->storage->ms, false); } mailbox_save_copy_flags(save_ctx, mail); @@ -152,25 +154,26 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter_alt(r_storage->alt->get_io_ctx().nobjects_begin()); - r_storage->ms->get_storage()->set_io_ctx(&r_storage->alt->get_io_ctx()); - std::vector objects_alt; + librados::NObjectIterator iter_alt(r_storage->alt->get_io_ctx_wrapper().nobjects_begin()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->alt->get_io_ctx_wrapper()); + std::vector objects_alt; while (iter_alt != librados::NObjectIterator::__EndObjectIterator) { - librmb::RadosMail *obj = new librmb::RadosMail(); + storage_interface::RadosMail *obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); obj->set_oid((*iter_alt).get_oid()); r_storage->ms->get_storage()->load_metadata(obj); objects_alt.push_back(obj); iter_alt++; } - r_storage->ms->get_storage()->set_io_ctx(&r_storage->s->get_io_ctx()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->s->get_io_ctx_wrapper()); ASSERT_EQ(2, (int)objects_alt.size()); - librmb::RadosMail *mail1 = objects_alt[0]; - librmb::RadosMail *mail2 = objects_alt[1]; + storage_interface::RadosMail *mail1 = objects_alt[0]; + storage_interface::RadosMail *mail2 = objects_alt[1]; char *val = NULL; char *val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " FLAGS " << std::endl; ASSERT_STREQ(val, val2); @@ -178,8 +181,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { FAIL(); } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " extref " << std::endl; ASSERT_STREQ(val, val2); @@ -188,8 +191,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " from " << std::endl; ASSERT_STREQ(val, val2); @@ -198,8 +201,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " guid " << std::endl; ASSERT_STREQ(val, val2); @@ -207,8 +210,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { FAIL(); } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " mguid" << std::endl; ASSERT_STREQ(val, val2); @@ -216,8 +219,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { FAIL(); } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " mname" << std::endl; ASSERT_STREQ(val, val2); @@ -226,8 +229,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " psize " << val << " val2 " << val2 << std::endl; // ASSERT_STREQ(val, val2); @@ -235,8 +238,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { FAIL(); } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " pop3o" << std::endl; ASSERT_STREQ(val, val2); @@ -244,8 +247,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { FAIL(); } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " pop3uidl" << std::endl; ASSERT_STREQ(val, val2); @@ -254,8 +257,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " pvtfl" << std::endl; ASSERT_STREQ(val, val2); @@ -264,8 +267,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " rtime" << std::endl; ASSERT_STREQ(val, val2); @@ -274,8 +277,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_VERSION, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " versio" << std::endl; ASSERT_STREQ(val, val2); @@ -284,8 +287,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " vsiz" << std::endl; ASSERT_STREQ(val, val2); @@ -293,8 +296,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { FAIL(); } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " stim" << std::endl; ASSERT_STREQ(val, val2); @@ -303,8 +306,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); if (val != NULL && val2 != NULL) { std::cout << " mailuid" << std::endl; ASSERT_STRNE(val, val2); @@ -313,8 +316,8 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { } ASSERT_EQ(2, (int)box->index->map->hdr.messages_count); - r_storage->alt->delete_mail(mail1); - r_storage->alt->delete_mail(mail2); + r_storage->alt->delete_mail(*mail1->get_oid()); + r_storage->alt->delete_mail(*mail2->get_oid()); delete mail1; delete mail2; diff --git a/src/tests/storage-rbox/it_test_copy_rbox_fail.cpp b/src/tests/storage-rbox/it_test_copy_rbox_fail.cpp index 867229be..151e69ba 100644 --- a/src/tests/storage-rbox/it_test_copy_rbox_fail.cpp +++ b/src/tests/storage-rbox/it_test_copy_rbox_fail.cpp @@ -120,22 +120,23 @@ TEST_F(StorageTest, mail_copy_mail_in_inbox) { if (mailbox_search_deinit(&search_ctx) < 0) { FAIL() << "search deinit failed"; } - - if (mailbox_transaction_commit(&desttrans) < 0) { - FAIL() << "tnx commit failed"; - } + mailbox_transaction_commit(&desttrans); + // if (mailbox_transaction_commit(&desttrans) < 0) { + // FAIL() << "tnx commit failed"; + // } if (mailbox_sync(box, static_cast(0)) < 0) { FAIL() << "sync failed"; } struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper()->nobjects_begin()); int msg_count = 0; while (iter != librados::NObjectIterator::__EndObjectIterator) { iter++; msg_count++; } + ASSERT_EQ(1, msg_count); - ASSERT_EQ(1, (int)box->index->map->hdr.messages_count); + ASSERT_EQ(0, (int)box->index->map->hdr.messages_count); mailbox_free(&box); } diff --git a/src/tests/storage-rbox/it_test_doveadm_backup_rbox.cpp b/src/tests/storage-rbox/it_test_doveadm_backup_rbox.cpp index d52b2f43..58284a2d 100644 --- a/src/tests/storage-rbox/it_test_doveadm_backup_rbox.cpp +++ b/src/tests/storage-rbox/it_test_doveadm_backup_rbox.cpp @@ -41,7 +41,7 @@ extern "C" { #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" #include "rbox-copy.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" using ::testing::AtLeast; using ::testing::Return; @@ -71,7 +71,7 @@ TEST_F(StorageTest, mail_doveadm_backup_copy_mail_in_inbox) { struct mail_search_context *search_ctx; struct mail_search_args *search_args; struct mail_search_arg *sarg; - + librmb::RadosUtilsImpl rados_utils; const char *message = "From: user@domain.org\n" "Date: Mon, 06 Nov 2017 09:20:00 +0100\n" @@ -131,10 +131,11 @@ TEST_F(StorageTest, mail_doveadm_backup_copy_mail_in_inbox) { } struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); - std::vector objects; - while (iter != r_storage->s->get_io_ctx().nobjects_end()) { - librmb::RadosMail *obj = new librmb::RadosMail(); + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper().nobjects_begin()); + std::vector objects; + while (iter != r_storage->s->get_io_ctx_wrapper().nobjects_end()) { + storage_interface::RadosMail *obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH) ; obj->set_oid((*iter).get_oid()); r_storage->ms->get_storage()->load_metadata(obj); objects.push_back(obj); @@ -144,71 +145,71 @@ TEST_F(StorageTest, mail_doveadm_backup_copy_mail_in_inbox) { // compare objects ASSERT_EQ(2, (int)objects.size()); - librmb::RadosMail *mail1 = objects[0]; - librmb::RadosMail *mail2 = objects[1]; + storage_interface::RadosMail *mail1 = objects[0]; + storage_interface::RadosMail *mail2 = objects[1]; char *val = NULL; char *val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_VERSION, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); ASSERT_STRNE(val, val2); ASSERT_EQ(2, (int)box->index->map->hdr.messages_count); delete mail1; diff --git a/src/tests/storage-rbox/it_test_lda_rbox.cpp b/src/tests/storage-rbox/it_test_lda_rbox.cpp index e4361360..0ddb5324 100644 --- a/src/tests/storage-rbox/it_test_lda_rbox.cpp +++ b/src/tests/storage-rbox/it_test_lda_rbox.cpp @@ -41,8 +41,8 @@ extern "C" { #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" #include "rbox-copy.h" -#include "rados-util.h" - +#include "../../librmb/rados-util-impl.h" +#include "../../storage-engine/storage-backend-factory.h" using ::testing::AtLeast; using ::testing::Return; @@ -78,7 +78,7 @@ TEST_F(StorageTest, mail_lda_copy_mail_in_inbox) { struct mail_search_context *search_ctx; struct mail_search_args *search_args; struct mail_search_arg *sarg; - + librmb::RadosUtilsImpl rados_utils; const char *message = "From: user@domain.org\n" "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" @@ -138,10 +138,11 @@ TEST_F(StorageTest, mail_lda_copy_mail_in_inbox) { } struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); - std::vector objects; - while (iter != r_storage->s->get_io_ctx().nobjects_end()) { - librmb::RadosMail *obj = new librmb::RadosMail(); + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper()->nobjects_begin()); + std::vector objects; + while (iter != r_storage->s->get_io_ctx_wrapper()->nobjects_end()) { + storage_interface::RadosMail *obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); obj->set_oid((*iter).get_oid()); r_storage->ms->get_storage()->load_metadata(obj); objects.push_back(obj); @@ -151,71 +152,72 @@ TEST_F(StorageTest, mail_lda_copy_mail_in_inbox) { // compare objects ASSERT_EQ(2, (int)objects.size()); - librmb::RadosMail *mail1 = objects[0]; - librmb::RadosMail *mail2 = objects[1]; + storage_interface::RadosMail *mail1 = objects[0]; + storage_interface::RadosMail *mail2 = objects[1]; char *val = NULL; char *val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_EXT_REF, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_EXT_REF, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_FROM_ENVELOPE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_FROM_ENVELOPE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); - ASSERT_STREQ(val, val2); - + // changed: as we are copy the mail instead of moving it, that is why we need + // to have a different GUID + rados_utils.get_metadata(storage_interface::RBOX_METADATA_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_GUID, mail2->get_metadata(), &val2); + ASSERT_NE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); - ASSERT_STREQ(val, val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, mail2->get_metadata(), &val2); + ASSERT_NE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_ORDER, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_ORDER, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_UIDL, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_POP3_UIDL, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PVT_FLAGS, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PVT_FLAGS, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VERSION, mail2->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VERSION, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_SAVE_TIME, mail2->get_metadata(), &val2); ASSERT_STREQ(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, mail2->get_metadata(), &val2); ASSERT_STRNE(val, val2); ASSERT_EQ(2, (int)box->index->map->hdr.messages_count); diff --git a/src/tests/storage-rbox/it_test_move_rbox.cpp b/src/tests/storage-rbox/it_test_move_rbox.cpp index 4e9dbf62..af0e2525 100644 --- a/src/tests/storage-rbox/it_test_move_rbox.cpp +++ b/src/tests/storage-rbox/it_test_move_rbox.cpp @@ -41,7 +41,8 @@ extern "C" { #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" +#include "../../storage-engine/storage-backend-factory.h" using ::testing::AtLeast; using ::testing::Return; @@ -66,7 +67,7 @@ TEST_F(StorageTest, move_mail_test) { struct mail_search_context *search_ctx; struct mail_search_args *search_args; struct mail_search_arg *sarg; - + librmb::RadosUtilsImpl rados_utils; const char *message = "From: user@domain.org\n" "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" @@ -123,44 +124,46 @@ TEST_F(StorageTest, move_mail_test) { if (mailbox_sync(box, static_cast(0)) < 0) { FAIL() << "sync failed"; } - + struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); - std::vector objects; - while (iter != r_storage->s->get_io_ctx().nobjects_end()) { - librmb::RadosMail *obj = new librmb::RadosMail(); + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper()->nobjects_begin()); + std::vector objects; + while (iter != r_storage->s->get_io_ctx_wrapper()->nobjects_end()) { + storage_interface::RadosMail *obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); obj->set_oid((*iter).get_oid()); r_storage->ms->get_storage()->load_metadata(obj); objects.push_back(obj); iter++; + iter++; } - + // compare objects ASSERT_EQ(1, (int)objects.size()); - librmb::RadosMail *mail1 = objects[0]; + storage_interface::RadosMail *mail1 = objects[0]; char *val = NULL; char *val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_GUID, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); + rados_utils.get_metadata(storage_interface::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); ASSERT_EQ(1, (int)box->index->map->hdr.messages_count); diff --git a/src/tests/storage-rbox/it_test_move_rbox_alt.cpp b/src/tests/storage-rbox/it_test_move_rbox_alt.cpp index 500344a1..b4111a61 100644 --- a/src/tests/storage-rbox/it_test_move_rbox_alt.cpp +++ b/src/tests/storage-rbox/it_test_move_rbox_alt.cpp @@ -42,7 +42,7 @@ extern "C" { #include "../test-utils/it_utils.h" #include "rbox-mail.h" #include "rbox-storage.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" using ::testing::AtLeast; using ::testing::Return; @@ -68,7 +68,7 @@ TEST_F(StorageTest, move_mail_from_alt_storage) { struct mail_search_context *search_ctx; struct mail_search_args *search_args; struct mail_search_arg *sarg; - + librmb::RadosUtilsImpl rados_utils; const char *message = "From: user@domain.org\n" "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" @@ -129,7 +129,7 @@ TEST_F(StorageTest, move_mail_from_alt_storage) { struct rbox_mailbox *mbox = (struct rbox_mailbox *)box; // MOVE TO ALT std::string oid = *r_mail->rados_mail->get_oid(); - librmb::RadosUtils::move_to_alt(oid, mbox->storage->s, mbox->storage->alt, mbox->storage->ms, false); + rados_utils.move_to_alt(oid, mbox->storage->s, mbox->storage->alt, mbox->storage->ms, false); } mailbox_save_copy_flags(save_ctx, mail); @@ -150,11 +150,12 @@ TEST_F(StorageTest, move_mail_from_alt_storage) { FAIL() << "sync failed"; } - librados::NObjectIterator iter(r_storage->alt->get_io_ctx().nobjects_begin()); - r_storage->ms->get_storage()->set_io_ctx(&r_storage->alt->get_io_ctx()); - std::vector objects; - while (iter != r_storage->alt->get_io_ctx().nobjects_end()) { - librmb::RadosMail *obj = new librmb::RadosMail(); + librados::NObjectIterator iter(r_storage->alt->get_io_ctx_wrapper().nobjects_begin()); + r_storage->ms->get_storage()->set_io_ctx(r_storage->alt->get_io_ctx_wrapper()); + std::vector objects; + while (iter != r_storage->alt->get_io_ctx_wrapper().nobjects_end()) { + storage_interface::RadosMail *obj = + storage_engine::StorageBackendFactory::create_mail(storage_engine::CEPH); obj->set_oid((*iter).get_oid()); r_storage->ms->get_storage()->load_metadata(obj); objects.push_back(obj); @@ -163,34 +164,34 @@ TEST_F(StorageTest, move_mail_from_alt_storage) { // compare objects ASSERT_EQ(1, (int)objects.size()); - librmb::RadosMail *mail1 = objects[0]; + storage_interface::RadosMail *mail1 = objects[0]; char *val = NULL; char *val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAIL_UID, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_GUID, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_MAILBOX_GUID, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_PHYSICAL_SIZE, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_VIRTUAL_SIZE, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_RECEIVED_TIME, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); val = val2 = NULL; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); + rados_utils.get_metadata(librmb::RBOX_METADATA_ORIG_MAILBOX, mail1->get_metadata(), &val); ASSERT_STRNE(val, val2); ASSERT_EQ(1, (int)box->index->map->hdr.messages_count); - r_storage->alt->delete_mail(mail1); + r_storage->alt->delete_mail(*mail1->get_oid); delete mail1; mailbox_free(&box); } diff --git a/src/tests/storage-rbox/it_test_read_mail_failed_rbox.cpp b/src/tests/storage-rbox/it_test_read_mail_failed_rbox.cpp index 53cd21d6..1ab5bb6a 100644 --- a/src/tests/storage-rbox/it_test_read_mail_failed_rbox.cpp +++ b/src/tests/storage-rbox/it_test_read_mail_failed_rbox.cpp @@ -113,7 +113,7 @@ TEST_F(StorageTest, read_mail_fails) { struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; // delete mail. - r_storage->s->delete_mail(r_mail->rados_mail); + r_storage->s->delete_mail(*r_mail->rados_mail->get_oid()); int ret2 = mail_get_stream(mail, &hdr_size, &body_size, &input); EXPECT_EQ(ret2, -1); diff --git a/src/tests/storage-rbox/it_test_read_mail_rbox.cpp b/src/tests/storage-rbox/it_test_read_mail_rbox.cpp index 33832077..be4a7a06 100644 --- a/src/tests/storage-rbox/it_test_read_mail_rbox.cpp +++ b/src/tests/storage-rbox/it_test_read_mail_rbox.cpp @@ -12,6 +12,8 @@ #include "gtest/gtest.h" #include "gmock/gmock.h" #include "TestCase.h" +#include "../../storage-engine/storage-backend-factory.h" +#include "../../librmb/rados-util-impl.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" // turn off warnings for Dovecot :-( @@ -37,6 +39,7 @@ extern "C" { #include "mail-search.h" } #include "rbox-storage.hpp" +#include "rbox-mail.h" #include "../mocks/mock_test.h" #include "dovecot-ceph-plugin-config.h" #include "../test-utils/it_utils.h" @@ -52,6 +55,118 @@ TEST_F(StorageTest, mailbox_open_inbox) { ASSERT_GE(mailbox_open(box), 0); mailbox_free(&box); } + + +TEST_F(StorageTest, load_update_metadata){ + librados::ObjectWriteOperation write_op; + ceph::bufferlist bl; + std::string val; + + val = "123"; + bl.append(val.c_str(), val.length() + 1); + write_op.setxattr("U",bl); + bl.clear(); + + val = "4567"; + bl.append(val.c_str(), val.length() + 1); + write_op.setxattr("R",bl); + bl.clear(); + + val = "8910111213"; + bl.append(val.c_str(), val.length() + 1); + write_op.setxattr("Z",bl); + bl.clear(); + + val = "111213"; + bl.append(val.c_str(), val.length() + 1); + write_op.setxattr("V",bl); + bl.clear(); + + val = "it is a test to put it not NULL"; + bl.append(val.c_str(), val.length() + 1); + write_op.setxattr("M",bl); + bl.clear(); + + val = "it is not NULL"; + bl.append(val.c_str(), val.length() + 1); + write_op.setxattr("G",bl); + bl.clear(); + + storage_interface::RadosCluster *cluster=storage_engine::StorageBackendFactory::create_cluster(storage_engine::CEPH); + storage_interface::RadosStorage *storage=storage_engine::StorageBackendFactory::create_storage(storage_engine::CEPH,cluster); + std::string pool_name="test_pool"; + storage->open_connection(pool_name); + std::string oid = "test_oid"; + + /**save metadata**/ + storage->get_io_ctx_wrapper()->operate(oid,&write_op); + + /**load metadata**/ + storage_interface::RadosMail *mail_read = storage->alloc_rados_mail(); + mail_read->set_oid("test_oid"); + std::map metadata_map; + int ret = storage->get_io_ctx_wrapper()->getxattrs(*mail_read->get_oid(), metadata_map); + std::cout<<"it is the metadata size"<::iterator it=metadata_map.begin(); it!=metadata_map.end(); ++it){ + std::string key = (*it).first; + std::string value = std::string((*it).second.c_str()); + storage_interface::RadosMetadata *xattr= new librmb::RadosMetadataImpl(key,value); + mail_read->add_metadata(xattr); + } + librmb::RadosUtilsImpl rados_utils; + ret = rados_utils.get_all_keys_and_values(storage->get_io_ctx_wrapper(), *mail_read->get_oid(), mail_read); + + /**validate_metadata**/ + librmb::RadosUtilsImpl utils; + std::string buf_str; + int nummeric = 0; + std::map* metadata=mail_read->get_metadata(); + + if(metadata->find("Z") != metadata->end()){ + buf_str = std::string(((ceph::bufferlist*)(*metadata)["Z"])->c_str()); + std::cout<<"key is Z, value is::"<c_str()) ? 0 : 1; + std::cout<<"key is equal to PSIZE::"<find("R") != metadata->end()){ + buf_str = std::string(((ceph::bufferlist*)(*metadata)["R"])->c_str()); + std::cout<<"key is R, value is::"<c_str()) ? 0 : 1; + std::cout<<"key is equal to TIME::"<find("U") != metadata->end()){ + buf_str = std::string(((ceph::bufferlist*)(*metadata)["U"])->c_str()); + std::cout<<"key is U, value is::"<c_str()) ? 0 : 1; + std::cout<<"key is equal to UID::"<find("V") != metadata->end()){ + buf_str = std::string(((ceph::bufferlist*)(*metadata)["V"])->c_str()); + std::cout<<"key is V, value is::"<c_str()) ? 0 : 1; + std::cout<<"key is equal to VSIZE::"<find("M") != metadata->end()){ + nummeric= ((ceph::bufferlist*)(*metadata)["M"])->c_str() == NULL ? 1 : 0; + std::cout<<"key is equal to mailbox_guid::"<find("G") != metadata->end()){ + nummeric= ((ceph::bufferlist*)(*metadata)["G"])->c_str() == NULL ? 1 : 0; + std::cout<<"key is equal to metadata_guid::"<namespaces); - search_args = mail_search_build_init(); sarg = mail_search_build_add(search_args, SEARCH_ALL); ASSERT_NE(sarg, nullptr); @@ -103,7 +216,6 @@ TEST_F(StorageTest, read_mail_test) { search_ctx = mailbox_search_init(desttrans, search_args, NULL, static_cast(0), NULL); mail_search_args_unref(&search_args); - struct message_size hdr_size, body_size; struct istream *input = NULL; while (mailbox_search_next(search_ctx, &mail)) { @@ -171,6 +283,73 @@ TEST_F(StorageTest, read_mail_test) { mailbox_free(&box); } +TEST_F(StorageTest, read_deleted_mail_test) { + struct mailbox_transaction_context *desttrans; + struct mail *mail; + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + + const char *message = + "From: user@domain.org\n" + "Date: Sat, 24 Mar 2017 23:00:00 +0200\n" + "Mime-Version: 1.0\n" + "Content-Type: text/plain; charset=us-ascii\n" + "\n" + "body\n"; + + const char *mailbox = "INBOX"; + // testdata + testutils::ItUtils::add_mail(message, mailbox, StorageTest::s_test_mail_user->namespaces); + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_ALL); + ASSERT_NE(sarg, nullptr); + + struct mail_namespace *ns = mail_namespace_find_inbox(s_test_mail_user->namespaces); + ASSERT_NE(ns, nullptr); + + struct mailbox *box = mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_SAVEONLY); + + if (mailbox_open(box) < 0) { + i_error("Opening mailbox %s failed: %s", mailbox, mailbox_get_last_internal_error(box, NULL)); + FAIL() << " Forcing a resync on mailbox INBOX Failed"; + } +#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE + desttrans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); +#else + char reason[256]; + memset(reason, '\0', sizeof(reason)); + desttrans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); +#endif + + search_ctx = mailbox_search_init(desttrans, search_args, NULL, static_cast(0), NULL); + mail_search_args_unref(&search_args); + struct message_size hdr_size, body_size; + struct istream *input = NULL; + struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; + while(mailbox_search_next(search_ctx, &mail)){ + struct rbox_mail *rmail = (struct rbox_mail *)mail; + const std::string mail_oid=guid_128_to_string(rmail->index_oid); + r_storage->s->delete_mail(mail_oid); + int ret2 = mail_get_stream(mail, &hdr_size, &body_size, &input); + EXPECT_NE(ret2,0); + break; + } + + if (mailbox_search_deinit(&search_ctx) < 0) { + FAIL() << "search deinit failed"; + } + + if (mailbox_transaction_commit(&desttrans) < 0) { + FAIL() << "tnx commit failed"; + } + + if (mailbox_sync(box, static_cast(0)) < 0) { + std::cout<< "sync failed"<rados_mail->get_oid(); - librmb::RadosUtils::move_to_alt(oid, mbox->storage->s, mbox->storage->alt, mbox->storage->ms, false); + librmb::RadosUtilsImpl rados_utils; + rados_utils.move_to_alt(oid, mbox->storage->s, mbox->storage->alt, mbox->storage->ms, false); } int ret2 = mail_get_stream(mail, &hdr_size, &body_size, &input); diff --git a/src/tests/storage-rbox/it_test_storage_rbox.cpp b/src/tests/storage-rbox/it_test_storage_rbox.cpp index c22ad74f..86f0a264 100644 --- a/src/tests/storage-rbox/it_test_storage_rbox.cpp +++ b/src/tests/storage-rbox/it_test_storage_rbox.cpp @@ -12,6 +12,7 @@ #include "gtest/gtest.h" #include "gmock/gmock.h" #include "TestCase.h" +#include "../../storage-interface/rados-types.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" // turn off warnings for Dovecot :-( @@ -35,7 +36,8 @@ extern "C" { #include "rbox-storage.hpp" #include "../mocks/mock_test.h" #include "rbox-save.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" +#include "../../librmb/rados-mail-impl.h" using ::testing::AtLeast; using ::testing::Return; @@ -158,11 +160,11 @@ TEST_F(StorageTest, mail_save_to_inbox_with_flags) { struct istream *input = i_stream_create_from_data(message, strlen(message)); #ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); + struct mailbox_transaction_context *trans = mailbox_transaction_begin(box,MAILBOX_TRANSACTION_FLAG_EXTERNAL); #else char reason[256]; memset(reason, '\0', sizeof(reason)); - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); + struct mailbox_transaction_context *trans = mailbox_transaction_begin(box,MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); #endif struct mail_save_context *save_ctx = mailbox_save_alloc(trans); struct rbox_save_context *r_ctx = (struct rbox_save_context *)save_ctx; @@ -221,16 +223,17 @@ TEST_F(StorageTest, mail_save_to_inbox_with_flags) { } i_stream_unref(&input); - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); - while (iter != r_storage->s->get_io_ctx().nobjects_end()) { + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper()->nobjects_begin()); + while (iter != r_storage->s->get_io_ctx_wrapper()->nobjects_end()) { if (test_oid.compare((*iter).get_oid()) == 0) { - librmb::RadosMail obj; + librmb::RadosMailImpl obj; obj.set_oid((*iter).get_oid()); r_storage->ms->get_storage()->load_metadata(&obj); char *str; - librmb::RadosUtils::get_metadata(librmb::RBOX_METADATA_OLDV1_FLAGS, obj.get_metadata(), &str); + librmb::RadosUtilsImpl rados_utils; + rados_utils.get_metadata(storage_interface::RBOX_METADATA_OLDV1_FLAGS, obj.get_metadata(), &str); uint8_t flags; - librmb::RadosUtils::string_to_flags(str, &flags); + rados_utils.string_to_flags(str, &flags); EXPECT_EQ(0x01, flags); } iter++; @@ -258,11 +261,11 @@ TEST_F(StorageTest, mail_save_to_inbox_reuse_save_context) { "body\n"; #ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); + struct mailbox_transaction_context *trans = mailbox_transaction_begin(box,MAILBOX_TRANSACTION_FLAG_EXTERNAL); #else char reason[256]; memset(reason, '\0', sizeof(reason)); - struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); + struct mailbox_transaction_context *trans = mailbox_transaction_begin(box,MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); #endif struct mail_save_context *save_ctx = NULL; ssize_t ret; diff --git a/src/tests/sync-rbox/it_test_sync_rbox.cpp b/src/tests/sync-rbox/it_test_sync_rbox.cpp index 2caf52c9..80481d15 100644 --- a/src/tests/sync-rbox/it_test_sync_rbox.cpp +++ b/src/tests/sync-rbox/it_test_sync_rbox.cpp @@ -73,7 +73,7 @@ TEST_F(SyncTest, force_resync_missing_rados_object) { } else { // removing one mail from rados!! struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librados::NObjectIterator iter(r_storage->s->get_io_ctx().nobjects_begin()); + librados::NObjectIterator iter(r_storage->s->get_io_ctx_wrapper()->nobjects_begin()); std::string oid_to_delete; while (iter != librados::NObjectIterator::__EndObjectIterator) { oid_to_delete = (*iter).get_oid(); @@ -86,7 +86,6 @@ TEST_F(SyncTest, force_resync_missing_rados_object) { uint32_t msg_count_before = mail_index_view_get_messages_count(box->view); i_debug("Message count before = %u", msg_count_before); EXPECT_EQ((uint32_t)3, msg_count_before); - if (mailbox_sync(box, static_cast(MAILBOX_SYNC_FLAG_FORCE_RESYNC | MAILBOX_SYNC_FLAG_FIX_INCONSISTENT)) < 0) { i_error("Forcing a resync on mailbox %s failed: %s", mailbox, mailbox_get_last_internal_error(box, NULL)); diff --git a/src/tests/sync-rbox/it_test_sync_rbox_2.cpp b/src/tests/sync-rbox/it_test_sync_rbox_2.cpp index 5946a688..5655ae92 100644 --- a/src/tests/sync-rbox/it_test_sync_rbox_2.cpp +++ b/src/tests/sync-rbox/it_test_sync_rbox_2.cpp @@ -37,7 +37,7 @@ extern "C" { #include "rbox-storage.hpp" #include "../mocks/mock_test.h" #include "../test-utils/it_utils.h" - +#include "../../librmb/rados-metadata-impl.h" using ::testing::AtLeast; using ::testing::Return; @@ -48,13 +48,12 @@ TEST_F(SyncTest, init) {} static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librmb::RadosMetadata xattr(librmb::rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, box->name); - librados::NObjectIterator iter = r_storage->s->find_mails(&xattr); - + librmb::RadosMetadataImpl xattr(storage_interface::rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, box->name); + std::set mail_list = r_storage->s->find_mails(&xattr); + std::set::iterator mail_iter; std::string oid; - while (iter != librados::NObjectIterator::__EndObjectIterator) { - oid = iter->get_oid(); - iter++; + for(mail_iter=mail_list.begin();mail_iter!=mail_list.end();++mail_iter){ + oid = *mail_iter; } EXPECT_TRUE(oid.length() > 0); @@ -68,25 +67,25 @@ static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { uint64_t size = 0; time_t pmtime = 0; - int stat_ret = r_storage->s->get_io_ctx().stat(oid, &size, &pmtime); + int stat_ret = r_storage->s->get_io_ctx_wrapper()->stat(oid, &size, &pmtime); EXPECT_NE(size, 0); - i_debug("Last Version = %lu for obj: %s , stat =%d, %ld", r_storage->s->get_io_ctx().get_last_version(), oid.c_str(), + i_debug("Last Version = %lu for obj: %s , stat =%d, %ld", r_storage->s->get_io_ctx_wrapper()->get_last_version(), oid.c_str(), stat_ret, size); #if LIBRADOS_VERSION_CODE >= 30000 - write_op.copy_from(oid, r_storage->s->get_io_ctx(), r_storage->s->get_io_ctx().get_last_version(), 0); + write_op.copy_from(oid, r_storage->s->get_io_ctx_wrapper()->get_io_ctx() , r_storage->s->get_io_ctx_wrapper()->get_last_version(), 0); #else - write_op.copy_from(oid, r_storage->s->get_io_ctx(), r_storage->s->get_io_ctx().get_last_version()); + write_op.copy_from(oid, r_storage->s->get_io_ctx_wrapper()->get_io_ctx(), r_storage->s->get_io_ctx_wrapper()->get_last_version()); #endif - int ret = r_storage->s->get_io_ctx().operate(test_oid, &write_op); + int ret = r_storage->s->get_io_ctx_wrapper()->operate(test_oid, &write_op); i_debug("copy operate: %d for %s", ret, test_oid.c_str()); EXPECT_EQ(ret, 0); const char *metadata_name = "U"; librados::bufferlist list; list.append("100", 4); - ret = r_storage->s->get_io_ctx().setxattr(test_oid, metadata_name, list); + ret = r_storage->s->get_io_ctx_wrapper()->setxattr(test_oid, metadata_name, list); i_debug("copy operate setxattr: %d for %s", ret, test_oid.c_str()); EXPECT_EQ(ret, 0); diff --git a/src/tests/sync-rbox/it_test_sync_rbox_alt.cpp b/src/tests/sync-rbox/it_test_sync_rbox_alt.cpp index da54760a..25e2f1dc 100644 --- a/src/tests/sync-rbox/it_test_sync_rbox_alt.cpp +++ b/src/tests/sync-rbox/it_test_sync_rbox_alt.cpp @@ -37,7 +37,7 @@ extern "C" { #include "rbox-storage.hpp" #include "../mocks/mock_test.h" #include "../test-utils/it_utils.h" -#include "rados-util.h" +#include "../../librmb/rados-util-impl.h" #include "rbox-storage.h" using ::testing::AtLeast; @@ -48,13 +48,12 @@ TEST_F(SyncTest, init) {} static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librmb::RadosMetadata xattr(librmb::rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, box->name); - librados::NObjectIterator iter = r_storage->s->find_mails(&xattr); - + librmb::RadosMetadataImpl xattr(librmb::rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, box->name); + std::set mail_list = r_storage->s->find_mails(&xattr); + std::set::iterator mail_iter; std::string oid; - while (iter != librados::NObjectIterator::__EndObjectIterator) { - oid = iter->get_oid(); - iter++; + for(mail_iter=mail_list.begin();mail_iter!=mail_list.end();++mail_iter){ + oid = *mail_iter; } EXPECT_TRUE(oid.length() > 0); @@ -68,18 +67,18 @@ static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { // wait for test cluster (object exists...) uint64_t size = 0; time_t pmtime = 0; - int stat_ret = r_storage->s->get_io_ctx().stat(oid, &size, &pmtime); + int stat_ret = r_storage->s->get_io_ctx_wrapper().stat(oid, &size, &pmtime); EXPECT_NE(size, 0); - i_debug("Last Version = %lu for obj: %s , stat =%d, %ld", r_storage->s->get_io_ctx().get_last_version(), oid.c_str(), + i_debug("Last Version = %lu for obj: %s , stat =%d, %ld", r_storage->s->get_io_ctx_wrapper().get_last_version(), oid.c_str(), stat_ret, size); #if LIBRADOS_VERSION_CODE >= 30000 - write_op.copy_from(oid, r_storage->s->get_io_ctx(), r_storage->s->get_io_ctx().get_last_version(), 0); + write_op.copy_from(oid, r_storage->s->get_io_ctx_wrapper().get_io_ctx , r_storage->s->get_io_ctx_wrapper().get_last_version(), 0); #else - write_op.copy_from(oid, r_storage->s->get_io_ctx(), r_storage->s->get_io_ctx().get_last_version()); + write_op.copy_from(oid, r_storage->s->get_io_ctx_wrapper().get_io_ctx, r_storage->s->get_io_ctx_wrapper().get_last_version()); #endif - int ret = r_storage->s->get_io_ctx().operate(test_oid, &write_op); + int ret = r_storage->s->get_io_ctx_wrapper().operate(test_oid, &write_op); i_debug("copy operate: %d for %s", ret, test_oid.c_str()); EXPECT_EQ(ret, 0); @@ -88,14 +87,14 @@ static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { librados::bufferlist list; std::string id = "100"; list.append(id.c_str(), id.length() + 1); - ret = r_storage->s->get_io_ctx().setxattr(test_oid, metadata_name, list); + ret = r_storage->s->get_io_ctx_wrapper().setxattr(test_oid, metadata_name, list); i_debug("copy operate setxattr: %d for %s", ret, test_oid.c_str()); if (rbox_open_rados_connection(box, true) < 0) { FAIL() << "error opening connection"; } - - if (librmb::RadosUtils::move_to_alt(test_oid, r_storage->s, r_storage->alt, r_storage->ms, false) < 0) { + librmb::RadosUtilsImpl rados_utils; + if (rados_utils.move_to_alt(test_oid, r_storage->s, r_storage->alt, r_storage->ms, false) < 0) { FAIL() << "error moving object to alt storage"; } i_debug("ok, mail moved to alt"); diff --git a/src/tests/sync-rbox/it_test_sync_rbox_duplicate_uid.cpp b/src/tests/sync-rbox/it_test_sync_rbox_duplicate_uid.cpp index 2b82be17..3d6cd49b 100644 --- a/src/tests/sync-rbox/it_test_sync_rbox_duplicate_uid.cpp +++ b/src/tests/sync-rbox/it_test_sync_rbox_duplicate_uid.cpp @@ -37,6 +37,7 @@ extern "C" { #include "rbox-storage.hpp" #include "../mocks/mock_test.h" #include "../test-utils/it_utils.h" +#include "../../librmb/rados-metadata-impl.h" using ::testing::AtLeast; using ::testing::Return; @@ -48,13 +49,12 @@ TEST_F(SyncTest, init) {} static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { struct rbox_storage *r_storage = (struct rbox_storage *)box->storage; - librmb::RadosMetadata xattr(librmb::rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, box->name); - librados::NObjectIterator iter = r_storage->s->find_mails(&xattr); - + librmb::RadosMetadataImpl xattr(storage_interface::rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, box->name); + std::set mail_list = r_storage->s->find_mails(&xattr); + std::set::iterator mail_iter; std::string oid; - while (iter != librados::NObjectIterator::__EndObjectIterator) { - oid = iter->get_oid(); - iter++; + for(mail_iter=mail_list.begin();mail_iter!=mail_list.end();++mail_iter){ + oid = *mail_iter; } EXPECT_TRUE(oid.length() > 0); @@ -68,18 +68,18 @@ static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { uint64_t size = 0; time_t pmtime = 0; - int stat_ret = r_storage->s->get_io_ctx().stat(oid, &size, &pmtime); + int stat_ret = r_storage->s->get_io_ctx_wrapper()->stat(oid, &size, &pmtime); EXPECT_NE(size, 0); - i_debug("Last Version = %lu for obj: %s , stat =%d, %ld", r_storage->s->get_io_ctx().get_last_version(), oid.c_str(), + i_debug("Last Version = %lu for obj: %s , stat =%d, %ld", r_storage->s->get_io_ctx_wrapper()->get_last_version(), oid.c_str(), stat_ret, size); #if LIBRADOS_VERSION_CODE >= 30000 - write_op.copy_from(oid, r_storage->s->get_io_ctx(), r_storage->s->get_io_ctx().get_last_version(), 0); + write_op.copy_from(oid, r_storage->s->get_io_ctx_wrapper()->get_io_ctx(), r_storage->s->get_io_ctx_wrapper()->get_last_version(), 0); #else - write_op.copy_from(oid, r_storage->s->get_io_ctx(), r_storage->s->get_io_ctx().get_last_version()); + write_op.copy_from(oid, r_storage->s->get_io_ctx_wrapper()->get_io_ctx(), r_storage->s->get_io_ctx_wrapper()->get_last_version()); #endif - int ret = r_storage->s->get_io_ctx().operate(test_oid, &write_op); + int ret = r_storage->s->get_io_ctx_wrapper()->operate(test_oid, &write_op); i_debug("copy operate: %d for %s", ret, test_oid.c_str()); EXPECT_EQ(ret, 0); @@ -87,7 +87,7 @@ static void copy_object(struct mail_namespace *_ns, struct mailbox *box) { librados::bufferlist list; std::string id = "1"; list.append(id.c_str(), id.length() + 1); - ret = r_storage->s->get_io_ctx().setxattr(test_oid, metadata_name, list); + ret = r_storage->s->get_io_ctx_wrapper()->setxattr(test_oid, metadata_name, list); EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0); diff --git a/src/tests/test-utils/it_utils.cpp b/src/tests/test-utils/it_utils.cpp index e1da9c16..3e97e6e2 100644 --- a/src/tests/test-utils/it_utils.cpp +++ b/src/tests/test-utils/it_utils.cpp @@ -40,17 +40,18 @@ ItUtils::ItUtils() {} ItUtils::~ItUtils() {} void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns, - librmb::RadosStorage *storage_impl) { + storage_interface::RadosStorage *storage_impl) { struct mail_namespace *ns = mail_namespace_find_inbox(_ns); ASSERT_NE(ns, nullptr); struct mailbox *box = mailbox_alloc(ns->list, mailbox, (mailbox_flags)0); ASSERT_NE(box, nullptr); ASSERT_GE(mailbox_open(box), 0); - - librados::bufferlist *i_stream_buffer = new librados::bufferlist(); - i_stream_buffer->append(message); - struct istream *input = i_stream_create_from_bufferlist(i_stream_buffer, i_stream_buffer->length()); - + std::stringstream i_stream_buffer; + i_stream_buffer<append_to_buffer(message_buff,message,i_stream_buffer.str().length()); + struct istream *input = i_stream_create_from_bufferlist(message_buff,message,i_stream_buffer.str().length()); + #ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); #else @@ -75,17 +76,19 @@ void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_nam librmbtest::RadosStorageMetadataMock ms_mock; EXPECT_CALL(*ms_p_mock, get_storage()).WillRepeatedly(Return(&ms_mock)); - EXPECT_CALL(ms_mock, set_metadata(_, _)).WillRepeatedly(Return(0)); + EXPECT_CALL(ms_mock, set_metadata(_)).WillRepeatedly(Return(0)); std::string user = "client.admin"; std::string cluster = "ceph"; std::string pool = "mail_storage"; std::string suffix = "_u"; + EXPECT_CALL(*cfg_mock, get_index_pool_name()).WillRepeatedly(ReturnRef(pool)); EXPECT_CALL(*cfg_mock, get_rados_username()).WillRepeatedly(ReturnRef(user)); EXPECT_CALL(*cfg_mock, get_rados_cluster_name()).WillRepeatedly(ReturnRef(cluster)); EXPECT_CALL(*cfg_mock, get_pool_name()).WillRepeatedly(ReturnRef(pool)); EXPECT_CALL(*cfg_mock, get_user_suffix()).WillRepeatedly(ReturnRef(suffix)); + EXPECT_CALL(*cfg_mock, get_chunk_size()).WillOnce(Return(100)); storage->ns_mgr->set_config(cfg_mock); storage->config = cfg_mock; @@ -100,10 +103,12 @@ void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_nam struct mailbox *box = mailbox_alloc(ns->list, mailbox, (mailbox_flags)0); ASSERT_NE(box, nullptr); ASSERT_GE(mailbox_open(box), 0); - librados::bufferlist *i_stream_buffer = new librados::bufferlist(); - i_stream_buffer->append(message); - struct istream *input = i_stream_create_from_bufferlist(i_stream_buffer, i_stream_buffer->length()); - + struct rbox_storage *storage = (struct rbox_storage *)box->storage; + std::stringstream i_stream_buffer; + i_stream_buffer<s->append_to_buffer(message_buff,message,i_stream_buffer.str().length()); + struct istream *input = i_stream_create_from_bufferlist(message_buff,message,i_stream_buffer.str().length()); #ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); #else @@ -112,9 +117,7 @@ void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_nam struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); #endif struct mail_save_context *save_ctx = mailbox_save_alloc(trans); - ItUtils::add_mail(save_ctx, input, box, trans); - i_stream_unref(&input); mailbox_free(&box); } @@ -122,7 +125,6 @@ void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_nam void ItUtils::add_mail(struct mail_save_context *save_ctx, struct istream *input, struct mailbox *box, struct mailbox_transaction_context *trans) { bool save_failed = FALSE; - if (mailbox_save_begin(&save_ctx, input) < 0) { i_error("Saving failed: %s", mailbox_get_last_internal_error(box, NULL)); mailbox_transaction_rollback(&trans); diff --git a/src/tests/test-utils/it_utils.h b/src/tests/test-utils/it_utils.h index 2a339d6d..445b03b1 100644 --- a/src/tests/test-utils/it_utils.h +++ b/src/tests/test-utils/it_utils.h @@ -51,7 +51,7 @@ class ItUtils { virtual ~ItUtils(); static void add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns); static void add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns, - librmb::RadosStorage *storage_impl); + storage_interface::RadosStorage *storage_impl); static void add_mail(struct mail_save_context *save_ctx, struct istream *input, struct mailbox *box, struct mailbox_transaction_context *trans); }; diff --git a/src/tests/test_storage_mock_rbox_bugs b/src/tests/test_storage_mock_rbox_bugs new file mode 100755 index 00000000..2ec21caa --- /dev/null +++ b/src/tests/test_storage_mock_rbox_bugs @@ -0,0 +1,228 @@ +#! /bin/sh + +# test_storage_mock_rbox_bugs - temporary wrapper script for .libs/test_storage_mock_rbox_bugs +# Generated by libtool (GNU libtool) 2.4.2 +# +# The test_storage_mock_rbox_bugs program cannot be directly executed until all the libtool +# libraries that it depends on are installed. +# +# This wrapper script should never be moved out of the build directory. +# If it is, it will not operate correctly. + +# Sed substitution that helps us do robust quoting. It backslashifies +# metacharacters that are still active within double-quoted strings. +sed_quote_subst='s/\([`"$\\]\)/\\\1/g' + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac +fi +BIN_SH=xpg4; export BIN_SH # for Tru64 +DUALCASE=1; export DUALCASE # for MKS sh + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +relink_command="(cd /repo/src/tests; { test -z \"\${LIBRARY_PATH+set}\" || unset LIBRARY_PATH || { LIBRARY_PATH=; export LIBRARY_PATH; }; }; { test -z \"\${COMPILER_PATH+set}\" || unset COMPILER_PATH || { COMPILER_PATH=; export COMPILER_PATH; }; }; { test -z \"\${GCC_EXEC_PREFIX+set}\" || unset GCC_EXEC_PREFIX || { GCC_EXEC_PREFIX=; export GCC_EXEC_PREFIX; }; }; { test -z \"\${LD_RUN_PATH+set}\" || unset LD_RUN_PATH || { LD_RUN_PATH=; export LD_RUN_PATH; }; }; { test -z \"\${LD_LIBRARY_PATH+set}\" || unset LD_LIBRARY_PATH || { LD_LIBRARY_PATH=; export LD_LIBRARY_PATH; }; }; PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; export PATH; g++ -std=gnu++11 -pthread -fpermissive -std=c++11 -Wmissing-declarations -g -O0 -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-undef -Wno-redundant-decls -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-error=undef -Wno-error=redundant-decls -fpermissive -std=c++11 -fPIC -DPIC -o \$progdir/\$file storage-mock-rbox/test_storage_mock_rbox_bugs-test_storage_mock_rbox_bugs.o storage-mock-rbox/test_storage_mock_rbox_bugs-TestCase.o test-utils/test_storage_mock_rbox_bugs-it_utils.o ../../src/storage-rbox/.libs/libstorage_rbox_plugin.so -L/usr/local/lib/dovecot /repo/src/librmb/.libs/librmb.so /usr/local/lib/dovecot/libdovecot-storage.so /usr/local/lib/dovecot/libdovecot.so -ldl ../../src/librmb/.libs/librmb.so -ljansson ./.libs/libgmockgtest.a -lrados -pthread -Wl,-rpath -Wl,/repo/src/storage-rbox/.libs -Wl,-rpath -Wl,/repo/src/librmb/.libs -Wl,-rpath -Wl,/usr/local/lib/dovecot -Wl,-rpath -Wl,/usr/local/lib/dovecot -Wl,-rpath -Wl,/usr/local/lib)" + +# This environment variable determines our operation mode. +if test "$libtool_install_magic" = "%%%MAGIC variable%%%"; then + # install mode needs the following variables: + generated_by_libtool_version='2.4.2' + notinst_deplibs=' ../../src/storage-rbox/libstorage_rbox_plugin.la /repo/src/librmb/librmb.la ../../src/librmb/librmb.la' +else + # When we are sourced in execute mode, $file and $ECHO are already set. + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + file="$0" + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +$1 +_LTECHO_EOF' +} + ECHO="printf %s\\n" + fi + +# Very basic option parsing. These options are (a) specific to +# the libtool wrapper, (b) are identical between the wrapper +# /script/ and the wrapper /executable/ which is used only on +# windows platforms, and (c) all begin with the string --lt- +# (application programs are unlikely to have options which match +# this pattern). +# +# There are only two supported options: --lt-debug and +# --lt-dump-script. There is, deliberately, no --lt-help. +# +# The first argument to this parsing function should be the +# script's ../../libtool value, followed by no. +lt_option_debug= +func_parse_lt_options () +{ + lt_script_arg0=$0 + shift + for lt_opt + do + case "$lt_opt" in + --lt-debug) lt_option_debug=1 ;; + --lt-dump-script) + lt_dump_D=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%/[^/]*$%%'` + test "X$lt_dump_D" = "X$lt_script_arg0" && lt_dump_D=. + lt_dump_F=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%^.*/%%'` + cat "$lt_dump_D/$lt_dump_F" + exit 0 + ;; + --lt-*) + $ECHO "Unrecognized --lt- option: '$lt_opt'" 1>&2 + exit 1 + ;; + esac + done + + # Print the debug banner immediately: + if test -n "$lt_option_debug"; then + echo "test_storage_mock_rbox_bugs:test_storage_mock_rbox_bugs:${LINENO}: libtool wrapper (GNU libtool) 2.4.2" 1>&2 + fi +} + +# Used when --lt-debug. Prints its arguments to stdout +# (redirection is the responsibility of the caller) +func_lt_dump_args () +{ + lt_dump_args_N=1; + for lt_arg + do + $ECHO "test_storage_mock_rbox_bugs:test_storage_mock_rbox_bugs:${LINENO}: newargv[$lt_dump_args_N]: $lt_arg" + lt_dump_args_N=`expr $lt_dump_args_N + 1` + done +} + +# Core function for launching the target application +func_exec_program_core () +{ + + if test -n "$lt_option_debug"; then + $ECHO "test_storage_mock_rbox_bugs:test_storage_mock_rbox_bugs:${LINENO}: newargv[0]: $progdir/$program" 1>&2 + func_lt_dump_args ${1+"$@"} 1>&2 + fi + exec "$progdir/$program" ${1+"$@"} + + $ECHO "$0: cannot exec $program $*" 1>&2 + exit 1 +} + +# A function to encapsulate launching the target application +# Strips options in the --lt-* namespace from $@ and +# launches target application with the remaining arguments. +func_exec_program () +{ + case " $* " in + *\ --lt-*) + for lt_wr_arg + do + case $lt_wr_arg in + --lt-*) ;; + *) set x "$@" "$lt_wr_arg"; shift;; + esac + shift + done ;; + esac + func_exec_program_core ${1+"$@"} +} + + # Parse options + func_parse_lt_options "$0" ${1+"$@"} + + # Find the directory that this script lives in. + thisdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'` + test "x$thisdir" = "x$file" && thisdir=. + + # Follow symbolic links until we get to the real thisdir. + file=`ls -ld "$file" | /usr/bin/sed -n 's/.*-> //p'` + while test -n "$file"; do + destdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'` + + # If there was a directory component, then change thisdir. + if test "x$destdir" != "x$file"; then + case "$destdir" in + [\\/]* | [A-Za-z]:[\\/]*) thisdir="$destdir" ;; + *) thisdir="$thisdir/$destdir" ;; + esac + fi + + file=`$ECHO "$file" | /usr/bin/sed 's%^.*/%%'` + file=`ls -ld "$thisdir/$file" | /usr/bin/sed -n 's/.*-> //p'` + done + + # Usually 'no', except on cygwin/mingw when embedded into + # the cwrapper. + WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=no + if test "$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR" = "yes"; then + # special case for '.' + if test "$thisdir" = "."; then + thisdir=`pwd` + fi + # remove .libs from thisdir + case "$thisdir" in + *[\\/].libs ) thisdir=`$ECHO "$thisdir" | /usr/bin/sed 's%[\\/][^\\/]*$%%'` ;; + .libs ) thisdir=. ;; + esac + fi + + # Try to get the absolute directory name. + absdir=`cd "$thisdir" && pwd` + test -n "$absdir" && thisdir="$absdir" + + program=lt-'test_storage_mock_rbox_bugs' + progdir="$thisdir/.libs" + + if test ! -f "$progdir/$program" || + { file=`ls -1dt "$progdir/$program" "$progdir/../$program" 2>/dev/null | /usr/bin/sed 1q`; \ + test "X$file" != "X$progdir/$program"; }; then + + file="$$-$program" + + if test ! -d "$progdir"; then + mkdir "$progdir" + else + rm -f "$progdir/$file" + fi + + # relink executable if necessary + if test -n "$relink_command"; then + if relink_command_output=`eval $relink_command 2>&1`; then : + else + printf %s\n "$relink_command_output" >&2 + rm -f "$progdir/$file" + exit 1 + fi + fi + + mv -f "$progdir/$file" "$progdir/$program" 2>/dev/null || + { rm -f "$progdir/$program"; + mv -f "$progdir/$file" "$progdir/$program"; } + rm -f "$progdir/$file" + fi + + if test -f "$progdir/$program"; then + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + # Run the actual program with our arguments. + func_exec_program ${1+"$@"} + fi + else + # The program doesn't exist. + $ECHO "$0: error: \`$progdir/$program' does not exist" 1>&2 + $ECHO "This script is just a wrapper for $program." 1>&2 + $ECHO "See the libtool documentation for more information." 1>&2 + exit 1 + fi +fi diff --git a/startup.sh b/startup.sh new file mode 100755 index 00000000..b03e7202 --- /dev/null +++ b/startup.sh @@ -0,0 +1,6 @@ +echo "starting up...." +echo "changing permissions" +chmod -R 777 /usr/local/var/ & ldconfig +dovecot +echo "dovecot up and running" +tail -f /var/log/dovecot.log