diff --git a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java index a9b5a0223f8..66988406288 100644 --- a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java +++ b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java @@ -92,6 +92,6 @@ private static void createKeyspace(KeyspaceConfiguration keyspaceConfiguration, } private static void ensureContactable(CqlSession session) { - session.execute("SELECT dateof(now()) FROM system.local ;"); + session.execute("SELECT toTimestamp(now()) FROM system.local ;"); } } diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java index 618125b1366..f8073c63ae3 100644 --- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java +++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java @@ -137,7 +137,7 @@ private static Optional getFixedBuildId() { @SuppressWarnings("resource") public DockerCassandra() { - this("cassandra_4_1_5-" + buildSpecificImageDiscriminator(), + this("cassandra_5_0_4-" + buildSpecificImageDiscriminator(), getFixedBuildId().isEmpty(), AdditionalDockerFileStep.IDENTITY); } @@ -170,18 +170,14 @@ public void close() { } }); - String memorySettingCommand = Optional.ofNullable(System.getenv("CI")) - .map(Boolean::parseBoolean) - .filter(Boolean.TRUE::equals) - .map(ci -> "echo \"\" ") - .orElse("echo \"-Xms" + CASSANDRA_MEMORY + "M\" >> " + JVM_OPTIONS - + "&& echo \"-Xmx" + CASSANDRA_MEMORY + "M\" >> " + JVM_OPTIONS); + String memorySettingCommand = "echo \"-Xms" + CASSANDRA_MEMORY + "M\" >> " + JVM_OPTIONS + + "&& echo \"-Xmx" + CASSANDRA_MEMORY + "M\" >> " + JVM_OPTIONS; cassandraContainer = new GenericContainer<>( new ImageFromDockerfile(imageName,deleteImageAfterUsage) .withDockerfileFromBuilder(builder -> additionalSteps.applyStep(builder - .from("cassandra:4.1.9") + .from("cassandra:5.0.4") .env("CASSANDRA_CONFIG", "/etc/cassandra") .run(memorySettingCommand + "&& echo \"-Dcassandra.skip_wait_for_gossip_to_settle=0\" >> " + JVM_OPTIONS diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java index 5908eb7a3fa..cabf5ec14b2 100644 --- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java +++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java @@ -66,7 +66,7 @@ void createShouldReturnAContactableCluster(DockerCassandra dockerCassandra) { void assertThatClusterIsContactable(CqlSession session) { try { - session.execute("SELECT dateof(now()) FROM system.local ;"); + session.execute("SELECT toTimestamp(now()) FROM system.local ;"); } catch (Exception e) { throw new AssertionError("expecting cluster can be connected but actually not", e); } diff --git a/docs/modules/servers/pages/distributed/operate/logging/docker-compose-block.adoc b/docs/modules/servers/pages/distributed/operate/logging/docker-compose-block.adoc index 3a096778a38..a63762603b9 100644 --- a/docs/modules/servers/pages/distributed/operate/logging/docker-compose-block.adoc +++ b/docs/modules/servers/pages/distributed/operate/logging/docker-compose-block.adoc @@ -35,7 +35,7 @@ services: - discovery.type=single-node cassandra: - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" diff --git a/docs/modules/servers/pages/distributed/run/run-docker.adoc b/docs/modules/servers/pages/distributed/run/run-docker.adoc index f05f2f71d7e..5d44dcd1a08 100644 --- a/docs/modules/servers/pages/distributed/run/run-docker.adoc +++ b/docs/modules/servers/pages/distributed/run/run-docker.adoc @@ -53,7 +53,7 @@ Firstly, create your own user network on Docker for the James environment: You need a running *cassandra* in docker which connects to *james* network. To achieve this run: - $ docker run -d --network james --name=cassandra cassandra:4.1.9 + $ docker run -d --network james --name=cassandra cassandra:5.0.4 You need a running *rabbitmq* in docker which connects to *james* network. To achieve this run: diff --git a/docs/modules/servers/pages/distributed/run/run-java.adoc b/docs/modules/servers/pages/distributed/run/run-java.adoc index 95060038431..cc8e32bee46 100644 --- a/docs/modules/servers/pages/distributed/run/run-java.adoc +++ b/docs/modules/servers/pages/distributed/run/run-java.adoc @@ -50,7 +50,7 @@ running. You can either install the servers or launch them via docker: [source,bash] ---- -$ docker run -d -p 9042:9042 --name=cassandra cassandra:4.1.9 +$ docker run -d -p 9042:9042 --name=cassandra cassandra:5.0.4 $ docker run -d --network james -p 9200:9200 --name=opensearch --env 'discovery.type=single-node' opensearchproject/opensearch:2.19.2 $ docker run -d -p 5672:5672 -p 15672:15672 --name=rabbitmq rabbitmq:4.1.1-management $ docker run -d --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 ghcr.io/scality/cloudserver:c1ba296859690c1cbbec609aaae430f6b04b4745 diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMailboxDataDefinition.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMailboxDataDefinition.java index 42f97eb97c7..ee58bd1b3ab 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMailboxDataDefinition.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMailboxDataDefinition.java @@ -40,8 +40,7 @@ public interface CassandraMailboxDataDefinition { .table(CassandraMailboxTable.TABLE_NAME) .comment("Holds the mailboxes information.") .options(options -> options - .withCaching(true, rows(CassandraConstants.DEFAULT_CACHED_ROW_PER_PARTITION)) - .withLZ4Compression(8, 1)) + .withCaching(true, rows(CassandraConstants.DEFAULT_CACHED_ROW_PER_PARTITION))) .statement(statement -> types -> statement .withPartitionKey(CassandraMailboxTable.ID, TIMEUUID) .withColumn(CassandraMailboxTable.MAILBOX_BASE, types.getDefinedUserType(CassandraMailboxTable.MAILBOX_BASE.asCql(true))) diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMessageDataDefinition.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMessageDataDefinition.java index 51b96d864e8..7944ad33526 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMessageDataDefinition.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/modules/CassandraMessageDataDefinition.java @@ -74,7 +74,6 @@ public interface CassandraMessageDataDefinition { .comment("Holds mailbox and flags for each message, lookup by message ID") .options(options -> options .withCompaction(SchemaBuilder.sizeTieredCompactionStrategy()) - .withLZ4Compression(8, 1) .withCaching(true, rows(CACHED_IMAP_UID_ROWS))) .statement(statement -> types -> statement .withPartitionKey(CassandraMessageIds.MESSAGE_ID, TIMEUUID) diff --git a/server/apps/cassandra-app/README.adoc b/server/apps/cassandra-app/README.adoc index 353148e0d22..352cce3a926 100644 --- a/server/apps/cassandra-app/README.adoc +++ b/server/apps/cassandra-app/README.adoc @@ -26,7 +26,7 @@ Third party compulsory dependencies: [source] ---- -$ docker run -d --network james -p 9042:9042 --name=cassandra cassandra:4.1.9 +$ docker run -d --network james -p 9042:9042 --name=cassandra cassandra:5.0.4 $ docker run -d --network james -p 9200:9200 --name=opensearch --env 'discovery.type=single-node' opensearchproject/opensearch:2.19.2 ---- diff --git a/server/apps/cassandra-app/docker-compose.yml b/server/apps/cassandra-app/docker-compose.yml index b162b697056..05f86823462 100644 --- a/server/apps/cassandra-app/docker-compose.yml +++ b/server/apps/cassandra-app/docker-compose.yml @@ -39,7 +39,7 @@ services: - DISABLE_SECURITY_PLUGIN=true cassandra: - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" healthcheck: diff --git a/server/apps/distributed-app/README.adoc b/server/apps/distributed-app/README.adoc index 1684f883bdc..403cf3ef400 100644 --- a/server/apps/distributed-app/README.adoc +++ b/server/apps/distributed-app/README.adoc @@ -17,7 +17,7 @@ Third party compulsory dependencies: [source] ---- -$ docker run -d --network james -p 9042:9042 --name=cassandra cassandra:4.1.9 +$ docker run -d --network james -p 9042:9042 --name=cassandra cassandra:5.0.4 $ docker run -d --network james -p 9200:9200 --name=opensearch --env 'discovery.type=single-node' --env 'DISABLE_SECURITY_PLUGIN=true' --env 'DISABLE_INSTALL_DEMO_CONFIG=true' opensearchproject/opensearch:2.19.2 $ docker run -d --network james -p 5672:5672 -p 15672:15672 --name=rabbitmq rabbitmq:4.1.1-management $ docker run -d --network james -p 8000:8000 --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 ghcr.io/scality/cloudserver:c1ba296859690c1cbbec609aaae430f6b04b4745 diff --git a/server/apps/distributed-app/docker-compose-with-pulsar.yml b/server/apps/distributed-app/docker-compose-with-pulsar.yml index 17719dee379..57636282168 100644 --- a/server/apps/distributed-app/docker-compose-with-pulsar.yml +++ b/server/apps/distributed-app/docker-compose-with-pulsar.yml @@ -48,7 +48,7 @@ services: cassandra: container_name: cassandra - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" healthcheck: diff --git a/server/apps/distributed-app/docker-compose.yml b/server/apps/distributed-app/docker-compose.yml index aef67a1a113..832ba753753 100644 --- a/server/apps/distributed-app/docker-compose.yml +++ b/server/apps/distributed-app/docker-compose.yml @@ -43,7 +43,7 @@ services: - elasticsearch cassandra: - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" healthcheck: diff --git a/server/apps/distributed-pop3-app/README.adoc b/server/apps/distributed-pop3-app/README.adoc index 48f64d1c864..496033ddf11 100644 --- a/server/apps/distributed-pop3-app/README.adoc +++ b/server/apps/distributed-pop3-app/README.adoc @@ -19,7 +19,7 @@ Third party compulsory dependencies: [source] ---- -$ docker run -d --network james -p 9042:9042 --name=cassandra cassandra:4.1.9 +$ docker run -d --network james -p 9042:9042 --name=cassandra cassandra:5.0.4 $ docker run -d --network james -p 5672:5672 -p 15672:15672 --name=rabbitmq rabbitmq:4.1.1-management $ docker run -d --network james --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 ghcr.io/scality/cloudserver:c1ba296859690c1cbbec609aaae430f6b04b4745 ---- diff --git a/server/apps/distributed-pop3-app/docker-compose.yml b/server/apps/distributed-pop3-app/docker-compose.yml index b67d8bb2c54..5ed5adedcfb 100644 --- a/server/apps/distributed-pop3-app/docker-compose.yml +++ b/server/apps/distributed-pop3-app/docker-compose.yml @@ -43,7 +43,7 @@ services: - elasticsearch cassandra: - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" healthcheck: diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobCacheDataDefinition.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobCacheDataDefinition.java index b222946419a..babcecc88a7 100644 --- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobCacheDataDefinition.java +++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobCacheDataDefinition.java @@ -34,8 +34,7 @@ public interface CassandraBlobCacheDataDefinition { .builder() .table(TABLE_NAME) .options(options -> options - .withCompaction(SchemaBuilder.sizeTieredCompactionStrategy()) - .withCompression("LZ4Compressor", 8, 1.0)) + .withCompaction(SchemaBuilder.sizeTieredCompactionStrategy())) .comment("Write through cache for small blobs stored in a slower blob store implementation.") .statement(statement -> types -> statement .withPartitionKey(ID, DataTypes.TEXT) diff --git a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/projections/CassandraMessageFastViewProjectionDataDefinition.java b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/projections/CassandraMessageFastViewProjectionDataDefinition.java index 28c7a8c9a01..9124aea37ba 100644 --- a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/projections/CassandraMessageFastViewProjectionDataDefinition.java +++ b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/projections/CassandraMessageFastViewProjectionDataDefinition.java @@ -34,8 +34,7 @@ public interface CassandraMessageFastViewProjectionDataDefinition { CassandraDataDefinition MODULE = CassandraDataDefinition.table(TABLE_NAME) .comment("Storing the JMAP projections for MessageFastView, an aggregation of JMAP properties expected to be fast to fetch.") .options(options -> options - .withCaching(true, rows(DEFAULT_CACHED_ROW_PER_PARTITION)) - .withLZ4Compression(8, 1.0)) + .withCaching(true, rows(DEFAULT_CACHED_ROW_PER_PARTITION))) .statement(statement -> types -> statement .withPartitionKey(MESSAGE_ID, DataTypes.UUID) .withColumn(PREVIEW, DataTypes.TEXT) diff --git a/server/task/task-distributed/src/test/java/org/apache/james/task/eventsourcing/distributed/DistributedTaskManagerTest.java b/server/task/task-distributed/src/test/java/org/apache/james/task/eventsourcing/distributed/DistributedTaskManagerTest.java index 607bcec9394..18146cbfa6c 100644 --- a/server/task/task-distributed/src/test/java/org/apache/james/task/eventsourcing/distributed/DistributedTaskManagerTest.java +++ b/server/task/task-distributed/src/test/java/org/apache/james/task/eventsourcing/distributed/DistributedTaskManagerTest.java @@ -667,14 +667,14 @@ public static TaskDTOModule m this.pause = pause; // Some task requires cassandra query execution upon their creation - Mono.from(session.executeReactive("SELECT dateof(now()) FROM system.local ;")) + Mono.from(session.executeReactive("SELECT toTimestamp(now()) FROM system.local ;")) .block(); } @Override public Result run() throws InterruptedException { // Task often execute Cassandra logic - Mono.from(session.executeReactive("SELECT dateof(now()) FROM system.local ;")) + Mono.from(session.executeReactive("SELECT toTimestamp(now()) FROM system.local ;")) .block(); if (pause) { @@ -692,7 +692,7 @@ public TaskType type() { @Override public Optional details() { // Some task requires cassandra query execution upon detail generation - Mono.from(session.executeReactive("SELECT dateof(now()) FROM system.local ;")) + Mono.from(session.executeReactive("SELECT toTimestamp(now()) FROM system.local ;")) .block(); return Optional.empty(); diff --git a/src/site/markdown/server/install/guice-cassandra-rabbitmq-s3.md b/src/site/markdown/server/install/guice-cassandra-rabbitmq-s3.md index 98d1880399d..fe381dfa5c9 100644 --- a/src/site/markdown/server/install/guice-cassandra-rabbitmq-s3.md +++ b/src/site/markdown/server/install/guice-cassandra-rabbitmq-s3.md @@ -47,7 +47,7 @@ $ keytool -genkey -alias james -keyalg RSA -keystore conf/keystore You need to have a Cassandra, OpenSearch, S3 and RabbitMQ instance running. You can either install the servers or launch them via docker: ```bash -$ docker run -d -p 9042:9042 --name=cassandra cassandra:4.1.9 +$ docker run -d -p 9042:9042 --name=cassandra cassandra:5.0.4 $ docker run -d --network james -p 9200:9200 --name=opensearch --env 'discovery.type=single-node' opensearchproject/opensearch:2.19.2 $ docker run -d -p 5672:5672 -p 15672:15672 --name=rabbitmq rabbitmq:3.13.3-management $ docker run -d --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 ghcr.io/scality/cloudserver:c1ba296859690c1cbbec609aaae430f6b04b4745 diff --git a/src/site/markdown/server/install/guice-cassandra.md b/src/site/markdown/server/install/guice-cassandra.md index 96b110407c7..703e849620b 100644 --- a/src/site/markdown/server/install/guice-cassandra.md +++ b/src/site/markdown/server/install/guice-cassandra.md @@ -48,7 +48,7 @@ $ keytool -genkey -alias james -keyalg RSA -keystore conf/keystore You need to have a Cassandra and an OpenSearch instance running. You can either install the servers or launch them via docker: ```bash -$ docker run -d -p 9042:9042 --name=cassandra cassandra:4.1.9 +$ docker run -d -p 9042:9042 --name=cassandra cassandra:5.0.4 $ docker run -d --network james -p 9200:9200 --name=opensearch --env 'discovery.type=single-node' opensearchproject/opensearch:2.19.2 ``` diff --git a/src/site/xdoc/server/quick-start-cassandra.xml b/src/site/xdoc/server/quick-start-cassandra.xml index 2e2b724b8dd..30f5bfa6ffd 100644 --- a/src/site/xdoc/server/quick-start-cassandra.xml +++ b/src/site/xdoc/server/quick-start-cassandra.xml @@ -64,7 +64,7 @@ Step 3: Deploy 3.1. Deploy Cassandra (optional) You may skip this part if you already have a running Cassandra on your network. -$ docker run --detach=true --name=cassandra cassandra:4.1.9 +$ docker run --detach=true --name=cassandra cassandra:5.0.4 3.2. Deploy OpenSearch (optional) You may skip this part if you already have a running OpenSearch on your network. diff --git a/third-party/clamav/docker-compose.yml b/third-party/clamav/docker-compose.yml index 776ed32ef52..2a614710b10 100644 --- a/third-party/clamav/docker-compose.yml +++ b/third-party/clamav/docker-compose.yml @@ -48,7 +48,7 @@ services: - elasticsearch cassandra: - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" healthcheck: diff --git a/third-party/rspamd/docker-compose-distributed.yml b/third-party/rspamd/docker-compose-distributed.yml index c5d879bb6d3..83c5abc04da 100644 --- a/third-party/rspamd/docker-compose-distributed.yml +++ b/third-party/rspamd/docker-compose-distributed.yml @@ -53,7 +53,7 @@ services: - elasticsearch cassandra: - image: cassandra:4.1.9 + image: cassandra:5.0.4 ports: - "9042:9042" healthcheck: diff --git a/upgrade-instructions.md b/upgrade-instructions.md index 9cfb89b8b53..abe7b09dc0d 100644 --- a/upgrade-instructions.md +++ b/upgrade-instructions.md @@ -37,6 +37,32 @@ Change list: - [JAMES-1409 Change JPARecipientRewriteTable to store separate record per target address](#james-1409-change-jparecipientrewritetable-to-store-separate-record-per-target-address) - [JAMES-4118 Cleanup message previews](#james-4118-cleanup-message-previews) - [JAMES-4128 Breaking Mailet API changes](#james-4128-breaking-mailet-api-changes) + - [JAMES-4142 Cassandra upgrade to version 5](#james-4142-cassandra-upgrade-to-version-5) + +### JAMES-4142 Cassandra upgrade to version 5 + +Date: 01/08/2025 + +JIRA: https://issues.apache.org/jira/browse/JAMES-4142 + +A breaking change was introduced in Cassandra 5.0 regarding a compression option that changed name from `chunk_length_kb` +to `chunk_length_in_kb`, which creates issues with the dedicated java driver for Cassandra. + +As such, the compression has been dropped on the following tables at creation: + +- CassandraBlobCacheDataDefinition: `blob_cache` table. +- CassandraMailboxDataDefinition: `mailbox` table. +- CassandraMessageDataDefinition: `imapUidTable` table. +- CassandraMessageFastViewProjectionDataDefinition: `message_fast_view_projection` table. + +If tables exist already and upgrade is done from Cassandra 4 to 5, nothing to do, the change in compression should be migrated. + +If you need are on a new setup and need compression on those tables, you can do this manually for each table via cqlsh with Cassandra, +like the following command: + +``` +ALTER TABLE james_keyspace.mailbox WITH compression = {'class': 'LZ4Compressor', 'chunk_length_in_kb': '8', 'crc_check_chance': '1.0'}; +``` ### JAMES-4128 Breaking Mailet API changes Date: 02/04/2025