From be50b5dccf82e11f11fbef385b97c610cd63f1fa Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Wed, 10 Jan 2024 15:32:21 +0700 Subject: [PATCH 01/81] Adapt tmail after migrating new code from James --- .../com/linagora/tmail/encrypted/EncryptedMessageManager.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/tmail-backend/mailbox/encrypted/api/src/main/scala/com/linagora/tmail/encrypted/EncryptedMessageManager.scala b/tmail-backend/mailbox/encrypted/api/src/main/scala/com/linagora/tmail/encrypted/EncryptedMessageManager.scala index deb5f19f66..90a3edb2ac 100644 --- a/tmail-backend/mailbox/encrypted/api/src/main/scala/com/linagora/tmail/encrypted/EncryptedMessageManager.scala +++ b/tmail-backend/mailbox/encrypted/api/src/main/scala/com/linagora/tmail/encrypted/EncryptedMessageManager.scala @@ -58,8 +58,6 @@ class EncryptedMessageManager @Inject()(messageManager: MessageManager, override def getMailboxEntity: Mailbox = messageManager.getMailboxEntity - override def getSupportedMessageCapabilities: util.EnumSet[MailboxManager.MessageCapabilities] = messageManager.getSupportedMessageCapabilities - override def getId: MailboxId = messageManager.getId override def getMailboxPath: MailboxPath = messageManager.getMailboxPath From cf5bc942ec3a7f1691f1062fedf7f9ad8919cfd9 Mon Sep 17 00:00:00 2001 From: vttran Date: Thu, 11 Jan 2024 11:01:24 +0700 Subject: [PATCH 02/81] ISSUE-883 Implement postgres-app: setup git submodule --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index cb3cf38a26..c5a70a2c4b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "james-project"] path = james-project url = https://github.com/apache/james-project + branch = postgresql From fc084a0578c42d6a85b4fa210c65180529d5dc9b Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Thu, 11 Jan 2024 10:55:56 +0700 Subject: [PATCH 03/81] ISSUE-883 Adapt Jenkinsfile --- Jenkinsfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b788969d7..d084e04764 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,22 +44,15 @@ pipeline { stage('Deliver Docker images') { when { anyOf { - branch 'master' + branch 'postgresql' buildingTag() } } steps { script { - env.DOCKER_TAG = 'branch-master' - if (env.TAG_NAME) { - env.DOCKER_TAG = env.TAG_NAME - } - - echo "Docker tag: ${env.DOCKER_TAG}" // build and push docker images dir("tmail-backend") { - sh 'mvn -Pci jib:build -Djib.to.auth.username=$DOCKER_HUB_CREDENTIAL_USR -Djib.to.auth.password=$DOCKER_HUB_CREDENTIAL_PSW -Djib.to.tags=distributed-$DOCKER_TAG -pl apps/distributed -X' - sh 'mvn -Pci jib:build -Djib.to.auth.username=$DOCKER_HUB_CREDENTIAL_USR -Djib.to.auth.password=$DOCKER_HUB_CREDENTIAL_PSW -Djib.to.tags=memory-$DOCKER_TAG -pl apps/memory -X' + sh "mvn -Pci jib:build -Djib.to.auth.username=${DOCKER_HUB_CREDENTIAL_USR} -Djib.to.auth.password=${DOCKER_HUB_CREDENTIAL_PSW} -Djib.to.image=linagora/tmail-backend:postgresql-experimental -pl apps/postgres -X" } } } From 34f80e1c45f5cd6e2e3ec0a1b21062c5a9502a9d Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Wed, 10 Jan 2024 17:45:00 +0700 Subject: [PATCH 04/81] ISSUE-883 Implement postgres-app --- tmail-backend/apps/pom.xml | 1 + .../postgres/docker-compose-distributed.yml | 93 ++++ .../apps/postgres/docker-compose.yml | 35 ++ tmail-backend/apps/postgres/pom.xml | 408 ++++++++++++++++++ .../batchsizes.properties | 9 + .../sample-configuration/blob.properties | 66 +++ .../distributed/blob.properties | 104 +++++ .../distributed/opensearch.properties | 101 +++++ .../distributed/rabbitmq.properties | 103 +++++ .../sample-configuration/dnsservice.xml | 27 ++ .../sample-configuration/domainlist.xml | 27 ++ .../sample-configuration/imapserver.xml | 62 +++ .../sample-configuration/jmx.properties | 26 ++ .../sample-configuration/jvm.properties | 52 +++ .../sample-configuration/listeners.xml | 33 ++ .../postgres/sample-configuration/logback.xml | 42 ++ .../sample-configuration/mailbox.properties | 1 + .../sample-configuration/mailetcontainer.xml | 145 +++++++ .../mailrepositorystore.xml | 35 ++ .../sample-configuration/postgres.properties | 20 + .../recipientrewritetable.xml | 28 ++ .../sample-configuration/search.properties | 2 + .../sample-configuration/smtpserver.xml | 104 +++++ .../sample-configuration/usersrepository.xml | 40 ++ .../sample-configuration/webadmin.properties | 54 +++ .../src/main/extensions-jars/README.md | 5 + .../postgres/src/main/glowroot/admin.json | 5 + .../src/main/glowroot/plugins/blobstore.json | 26 ++ .../src/main/glowroot/plugins/imap.json | 19 + .../src/main/glowroot/plugins/jmap.json | 19 + .../glowroot/plugins/mailboxListener.json | 19 + .../src/main/glowroot/plugins/smtp.json | 19 + .../src/main/glowroot/plugins/spooler.json | 45 ++ .../src/main/glowroot/plugins/task.json | 19 + .../james/app/PostgresTmailConfiguration.java | 235 ++++++++++ .../tmail/james/app/PostgresTmailServer.java | 228 ++++++++++ .../apps/postgres/src/main/scripts/james-cli | 3 + .../app/CombinedIdentityTmailServerTest.java | 87 ++++ .../DistributedPostgresTmailServerTest.java | 147 +++++++ .../james/app/PostgresTmailServerTest.java | 42 ++ .../src/test/resources/batchsizes.properties | 9 + .../src/test/resources/blob.properties | 66 +++ .../src/test/resources/dnsservice.xml | 25 ++ .../src/test/resources/domainlist.xml | 24 ++ .../src/test/resources/imapserver.xml | 56 +++ .../apps/postgres/src/test/resources/keystore | Bin 0 -> 2245 bytes .../postgres/src/test/resources/listeners.xml | 22 + .../src/test/resources/mailetcontainer.xml | 145 +++++++ .../test/resources/mailrepositorystore.xml | 35 ++ .../src/test/resources/rabbitmq.properties | 10 + .../src/test/resources/smtpserver.xml | 102 +++++ .../src/test/resources/webadmin.properties | 25 ++ .../combined/identity/CombinedUserDAO.java | 35 +- .../UsersRepositoryModuleChooser.java | 15 +- 54 files changed, 3088 insertions(+), 17 deletions(-) create mode 100644 tmail-backend/apps/postgres/docker-compose-distributed.yml create mode 100644 tmail-backend/apps/postgres/docker-compose.yml create mode 100644 tmail-backend/apps/postgres/pom.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/batchsizes.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/blob.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/distributed/blob.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/distributed/opensearch.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/distributed/rabbitmq.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/dnsservice.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/domainlist.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/imapserver.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/jmx.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/jvm.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/listeners.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/logback.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/mailbox.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/mailetcontainer.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/mailrepositorystore.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/postgres.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/recipientrewritetable.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/search.properties create mode 100644 tmail-backend/apps/postgres/sample-configuration/smtpserver.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/usersrepository.xml create mode 100644 tmail-backend/apps/postgres/sample-configuration/webadmin.properties create mode 100644 tmail-backend/apps/postgres/src/main/extensions-jars/README.md create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/admin.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/blobstore.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/imap.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/jmap.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/mailboxListener.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/smtp.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/spooler.json create mode 100644 tmail-backend/apps/postgres/src/main/glowroot/plugins/task.json create mode 100644 tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java create mode 100644 tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java create mode 100755 tmail-backend/apps/postgres/src/main/scripts/james-cli create mode 100644 tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java create mode 100644 tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/DistributedPostgresTmailServerTest.java create mode 100644 tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java create mode 100644 tmail-backend/apps/postgres/src/test/resources/batchsizes.properties create mode 100644 tmail-backend/apps/postgres/src/test/resources/blob.properties create mode 100644 tmail-backend/apps/postgres/src/test/resources/dnsservice.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/domainlist.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/imapserver.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/keystore create mode 100644 tmail-backend/apps/postgres/src/test/resources/listeners.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/mailrepositorystore.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/rabbitmq.properties create mode 100644 tmail-backend/apps/postgres/src/test/resources/smtpserver.xml create mode 100644 tmail-backend/apps/postgres/src/test/resources/webadmin.properties diff --git a/tmail-backend/apps/pom.xml b/tmail-backend/apps/pom.xml index 606ac7c2ae..33c6195116 100644 --- a/tmail-backend/apps/pom.xml +++ b/tmail-backend/apps/pom.xml @@ -16,5 +16,6 @@ memory distributed + postgres \ No newline at end of file diff --git a/tmail-backend/apps/postgres/docker-compose-distributed.yml b/tmail-backend/apps/postgres/docker-compose-distributed.yml new file mode 100644 index 0000000000..a6ee335375 --- /dev/null +++ b/tmail-backend/apps/postgres/docker-compose-distributed.yml @@ -0,0 +1,93 @@ +version: '3.4' + +x-common-environment: &common-environment + POSTGRES_DB: tmail + POSTGRES_USER: tmail + POSTGRES_PASSWORD: secret1 + +services: + + tmail-backend: + depends_on: + postgres: + condition: service_started + opensearch: + condition: service_healthy + s3: + condition: service_started + rabbitmq: + condition: service_started + image: linagora/tmail-backend-postgresql-experimental + container_name: tmail-backend + hostname: tmail-backend.local + command: + - --generate-keystore + environment: + <<: *common-environment + OBJECT_STORAGE_ENDPOINT: http://s3.docker.test:8000/ + OBJECT_STORAGE_REGION: us-east-1 + OBJECT_STORAGE_ACCESS_KEY_ID: accessKey1 + OBJECT_STORAGE_SECRET_KEY: secretKey1 + + ports: + - "80:80" + - "25:25" + - "110:110" + - "143:143" + - "465:465" + - "587:587" + - "993:993" + - "8000:8000" + volumes: + - ./sample-configuration/distributed/opensearch.properties:/root/conf/opensearch.properties + - ./sample-configuration/distributed/blob.properties:/root/conf/blob.properties + - ./sample-configuration/distributed/rabbitmq.properties:/root/conf/rabbitmq.properties + networks: + - tmail + + opensearch: + image: opensearchproject/opensearch:2.8.0 + container_name: opensearch + healthcheck: + test: ["CMD", "curl", "-s", "http://opensearch:9200"] + interval: 3s + timeout: 10s + retries: 5 + environment: + - discovery.type=single-node + - DISABLE_INSTALL_DEMO_CONFIG=true + - DISABLE_SECURITY_PLUGIN=true + networks: + - tmail + + postgres: + image: postgres:16.1 + container_name: postgres + ports: + - "5432:5432" + environment: + <<: *common-environment + networks: + - tmail + + s3: + image: registry.scality.com/cloudserver/cloudserver:8.7.25 + container_name: s3.docker.test + environment: + SCALITY_ACCESS_KEY_ID: accessKey1 + SCALITY_SECRET_ACCESS_KEY: secretKey1 + LOG_LEVEL: trace + REMOTE_MANAGEMENT_DISABLE: 1 + networks: + - tmail + + rabbitmq: + image: rabbitmq:3.12.1-management + ports: + - "5672:5672" + - "15672:15672" + networks: + - tmail + +networks: + tmail: \ No newline at end of file diff --git a/tmail-backend/apps/postgres/docker-compose.yml b/tmail-backend/apps/postgres/docker-compose.yml new file mode 100644 index 0000000000..97747ecaa6 --- /dev/null +++ b/tmail-backend/apps/postgres/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3.4' + +x-common-environment: &common-environment + POSTGRES_DB: tmail + POSTGRES_USER: tmail + POSTGRES_PASSWORD: secret1 + +services: + + tmail-backend: + depends_on: + - postgres + image: linagora/tmail-backend-postgresql-experimental + container_name: tmail-backend + hostname: tmail-backend.local + command: + - --generate-keystore + ports: + - "80:80" + - "25:25" + - "110:110" + - "143:143" + - "465:465" + - "587:587" + - "993:993" + - "8000:8000" + environment: + <<: *common-environment + + postgres: + image: postgres:16.1 + ports: + - "5432:5432" + environment: + <<: *common-environment \ No newline at end of file diff --git a/tmail-backend/apps/postgres/pom.xml b/tmail-backend/apps/postgres/pom.xml new file mode 100644 index 0000000000..030466bc84 --- /dev/null +++ b/tmail-backend/apps/postgres/pom.xml @@ -0,0 +1,408 @@ + + + 4.0.0 + + com.linagora.tmail + apps + 1.0.0-SNAPSHOT + + + postgres + Team-mail :: Apps :: Postgres + + + + ${james.groupId} + blob-s3 + test-jar + test + + + ${project.groupId} + combined-identity + ${project.version} + + + ${project.groupId} + combined-identity + ${project.version} + test-jar + + + ${project.groupId} + tmail-guice-distributed + + + ${james.groupId} + apache-james-backends-rabbitmq + test-jar + test + + + ${james.groupId} + james-server-data-ldap + test-jar + test + + + ${james.groupId} + blob-s3-guice + test-jar + test + + + ${james.groupId} + apache-james-mailbox-opensearch + test-jar + test + + + ${james.groupId} + james-server-guice-opensearch + test-jar + test + + + ${james.groupId} + james-server-guice-opensearch + + + ${james.groupId} + apache-james-backends-opensearch + test-jar + test + + + ${james.groupId} + james-server-postgres-app + test-jar + test + + + ${james.groupId} + apache-james-backends-postgres + test-jar + test + + + ${james.groupId} + apache-james-mailbox-postgres + test-jar + test + + + ${james.groupId} + james-server-data-postgres + + + ${james.groupId} + james-server-guice-mailbox-postgres + + + ${james.groupId} + james-server-guice-sieve-postgres + + + ${james.groupId} + james-server-postgres-common-guice + + + ${james.groupId} + james-server-postgres-common-guice + test-jar + test + + + ${james.groupId} + james-server-rate-limiter-redis + + + org.testcontainers + postgresql + 1.19.1 + test + + + + ${james.groupId} + apache-mime4j-dom + + + + ${project.groupId} + encrypted-mailbox-guice + + + ${project.groupId} + jmap-extensions + + + ${project.groupId} + jmap-extensions + test-jar + test + + + ${project.groupId} + smtp-extensions + + + ${project.groupId} + team-mailboxes + + + ${project.groupId} + team-mailboxes-guice + + + ${project.groupId} + tmail-guice-jmap + test-jar + test + + + ${project.groupId} + tmail-mailets + + + ${project.groupId} + tmail-webadmin-mailbox + + + ${project.groupId} + webadmin-email-address-contact + + + ${project.groupId} + webadmin-rate-limit + + + ${james.groupId} + apache-james-linshare + test-jar + test + + + ${project.groupId} + tmail-webadmin-team-mailboxes + + + ${james.groupId} + apache-mailet-icalendar + + + ${james.groupId} + blob-export-guice + test-jar + test + + + ${james.groupId} + jmap-rfc-8621-integration-tests-common + test + + + ${james.groupId} + james-server-cli + runtime + + + ${james.groupId} + james-server-guice-common + test-jar + test + + + ${james.groupId} + james-server-guice-jmap + test-jar + test + + + ${james.groupId} + james-server-rate-limiter + + + ${james.groupId} + james-server-testing + test + + + ${james.groupId} + james-server-webadmin-core + test-jar + test + + + ch.qos.logback.contrib + logback-json-classic + + + org.apache.james + james-server-postgres-app + + + com.linagora.tmail + combined-identity + test + + + + + + + com.googlecode.maven-download-plugin + download-maven-plugin + + + install-glowroot + + wget + + package + + + https://github.com/glowroot/glowroot/releases/download/v0.14.0/glowroot-0.14.0-dist.zip + + true + ${project.build.directory} + 16073f10204751cd71d3b4ea93be2649 + + + + package-async-profiler + + wget + + package + + + https://github.com/async-profiler/async-profiler/releases/download/v2.9/async-profiler-2.9-linux-x64.tar.gz + + true + ${project.build.directory} + 29127cee36b7acf069d31603b4558361 + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-glowroot-resources + + copy-resources + + package + + ${basedir}/target/glowroot + + + src/main/glowroot + true + + + + + + + + com.google.cloud.tools + jib-maven-plugin + + + ${jib.base.image} + + + linagora/tmail-backend-postgresql-experimental + + latest + + + + com.linagora.tmail.james.app.PostgresTmailServer + + 80 + + 143 + + 993 + + 25 + + 465 + + 587 + + 4000 + + 4190 + + 8000 + + + /root + USE_CURRENT_TIMESTAMP + + + + + sample-configuration + /root/conf + + + src/main/scripts + /usr/bin + + + target/glowroot + /root/glowroot + + + target/async-profiler-2.9-linux-x64 + /root/async-profiler + + + src/main/extensions-jars + /root/extensions-jars + + + + + /usr/bin/james-cli + 755 + + + + /root/async-profiler/profiler.sh + 755 + + + + /root/async-profiler/build/libasyncProfiler.so + 755 + + + + /root/async-profiler/build/jattach + 755 + + + + + + + + + buildTar + + package + + + + + + diff --git a/tmail-backend/apps/postgres/sample-configuration/batchsizes.properties b/tmail-backend/apps/postgres/sample-configuration/batchsizes.properties new file mode 100644 index 0000000000..1784f95d79 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/batchsizes.properties @@ -0,0 +1,9 @@ +# Those properties let you configure the number of messages queried at the same time. +# IMAP FETCH command +fetch.metadata=200 +fetch.headers=200 +fetch.full=50 +# IMAP COPY command +copy=100 +# IMAP MOVE command +move=100 diff --git a/tmail-backend/apps/postgres/sample-configuration/blob.properties b/tmail-backend/apps/postgres/sample-configuration/blob.properties new file mode 100644 index 0000000000..78ea935448 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/blob.properties @@ -0,0 +1,66 @@ +# ============================================= BlobStore Implementation ================================== +# Read https://james.apache.org/server/config-blobstore.html for further details + +# Choose your BlobStore implementation +# Mandatory, allowed values are: file, s3, postgres. +implementation=postgres + +# ========================================= Deduplication ======================================== +# If you choose to enable deduplication, the mails with the same content will be stored only once. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to the deletion of all +# the mails sharing the same content once one is deleted. +# Mandatory, Allowed values are: true, false +deduplication.enable=true + +# deduplication.family needs to be incremented every time the deduplication.generation.duration is changed +# Positive integer, defaults to 1 +# deduplication.gc.generation.family=1 + +# Duration of generation. +# Deduplication only takes place within a singe generation. +# Only items two generation old can be garbage collected. (This prevent concurrent insertions issues and +# accounts for a clock skew). +# deduplication.family needs to be incremented everytime this parameter is changed. +# Duration. Default unit: days. Defaults to 30 days. +# deduplication.gc.generation.duration=30days + +# ========================================= Encryption ======================================== +# If you choose to enable encryption, the blob content will be encrypted before storing them in the BlobStore. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to all content being +# encrypted. This comes at a performance impact but presents you from leaking data if, for instance the third party +# offering you a S3 service is compromised. +# Optional, Allowed values are: true, false, defaults to false +encryption.aes.enable=false + +# Mandatory (if AES encryption is enabled) salt and password. Salt needs to be an hexadecimal encoded string +#encryption.aes.password=xxx +#encryption.aes.salt=73616c7479 +# Optional, defaults to PBKDF2WithHmacSHA512 +#encryption.aes.private.key.algorithm=PBKDF2WithHmacSHA512 + +# ============================================ Blobs Exporting ============================================== +# Read https://james.apache.org/server/config-blob-export.html for further details + +# Choosing blob exporting mechanism, allowed mechanism are: localFile, linshare +# LinShare is a file sharing service, will be explained in the below section +# Optional, default is localFile +blob.export.implementation=localFile + +# ======================================= Local File Blobs Exporting ======================================== +# Optional, directory to store exported blob, directory path follows James file system format +# default is file://var/blobExporting +blob.export.localFile.directory=file://var/blobExporting + +# ======================================= LinShare File Blobs Exporting ======================================== +# LinShare is a sharing service where you can use james, connects to an existing LinShare server and shares files to +# other mail addresses as long as those addresses available in LinShare. For example you can deploy James and LinShare +# sharing the same LDAP repository +# Mandatory if you choose LinShare, url to connect to LinShare service +# blob.export.linshare.url=http://linshare:8080 + +# ======================================= LinShare Configuration BasicAuthentication =================================== +# Authentication is mandatory if you choose LinShare, TechnicalAccount is need to connect to LinShare specific service. +# For Example: It will be formalized to 'Authorization: Basic {Credential of UUID/password}' + +# blob.export.linshare.technical.account.uuid=Technical_Account_UUID +# blob.export.linshare.technical.account.password=password \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/distributed/blob.properties b/tmail-backend/apps/postgres/sample-configuration/distributed/blob.properties new file mode 100644 index 0000000000..a02854b231 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/distributed/blob.properties @@ -0,0 +1,104 @@ +# ============================================= BlobStore Implementation ================================== +# Read https://james.apache.org/server/config-blobstore.html for further details + +# Choose your BlobStore implementation +# Mandatory, allowed values are: file, s3, postgres. +implementation=s3 + +# ========================================= Deduplication ======================================== +# If you choose to enable deduplication, the mails with the same content will be stored only once. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to the deletion of all +# the mails sharing the same content once one is deleted. +# Mandatory, Allowed values are: true, false +deduplication.enable=true + +# deduplication.family needs to be incremented every time the deduplication.generation.duration is changed +# Positive integer, defaults to 1 +# deduplication.gc.generation.family=1 + +# Duration of generation. +# Deduplication only takes place within a singe generation. +# Only items two generation old can be garbage collected. (This prevent concurrent insertions issues and +# accounts for a clock skew). +# deduplication.family needs to be incremented everytime this parameter is changed. +# Duration. Default unit: days. Defaults to 30 days. +# deduplication.gc.generation.duration=30days + +# ========================================= Encryption ======================================== +# If you choose to enable encryption, the blob content will be encrypted before storing them in the BlobStore. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to all content being +# encrypted. This comes at a performance impact but presents you from leaking data if, for instance the third party +# offering you a S3 service is compromised. +# Optional, Allowed values are: true, false, defaults to false +encryption.aes.enable=false + +# Mandatory (if AES encryption is enabled) salt and password. Salt needs to be an hexadecimal encoded string +#encryption.aes.password=xxx +#encryption.aes.salt=73616c7479 +# Optional, defaults to PBKDF2WithHmacSHA512 +#encryption.aes.private.key.algorithm=PBKDF2WithHmacSHA512 + +# ============================================== ObjectStorage ============================================ + +# ========================================= ObjectStorage Buckets ========================================== +# bucket names prefix +# Optional, default no prefix +# objectstorage.bucketPrefix=prod- + +# Default bucket name +# Optional, default is bucketPrefix + `default` +# objectstorage.namespace=james + +# ========================================= ObjectStorage on S3 ============================================= +# Mandatory if you choose s3 storage service, S3 authentication endpoint +objectstorage.s3.endPoint=${env:OBJECT_STORAGE_ENDPOINT:-http://s3.docker.test:8000/} + +# Mandatory if you choose s3 storage service, S3 region +#objectstorage.s3.region=eu-west-1 +objectstorage.s3.region=${env:OBJECT_STORAGE_REGION:-eu-west-1} + +# Mandatory if you choose aws-s3 storage service, access key id configured in S3 +objectstorage.s3.accessKeyId=${env:OBJECT_STORAGE_ACCESS_KEY_ID:-accessKey1} + +# Mandatory if you choose s3 storage service, secret key configured in S3 +objectstorage.s3.secretKey=${env:OBJECT_STORAGE_SECRET_KEY:-secretKey1} + +# Optional if you choose s3 storage service: The trust store file, secret, and algorithm to use +# when connecting to the storage service. If not specified falls back to Java defaults. +#objectstorage.s3.truststore.path= +#objectstorage.s3.truststore.type=JKS +#objectstorage.s3.truststore.secret= +#objectstorage.s3.truststore.algorithm=SunX509 + + +# optional: Object read in memory will be rejected if they exceed the size limit exposed here. Size, exemple `100M`. +# Supported units: K, M, G, defaults to B if no unit is specified. If unspecified, big object won't be prevented +# from being loaded in memory. This settings complements protocol limits. +# objectstorage.s3.in.read.limit=50M + +# ============================================ Blobs Exporting ============================================== +# Read https://james.apache.org/server/config-blob-export.html for further details + +# Choosing blob exporting mechanism, allowed mechanism are: localFile, linshare +# LinShare is a file sharing service, will be explained in the below section +# Optional, default is localFile +blob.export.implementation=localFile + +# ======================================= Local File Blobs Exporting ======================================== +# Optional, directory to store exported blob, directory path follows James file system format +# default is file://var/blobExporting +blob.export.localFile.directory=file://var/blobExporting + +# ======================================= LinShare File Blobs Exporting ======================================== +# LinShare is a sharing service where you can use james, connects to an existing LinShare server and shares files to +# other mail addresses as long as those addresses available in LinShare. For example you can deploy James and LinShare +# sharing the same LDAP repository +# Mandatory if you choose LinShare, url to connect to LinShare service +# blob.export.linshare.url=http://linshare:8080 + +# ======================================= LinShare Configuration BasicAuthentication =================================== +# Authentication is mandatory if you choose LinShare, TechnicalAccount is need to connect to LinShare specific service. +# For Example: It will be formalized to 'Authorization: Basic {Credential of UUID/password}' + +# blob.export.linshare.technical.account.uuid=Technical_Account_UUID +# blob.export.linshare.technical.account.password=password diff --git a/tmail-backend/apps/postgres/sample-configuration/distributed/opensearch.properties b/tmail-backend/apps/postgres/sample-configuration/distributed/opensearch.properties new file mode 100644 index 0000000000..df261c5dee --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/distributed/opensearch.properties @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This template file can be used as example for James Server configuration +# DO NOT USE IT AS SUCH AND ADAPT IT TO YOUR NEEDS + +# Configuration file for OpenSearch +# Read https://james.apache.org/server/config-opensearch.html for further details + +opensearch.masterHost=opensearch +opensearch.port=9200 + +# Optional. Only http or https are accepted, default is http +# opensearch.hostScheme=http + +# Optional, default is `default` +# Choosing the SSL check strategy when using https scheme +# default: Use the default SSL TrustStore of the system. +# ignore: Ignore SSL Validation check (not recommended). +# override: Override the SSL Context to use a custom TrustStore containing ES server's certificate. +# opensearch.hostScheme.https.sslValidationStrategy=default + +# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy +# Configure OpenSearch rest client to use this trustStore file to recognize nginx's ssl certificate. +# You need to specify both trustStorePath and trustStorePassword +# opensearch.hostScheme.https.trustStorePath=/file/to/trust/keystore.jks + +# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy +# Configure OpenSearch rest client to use this trustStore file with the specified password. +# You need to specify both trustStorePath and trustStorePassword +# opensearch.hostScheme.https.trustStorePassword=myJKSPassword + +# Optional. default is `default` +# Configure OpenSearch rest client to use host name verifier during SSL handshake +# default: using the default hostname verifier provided by apache http client. +# accept_any_hostname: accept any host (not recommended). +# opensearch.hostScheme.https.hostNameVerifier=default + +# Optional. +# Basic auth username to access opensearch. +# Ignore opensearch.user and opensearch.password to not be using authentication (default behaviour). +# Otherwise, you need to specify both properties. +# opensearch.user=elasticsearch + +# Optional. +# Basic auth password to access opensearch. +# Ignore opensearch.user and opensearch.password to not be using authentication (default behaviour). +# Otherwise, you need to specify both properties. +# opensearch.password=secret + +# You can alternatively provide a list of hosts following this format : +# opensearch.hosts=host1:9200,host2:9200 +# opensearch.clusterName=cluster + +opensearch.nb.shards=5 +opensearch.nb.replica=1 +opensearch.index.waitForActiveShards=1 +opensearch.retryConnection.maxRetries=7 +opensearch.retryConnection.minDelay=3000 +# Index or not attachments (default value: true) +# Note: Attachments not implemented yet for postgresql, false for now +opensearch.indexAttachments=false + +# Search overrides allow resolution of predefined search queries against alternative sources of data +# and allow bypassing opensearch. This is useful to handle most resynchronisation queries that +# are simple enough to be resolved against Cassandra. +# +# Possible values are: +# - `org.apache.james.mailbox.postgres.search.AllSearchOverride` Some IMAP clients uses SEARCH ALL to fully list messages in +# a mailbox and detect deletions. This is typically done by clients not supporting QRESYNC and from an IMAP perspective +# is considered an optimisation as less data is transmitted compared to a FETCH command. Resolving such requests against +# Postgresql is enabled by this search override and likely desirable. +# - `org.apache.james.mailbox.postgres.search.UidSearchOverride`. Same as above but restricted by ranges. +# - `org.apache.james.mailbox.postgres.search.DeletedSearchOverride`. Find deleted messages by looking up in the relevant Postgresql +# table. +# - `org.apache.james.mailbox.postgres.search.DeletedWithRangeSearchOverride`. Same as above but limited by ranges. +# - `org.apache.james.mailbox.postgres.search.NotDeletedWithRangeSearchOverride`. List non deleted messages in a given range. +# Lists all messages and filters out deleted message thus this is based on the following heuristic: most messages are not marked as deleted. +# - `org.apache.james.mailbox.postgres.search.UnseenSearchOverride`. List unseen messages in the corresponding Postgresql index. +# +# Please note that custom overrides can be defined here. +# +# opensearch.search.overrides=org.apache.james.mailbox.postgres.search.AllSearchOverride,org.apache.james.mailbox.postgres.search.DeletedSearchOverride, org.apache.james.mailbox.postgres.search.DeletedWithRangeSearchOverride,org.apache.james.mailbox.postgres.search.NotDeletedWithRangeSearchOverride,org.apache.james.mailbox.postgres.search.UidSearchOverride,org.apache.james.mailbox.postgres.search.UnseenSearchOverride + +# Optional. Default is `false` +# When set to true, James will attempt to reindex from the indexed message when moved. If the message is not found, it will fall back to the old behavior (The message will be indexed from the blobStore source) +# opensearch.message.index.optimize.move=false \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/distributed/rabbitmq.properties b/tmail-backend/apps/postgres/sample-configuration/distributed/rabbitmq.properties new file mode 100644 index 0000000000..c8b3f52602 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/distributed/rabbitmq.properties @@ -0,0 +1,103 @@ +# RabbitMQ configuration + +# Read https://james.apache.org/server/config-rabbitmq.html for further details + +# Mandatory +uri=amqp://rabbitmq:5672 +# If you use a vhost, specify it as well at the end of the URI +# uri=amqp://rabbitmq:5672/vhost + +# Vhost to use for creating queues and exchanges +# Optional, only use this if you have invalid URIs containing characters like '_' +# vhost=vhost1 + +# Optional, default to the host specified as part of the URI. +# Allow creating cluster aware connections. +# hosts=ip1:5672,ip2:5672 + +# RabbitMQ Administration Management +# Mandatory +management.uri=http://rabbitmq:15672 +# Mandatory +management.user=guest +# Mandatory +management.password=guest + +# Configure retries count to retrieve a connection. Exponential backoff is performed between each retries. +# Optional integer, defaults to 10 +#connection.pool.retries=10 +# Configure initial duration (in ms) between two connection retries. Exponential backoff is performed between each retries. +# Optional integer, defaults to 100 +#connection.pool.min.delay.ms=100 +# Configure retries count to retrieve a channel. Exponential backoff is performed between each retries. +# Optional integer, defaults to 3 +#channel.pool.retries=3 +# Configure timeout duration (in ms) to obtain a rabbitmq channel. Defaults to 30 seconds. +# Optional integer, defaults to 30 seconds. +#channel.pool.max.delay.ms=30000 +# Configure the size of the channel pool. +# Optional integer, defaults to 3 +#channel.pool.size=3 + +# Boolean. Whether to activate Quorum queue usage for use cases that benefits from it (work queue). +# Quorum queues enables high availability. +# False (default value) results in the usage of classic queues. +#quorum.queues.enable=true + +# Strictly positive integer. The replication factor to use when creating quorum queues. +#quorum.queues.replication.factor + +# Parameters for the Cassandra administrative view + +# Whether the Cassandra administrative view should be activated. Boolean value defaulting to true. +# Not necessarily needed for MDA deployments, mail queue management adds significant complexity. +# cassandra.view.enabled=true + +# Period of the window. Too large values will lead to wide rows while too little values might lead to many queries. +# Use the number of mail per Cassandra row, along with your expected traffic, to determine this value +# This value can only be decreased to a value dividing the current value +# Optional, default 1h +mailqueue.view.sliceWindow=1h + +# Use to distribute the emails of a given slice within your cassandra cluster +# A good value is 2*cassandraNodeCount +# This parameter can only be increased. +# Optional, default 1 +mailqueue.view.bucketCount=1 + +# Determine the probability to update the browse start pointer +# Too little value will lead to unnecessary reads. Too big value will lead to more expensive browse. +# Choose this parameter so that it get's update one time every one-two sliceWindow +# Optional, default 1000 +mailqueue.view.updateBrowseStartPace=1000 + +# Enables or disables the gauge metric on the mail queue size +# Computing the size of the mail queue is currently implemented on top of browse operation and thus have a linear complexity +# Metrics get exported periodically as configured in opensearch.properties, thus getSize is also called periodically +# Choose to disable it when the mail queue size is getting too big +# Note that this is as well a temporary workaround until we get 'getSize' method better optimized +# Optional, default false +mailqueue.size.metricsEnabled=false + +# Whether to enable task consumption on this node. Tasks are WebAdmin triggered long running jobs. +# Disable with caution (this only makes sense in a distributed setup where other nodes consume tasks). +# Defaults to true. +task.consumption.enabled=true + +# Configure task queue consumer timeout. References: https://www.rabbitmq.com/consumers.html#acknowledgement-timeout. Required at least RabbitMQ version 3.12 to have effect. +# This is used to avoid the task queue consumer (which could run very long tasks) being disconnected by RabbitMQ after the default acknowledgement timeout 30 minutes. +# Optional. Duration (support multiple time units cf `DurationParser`), defaults to 1 day. +#task.queue.consumer.timeout=1day + +# Configure queue ttl (in ms). References: https://www.rabbitmq.com/ttl.html#queue-ttl. +# This is used only on queues used to share notification patterns, are exclusive to a node. If omitted, it will not add the TTL configure when declaring queues. +# Optional integer, defaults is 3600000. +#notification.queue.ttl=3600000 + +# AMQP resources parameters for the subscriber email address contact messages. In order to synchronize contacts between Team-mail backend and 3rd. +# AQMP uri +address.contact.uri=amqp://rabbitmq:5672 +address.contact.user=guest +address.contact.password=guest +# Queue name +address.contact.queue=AddressContactQueue1 \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/dnsservice.xml b/tmail-backend/apps/postgres/sample-configuration/dnsservice.xml new file mode 100644 index 0000000000..1c4fb9edc7 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/dnsservice.xml @@ -0,0 +1,27 @@ + + + + + + + true + false + 50000 + diff --git a/tmail-backend/apps/postgres/sample-configuration/domainlist.xml b/tmail-backend/apps/postgres/sample-configuration/domainlist.xml new file mode 100644 index 0000000000..605439fbd0 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/domainlist.xml @@ -0,0 +1,27 @@ + + + + + + + false + false + localhost + diff --git a/tmail-backend/apps/postgres/sample-configuration/imapserver.xml b/tmail-backend/apps/postgres/sample-configuration/imapserver.xml new file mode 100644 index 0000000000..e9208cff97 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/imapserver.xml @@ -0,0 +1,62 @@ + + + + + + + + + imapserver + 0.0.0.0:143 + 200 + + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + 120 + SECONDS + true + true + + + imapserver-ssl + 0.0.0.0:993 + 200 + + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + 120 + SECONDS + true + + diff --git a/tmail-backend/apps/postgres/sample-configuration/jmx.properties b/tmail-backend/apps/postgres/sample-configuration/jmx.properties new file mode 100644 index 0000000000..e56235f9b4 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/jmx.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This template file can be used as example for James Server configuration +# DO NOT USE IT AS SUCH AND ADAPT IT TO YOUR NEEDS + +# Read https://james.apache.org/server/config-system.html#jmx.properties for further details + +jmx.enabled=true +jmx.address=127.0.0.1 +jmx.port=9999 diff --git a/tmail-backend/apps/postgres/sample-configuration/jvm.properties b/tmail-backend/apps/postgres/sample-configuration/jvm.properties new file mode 100644 index 0000000000..e72ce63c6c --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/jvm.properties @@ -0,0 +1,52 @@ +# ============================================= Extra JVM System Properties =========================================== +# To avoid clutter on the command line, any properties in this file will be added as system properties on server start. + +# Example: If you need an option -Dmy.property=whatever, you can instead add it here as +# my.property=whatever + +# (Optional). String (size, integer + size units, example: `12 KIB`, supported units are bytes KIB MIB GIB TIB). Defaults to 100KIB. +# This governs the threshold MimeMessageInputStreamSource relies on for storing MimeMessage content on disk. +# Below, data is stored in memory. Above data is stored on disk. +# Lower values will lead to longer processing time but will minimize heap memory usage. Modern SSD hardware +# should however support a high throughput. Higher values will lead to faster single mail processing at the cost +# of higher heap usage. +#james.message.memory.threshold=12K + +# Optional. Boolean. Defaults to false. Recommended value is false. +# Should MimeMessageWrapper use a copy of the message in memory? Or should bigger message exceeding james.message.memory.threshold +# be copied to temporary files? +#james.message.usememorycopy=false + +# Mode level of resource leak detection. It is used to detect a resource not be disposed of before it's garbage-collected. +# Example `MimeMessageInputStreamSource` +# Optional. Allowed values are: none, simple, advanced, testing +# - none: Disables resource leak detection. +# - simple: Enables output a simplistic error log if a leak is encountered and would free the resources (default). +# - advanced: Enables output an advanced error log implying the place of allocation of the underlying object and would free resources. +# - testing: Enables output an advanced error log implying the place of allocation of the underlying object and rethrow an error, that action is being taken by the development team. +#james.lifecycle.leak.detection.mode=simple + +# Should we add the host in the MDC logging context for incoming IMAP, SMTP, POP3? Doing so, a DNS resolution +# is attempted for each incoming connection, which can be costly. Remote IP is always added to the logging context. +# Optional. Boolean. Defaults to true. +#james.protocols.mdc.hostname=true + +# Manage netty leak detection level see https://netty.io/wiki/reference-counted-objects.html#leak-detection-levels +# io.netty.leakDetection.level=SIMPLE + +# Should James exit on Startup error? Boolean, defaults to true. This prevents partial startup. +# james.exit.on.startup.error=true + +# Fails explicitly on missing configuration file rather that taking implicit values. Defautls to false. +# james.fail.on.missing.configuration=true + +# JMX, when enable causes RMI to plan System.gc every hour. Set this instead to once every 1000h. +sun.rmi.dgc.server.gcInterval=3600000000 +sun.rmi.dgc.client.gcInterval=3600000000 + +# Automatically generate a JMX password upon start. CLI is able to retrieve this password. +james.jmx.credential.generation=true + +# Disable Remote Code Execution feature from JMX +# CF https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.management/share/classes/com/sun/jmx/remote/security/MBeanServerAccessController.java#L646 +jmx.remote.x.mlet.allow.getMBeansFromURL=false diff --git a/tmail-backend/apps/postgres/sample-configuration/listeners.xml b/tmail-backend/apps/postgres/sample-configuration/listeners.xml new file mode 100644 index 0000000000..9e24cb47ed --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/listeners.xml @@ -0,0 +1,33 @@ + + + + + + + true + + + + \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/logback.xml b/tmail-backend/apps/postgres/sample-configuration/logback.xml new file mode 100644 index 0000000000..6f386173fe --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/logback.xml @@ -0,0 +1,42 @@ + + + + + true + + + + + %d{HH:mm:ss.SSS} %highlight([%-5level]) %logger{15} - %msg%n%rEx + false + + + + + /logs/james.log + + /logs/james.%i.log.tar.gz + 1 + 3 + + + + 100MB + + + + %d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx + false + + + + + + + + + + + + + diff --git a/tmail-backend/apps/postgres/sample-configuration/mailbox.properties b/tmail-backend/apps/postgres/sample-configuration/mailbox.properties new file mode 100644 index 0000000000..4f3fd78713 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/mailbox.properties @@ -0,0 +1 @@ +gpg.encryption.enable=false \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/mailetcontainer.xml b/tmail-backend/apps/postgres/sample-configuration/mailetcontainer.xml new file mode 100644 index 0000000000..c899874a42 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/mailetcontainer.xml @@ -0,0 +1,145 @@ + + + + + + + + + + + postmaster + + + + 20 + postgres://var/mail/error/ + + + + + + + postgres://var/mail/relay-limit-exceeded/ + + + transport + + + + + + mailetContainerErrors + + + ignore + + + postgres://var/mail/error/ + propagate + + + + + + + + + + + + bcc + ignore + + + rrt-error + + + + + + local-address-error + 550 - Requested action not taken: no such user here + + + + relay + + + + + + outgoing + 5000, 100000, 500000 + 3 + 0 + 10 + true + bounces + + + + + + mailetContainerLocalAddressError + + + none + + + postgres://var/mail/address-error/ + + + + + + mailetContainerRelayDenied + + + none + + + postgres://var/mail/relay-denied/ + Warning: You are sending an e-mail to a remote server. You must be authenticated to perform such an operation + + + + + + bounces + + + false + + + + + + postgres://var/mail/rrt-error/ + true + + + + + + + + \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/mailrepositorystore.xml b/tmail-backend/apps/postgres/sample-configuration/mailrepositorystore.xml new file mode 100644 index 0000000000..445f2727f2 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/mailrepositorystore.xml @@ -0,0 +1,35 @@ + + + + + + + + postgres + + + + postgres + + + + + + diff --git a/tmail-backend/apps/postgres/sample-configuration/postgres.properties b/tmail-backend/apps/postgres/sample-configuration/postgres.properties new file mode 100644 index 0000000000..a32efa6718 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/postgres.properties @@ -0,0 +1,20 @@ +# String. Optional, default to 'postgres'. Database name. +database.name=${env:POSTGRES_DB:-postgres} + +# String. Optional, default to 'public'. Database schema. +database.schema=${env:POSTGRES_SCHEMA:-public} + +# String. Optional, default to 'localhost'. Database host. +database.host=${env:POSTGRES_HOST:-postgres} + +# Integer. Optional, default to 5432. Database port. +database.port=${env:POSTGRES_PORT:-5432} + +# String. Required. Database username. +database.username=${env:POSTGRES_USER:-tmail} + +# String. Required. Database password of the user. +database.password=${env:POSTGRES_PASSWORD:-secret1} + +# Boolean. Optional, default to false. Whether to enable row level security. +row.level.security.enabled=false diff --git a/tmail-backend/apps/postgres/sample-configuration/recipientrewritetable.xml b/tmail-backend/apps/postgres/sample-configuration/recipientrewritetable.xml new file mode 100644 index 0000000000..1a512c6035 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/recipientrewritetable.xml @@ -0,0 +1,28 @@ + + + + + + + + true + 10 + + diff --git a/tmail-backend/apps/postgres/sample-configuration/search.properties b/tmail-backend/apps/postgres/sample-configuration/search.properties new file mode 100644 index 0000000000..51833746a9 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/search.properties @@ -0,0 +1,2 @@ +# not for production purposes. To be replaced by PG based search. +implementation=scanning \ No newline at end of file diff --git a/tmail-backend/apps/postgres/sample-configuration/smtpserver.xml b/tmail-backend/apps/postgres/sample-configuration/smtpserver.xml new file mode 100644 index 0000000000..d6ac746f79 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/smtpserver.xml @@ -0,0 +1,104 @@ + + + + + + + + + smtpserver-global + 0.0.0.0:25 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + false + 127.0.0.0/8 + false + 0 + true + Apache JAMES awesome SMTP Server + + + + + + smtpserver-TLS + 0.0.0.0:465 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + + true + 127.0.0.0/8 + + false + 0 + true + Apache JAMES awesome SMTP Server + + + + + + smtpserver-authenticated + 0.0.0.0:587 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + + true + 127.0.0.0/8 + + false + 0 + true + Apache JAMES awesome SMTP Server + + + + + + + diff --git a/tmail-backend/apps/postgres/sample-configuration/usersrepository.xml b/tmail-backend/apps/postgres/sample-configuration/usersrepository.xml new file mode 100644 index 0000000000..e32286f482 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/usersrepository.xml @@ -0,0 +1,40 @@ + + + + + + + SHA-512 + true + true + + + + diff --git a/tmail-backend/apps/postgres/sample-configuration/webadmin.properties b/tmail-backend/apps/postgres/sample-configuration/webadmin.properties new file mode 100644 index 0000000000..5d72d99b74 --- /dev/null +++ b/tmail-backend/apps/postgres/sample-configuration/webadmin.properties @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This template file can be used as example for James Server configuration +# DO NOT USE IT AS SUCH AND ADAPT IT TO YOUR NEEDS + +# Read https://james.apache.org/server/config-webadmin.html for further details + +enabled=true +port=8000 +host=0.0.0.0 + +# Defaults to false +https.enabled=false + +# Compulsory when enabling HTTPS +#https.keystore=/path/to/keystore +#https.password=password + +# Optional when enabling HTTPS (self signed) +#https.trust.keystore +#https.trust.password + +# Defaults to false +#jwt.enabled=true +# +## If you wish to use OAuth authentication, you should provide a valid JWT public key. +## The following entry specify the link to the URL of the public key file, +## which should be a PEM format file. +## +#jwt.publickeypem.url=file://conf/jwt_publickey + +# Defaults to false +#cors.enable=true +#cors.origin + +# List of fully qualified class names that should be exposed over webadmin +# in addition to your product default routes. Routes needs to be located +# within the classpath or in the ./extensions-jars folder. +#extensions.routes= \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/extensions-jars/README.md b/tmail-backend/apps/postgres/src/main/extensions-jars/README.md new file mode 100644 index 0000000000..2cea759981 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/extensions-jars/README.md @@ -0,0 +1,5 @@ +# Adding Jars to JAMES + +The jar in this folder will be added to JAMES classpath when mounted under /root/extensions-jars inside the running container. + +You can use it to add you customs Mailets/Matchers. diff --git a/tmail-backend/apps/postgres/src/main/glowroot/admin.json b/tmail-backend/apps/postgres/src/main/glowroot/admin.json new file mode 100644 index 0000000000..c75c59d555 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/admin.json @@ -0,0 +1,5 @@ +{ + "web": { + "bindAddress": "0.0.0.0" + } +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/blobstore.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/blobstore.json new file mode 100644 index 0000000000..84291c6e16 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/blobstore.json @@ -0,0 +1,26 @@ +{ + "name": "BlobStore Plugin", + "id": "blob_store", + "instrumentation": [ + { + "captureKind": "trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "blobstore", + "className": "org.apache.james.blob.objectstorage.BlobPutter", + "methodName": "putDirectly", + "methodParameterTypes": [ + ".." + ] + }, + { + "captureKind": "trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "blobstore", + "className": "org.apache.james.blob.objectstorage.BlobPutter", + "methodName": "putAndComputeId", + "methodParameterTypes": [ + ".." + ] + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/imap.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/imap.json new file mode 100644 index 0000000000..d27904feb5 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/imap.json @@ -0,0 +1,19 @@ +{ + "name": "IMAP Plugin", + "id": "imap", + "instrumentation": [ + { + "className": "org.apache.james.imap.processor.base.AbstractChainedProcessor", + "methodName": "doProcess", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "IMAP", + "transactionNameTemplate": "IMAP processor : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "imapProcessor" + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/jmap.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/jmap.json new file mode 100644 index 0000000000..9afce4bf94 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/jmap.json @@ -0,0 +1,19 @@ +{ + "name": "JMAP Plugin", + "id": "jmap", + "instrumentation": [ + { + "className": "org.apache.james.jmap.draft.methods.Method", + "methodName": "processToStream", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "JMAP", + "transactionNameTemplate": "JMAP method : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "jmapMethod" + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/mailboxListener.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/mailboxListener.json new file mode 100644 index 0000000000..54a55ac1e4 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/mailboxListener.json @@ -0,0 +1,19 @@ +{ + "name": "MailboxListener Plugin", + "id": "mailboxListener", + "instrumentation": [ + { + "className": "org.apache.james.mailbox.events.MailboxListener", + "methodName": "event", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "MailboxListener", + "transactionNameTemplate": "MailboxListener : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "mailboxListener" + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/smtp.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/smtp.json new file mode 100644 index 0000000000..393bac9d9c --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/smtp.json @@ -0,0 +1,19 @@ +{ + "name": "SMTP Plugin", + "id": "smtp", + "instrumentation": [ + { + "className": "org.apache.james.protocols.smtp.core.AbstractHookableCmdHandler", + "methodName": "onCommand", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "SMTP", + "transactionNameTemplate": "SMTP command : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "smtpProcessor" + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/spooler.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/spooler.json new file mode 100644 index 0000000000..fd7732de8b --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/spooler.json @@ -0,0 +1,45 @@ +{ + "name": "Spooler Plugin", + "id": "spooler", + "instrumentation": [ + { + "className": "org.apache.james.mailetcontainer.api.MailProcessor", + "methodName": "service", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "Spooler", + "transactionNameTemplate": "Mailet processor : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "mailetProcessor" + }, + { + "className": "org.apache.mailet.Mailet", + "methodName": "service", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "Mailet", + "transactionNameTemplate": "Mailet : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "mailet" + }, + { + "className": "org.apache.mailet.Matcher", + "methodName": "match", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "Matcher", + "transactionNameTemplate": "Mailet processor : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "matcher" + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/glowroot/plugins/task.json b/tmail-backend/apps/postgres/src/main/glowroot/plugins/task.json new file mode 100644 index 0000000000..8f04c69e74 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/glowroot/plugins/task.json @@ -0,0 +1,19 @@ +{ + "name": "Task Plugin", + "id": "task", + "instrumentation": [ + { + "className": "org.apache.james.task.Task", + "methodName": "run", + "methodParameterTypes": [ + ".." + ], + "captureKind": "transaction", + "transactionType": "TASK", + "transactionNameTemplate": "TASK : {{this.class.name}}", + "alreadyInTransactionBehavior": "capture-trace-entry", + "traceEntryMessageTemplate": "{{this.class.name}}.{{methodName}}", + "timerName": "task" + } + ] +} \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java new file mode 100644 index 0000000000..3fae8d2dc6 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java @@ -0,0 +1,235 @@ +package com.linagora.tmail.james.app; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.Optional; + +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.james.PostgresJamesConfiguration; +import org.apache.james.SearchConfiguration; +import org.apache.james.filesystem.api.FileSystem; +import org.apache.james.filesystem.api.JamesDirectoriesProvider; +import org.apache.james.jmap.draft.JMAPModule; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.modules.queue.rabbitmq.MailQueueViewChoice; +import org.apache.james.server.core.JamesServerResourceLoader; +import org.apache.james.server.core.MissingArgumentException; +import org.apache.james.server.core.configuration.Configuration; +import org.apache.james.server.core.configuration.FileConfigurationProvider; +import org.apache.james.server.core.filesystem.FileSystemImpl; +import org.apache.james.utils.PropertiesProvider; +import org.apache.james.vault.VaultConfiguration; + +import com.github.fge.lambdas.Throwing; +import com.linagora.tmail.combined.identity.UsersRepositoryModuleChooser; +import com.linagora.tmail.encrypted.MailboxConfiguration; +import com.linagora.tmail.james.jmap.firebase.FirebaseModuleChooserConfiguration; +import com.linagora.tmail.james.jmap.service.discovery.LinagoraServicesDiscoveryModuleChooserConfiguration; + +public record PostgresTmailConfiguration(ConfigurationPath configurationPath, JamesDirectoriesProvider directories, + MailboxConfiguration mailboxConfiguration, + BlobStoreConfiguration blobStoreConfiguration, + SearchConfiguration searchConfiguration, + UsersRepositoryModuleChooser.Implementation usersRepositoryImplementation, + MailQueueViewChoice mailQueueViewChoice, + FirebaseModuleChooserConfiguration firebaseModuleChooserConfiguration, + LinagoraServicesDiscoveryModuleChooserConfiguration linagoraServicesDiscoveryModuleChooserConfiguration, + boolean jmapEnabled, + PropertiesProvider propertiesProvider, + PostgresJamesConfiguration.EventBusImpl eventBusImpl, + VaultConfiguration deletedMessageVaultConfiguration) implements Configuration { + public static class Builder { + private Optional mailboxConfiguration; + private Optional searchConfiguration; + private Optional blobStoreConfiguration; + private Optional rootDirectory; + private Optional configurationPath; + private Optional usersRepositoryImplementation; + private Optional mailQueueViewChoice; + private Optional firebaseModuleChooserConfiguration; + private Optional linagoraServicesDiscoveryModuleChooserConfiguration; + private Optional jmapEnabled; + + private Optional eventBusImpl; + private Optional deletedMessageVaultConfiguration; + + private Builder() { + searchConfiguration = Optional.empty(); + mailboxConfiguration = Optional.empty(); + rootDirectory = Optional.empty(); + configurationPath = Optional.empty(); + blobStoreConfiguration = Optional.empty(); + usersRepositoryImplementation = Optional.empty(); + mailQueueViewChoice = Optional.empty(); + firebaseModuleChooserConfiguration = Optional.empty(); + linagoraServicesDiscoveryModuleChooserConfiguration = Optional.empty(); + jmapEnabled = Optional.empty(); + eventBusImpl = Optional.empty(); + deletedMessageVaultConfiguration = Optional.empty(); + } + + public Builder workingDirectory(String path) { + rootDirectory = Optional.of(path); + return this; + } + + public Builder workingDirectory(File file) { + rootDirectory = Optional.of(file.getAbsolutePath()); + return this; + } + + public Builder useWorkingDirectoryEnvProperty() { + rootDirectory = Optional.ofNullable(System.getProperty(WORKING_DIRECTORY)); + if (rootDirectory.isEmpty()) { + throw new MissingArgumentException("Server needs a working.directory env entry"); + } + return this; + } + + public Builder configurationPath(ConfigurationPath path) { + configurationPath = Optional.of(path); + return this; + } + + public Builder configurationFromClasspath() { + configurationPath = Optional.of(new ConfigurationPath(FileSystem.CLASSPATH_PROTOCOL)); + return this; + } + + public Builder blobStore(BlobStoreConfiguration blobStoreConfiguration) { + this.blobStoreConfiguration = Optional.of(blobStoreConfiguration); + return this; + } + + public Builder mailbox(MailboxConfiguration mailboxConfiguration) { + this.mailboxConfiguration = Optional.of(mailboxConfiguration); + return this; + } + + public Builder searchConfiguration(SearchConfiguration searchConfiguration) { + this.searchConfiguration = Optional.of(searchConfiguration); + return this; + } + + public Builder usersRepository(UsersRepositoryModuleChooser.Implementation implementation) { + this.usersRepositoryImplementation = Optional.of(implementation); + return this; + } + + public Builder mailQueueViewChoice(MailQueueViewChoice mailQueueViewChoice) { + this.mailQueueViewChoice = Optional.of(mailQueueViewChoice); + return this; + } + + public Builder firebaseModuleChooserConfiguration(FirebaseModuleChooserConfiguration firebaseModuleChooserConfiguration) { + this.firebaseModuleChooserConfiguration = Optional.of(firebaseModuleChooserConfiguration); + return this; + } + + public Builder linagoraServicesDiscoveryModuleChooserConfiguration(LinagoraServicesDiscoveryModuleChooserConfiguration servicesDiscoveryModuleChooserConfiguration) { + this.linagoraServicesDiscoveryModuleChooserConfiguration = Optional.of(servicesDiscoveryModuleChooserConfiguration); + return this; + } + + public Builder jmapEnabled(boolean enable) { + this.jmapEnabled = Optional.of(enable); + return this; + } + + public Builder eventBusImpl(PostgresJamesConfiguration.EventBusImpl eventBusImpl) { + this.eventBusImpl = Optional.of(eventBusImpl); + return this; + } + + public Builder deletedMessageVaultConfiguration(VaultConfiguration deletedMessageVaultConfiguration) { + this.deletedMessageVaultConfiguration = Optional.of(deletedMessageVaultConfiguration); + return this; + } + + public PostgresTmailConfiguration build() { + ConfigurationPath configurationPath = this.configurationPath.orElse(new ConfigurationPath(FileSystem.FILE_PROTOCOL_AND_CONF)); + JamesServerResourceLoader directories = new JamesServerResourceLoader(rootDirectory + .orElseThrow(() -> new MissingArgumentException("Server needs a working.directory env entry"))); + + FileSystemImpl fileSystem = new FileSystemImpl(directories); + PropertiesProvider propertiesProvider = new PropertiesProvider(fileSystem, configurationPath); + BlobStoreConfiguration blobStoreConfiguration = this.blobStoreConfiguration.orElseGet(Throwing.supplier( + () -> BlobStoreConfiguration.parse(propertiesProvider))); + + SearchConfiguration searchConfiguration = this.searchConfiguration.orElseGet(Throwing.supplier( + () -> SearchConfiguration.parse(propertiesProvider))); + + MailboxConfiguration mailboxConfiguration = this.mailboxConfiguration.orElseGet(Throwing.supplier( + () -> MailboxConfiguration.parse(propertiesProvider))); + + FileConfigurationProvider configurationProvider = new FileConfigurationProvider(fileSystem, Basic.builder() + .configurationPath(configurationPath) + .workingDirectory(directories.getRootDirectory()) + .build()); + + UsersRepositoryModuleChooser.Implementation usersRepositoryChoice = usersRepositoryImplementation.orElseGet( + () -> UsersRepositoryModuleChooser.Implementation.parse(configurationProvider)); + + MailQueueViewChoice mailQueueViewChoice = this.mailQueueViewChoice.orElseGet(Throwing.supplier( + () -> MailQueueViewChoice.parse(propertiesProvider))); + + FirebaseModuleChooserConfiguration firebaseModuleChooserConfiguration = this.firebaseModuleChooserConfiguration.orElseGet(Throwing.supplier( + () -> FirebaseModuleChooserConfiguration.parse(propertiesProvider))); + + LinagoraServicesDiscoveryModuleChooserConfiguration servicesDiscoveryModuleChooserConfiguration = this.linagoraServicesDiscoveryModuleChooserConfiguration + .orElseGet(Throwing.supplier(() -> LinagoraServicesDiscoveryModuleChooserConfiguration.parse(propertiesProvider))); + + boolean jmapEnabled = this.jmapEnabled.orElseGet(() -> { + try { + return JMAPModule.parseConfiguration(propertiesProvider).isEnabled(); + } catch (FileNotFoundException e) { + return false; + } catch (ConfigurationException e) { + throw new RuntimeException(e); + } + }); + + PostgresJamesConfiguration.EventBusImpl eventBusImpl = this.eventBusImpl.orElseGet(() -> PostgresJamesConfiguration.EventBusImpl.from(propertiesProvider)); + + VaultConfiguration deletedMessageVaultConfiguration = this.deletedMessageVaultConfiguration.orElseGet(() -> { + try { + return VaultConfiguration.from(propertiesProvider.getConfiguration("deletedMessageVault")); + } catch (FileNotFoundException e) { + return VaultConfiguration.DEFAULT; + } catch (ConfigurationException e) { + throw new RuntimeException(e); + } + }); + + return new PostgresTmailConfiguration( + configurationPath, + directories, + mailboxConfiguration, + blobStoreConfiguration, + searchConfiguration, + usersRepositoryChoice, + mailQueueViewChoice, + firebaseModuleChooserConfiguration, + servicesDiscoveryModuleChooserConfiguration, + jmapEnabled, + propertiesProvider, + eventBusImpl, + deletedMessageVaultConfiguration); + } + } + + public static PostgresTmailConfiguration.Builder builder() { + return new Builder(); + } + + public boolean hasConfigurationProperties(String configurationPropertiesName) { + try { + propertiesProvider().getConfiguration(configurationPropertiesName); + return true; + } catch (FileNotFoundException notFoundException) { + return false; + } catch (ConfigurationException e) { + throw new RuntimeException(e); + } + } +} diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java new file mode 100644 index 0000000000..1c75bf209b --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java @@ -0,0 +1,228 @@ +package com.linagora.tmail.james.app; + +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.james.ExtraProperties; +import org.apache.james.GuiceJamesServer; +import org.apache.james.JamesServerMain; +import org.apache.james.SearchConfiguration; +import org.apache.james.SearchModuleChooser; +import org.apache.james.backends.postgres.PostgresModule; +import org.apache.james.data.LdapUsersRepositoryModule; +import org.apache.james.modules.BlobExportMechanismModule; +import org.apache.james.modules.MailboxModule; +import org.apache.james.modules.MailetProcessingModule; +import org.apache.james.modules.RunArgumentsModule; +import org.apache.james.modules.blobstore.BlobStoreCacheModulesChooser; +import org.apache.james.modules.blobstore.BlobStoreModulesChooser; +import org.apache.james.modules.data.PostgresDataModule; +import org.apache.james.modules.data.PostgresUsersRepositoryModule; +import org.apache.james.modules.data.SievePostgresRepositoryModules; +import org.apache.james.modules.event.RabbitMQEventBusModule; +import org.apache.james.modules.events.PostgresDeadLetterModule; +import org.apache.james.modules.eventstore.MemoryEventStoreModule; +import org.apache.james.modules.mailbox.DefaultEventModule; +import org.apache.james.modules.mailbox.PostgresDeletedMessageVaultModule; +import org.apache.james.modules.mailbox.PostgresMailboxModule; +import org.apache.james.modules.mailbox.TikaMailboxModule; +import org.apache.james.modules.protocols.IMAPServerModule; +import org.apache.james.modules.protocols.LMTPServerModule; +import org.apache.james.modules.protocols.ManageSieveServerModule; +import org.apache.james.modules.protocols.POP3ServerModule; +import org.apache.james.modules.protocols.ProtocolHandlerModule; +import org.apache.james.modules.protocols.SMTPServerModule; +import org.apache.james.modules.queue.activemq.ActiveMQQueueModule; +import org.apache.james.modules.queue.rabbitmq.RabbitMQModule; +import org.apache.james.modules.server.DataRoutesModules; +import org.apache.james.modules.server.DefaultProcessorsConfigurationProviderModule; +import org.apache.james.modules.server.InconsistencyQuotasSolvingRoutesModule; +import org.apache.james.modules.server.JMXServerModule; +import org.apache.james.modules.server.MailQueueRoutesModule; +import org.apache.james.modules.server.MailRepositoriesRoutesModule; +import org.apache.james.modules.server.MailboxRoutesModule; +import org.apache.james.modules.server.NoJwtModule; +import org.apache.james.modules.server.RawPostDequeueDecoratorModule; +import org.apache.james.modules.server.ReIndexingModule; +import org.apache.james.modules.server.SieveRoutesModule; +import org.apache.james.modules.server.TaskManagerModule; +import org.apache.james.modules.server.WebAdminReIndexingTaskSerializationModule; +import org.apache.james.modules.server.WebAdminServerModule; +import org.apache.james.modules.vault.DeletedMessageVaultRoutesModule; +import org.apache.james.rate.limiter.redis.RedisRateLimiterModule; +import org.apache.james.server.core.configuration.ConfigurationProvider; +import org.apache.james.user.api.DelegationStore; +import org.apache.james.user.api.DelegationUsernameChangeTaskStep; +import org.apache.james.user.api.UsernameChangeTaskStep; +import org.apache.james.user.lib.UsersDAO; +import org.apache.james.user.postgres.PostgresDelegationStore; +import org.apache.james.user.postgres.PostgresUserModule; +import org.apache.james.user.postgres.PostgresUsersDAO; +import org.apache.james.user.postgres.PostgresUsersRepositoryConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.AbstractModule; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.multibindings.Multibinder; +import com.google.inject.name.Named; +import com.google.inject.util.Modules; +import com.linagora.tmail.combined.identity.CombinedUserDAO; +import com.linagora.tmail.combined.identity.CombinedUsersRepositoryModule; + +public class PostgresTmailServer { + static Logger LOGGER = LoggerFactory.getLogger("org.apache.james.CONFIGURATION"); + + // TODO refactor after: https://github.com/apache/james-project/pull/1919 + private static final Module POSTGRES_DELEGATION_STORE_MODULE = new AbstractModule() { + @Override + protected void configure() { + bind(DelegationStore.class).to(PostgresDelegationStore.class); + bind(PostgresDelegationStore.UserExistencePredicate.class).to(PostgresDelegationStore.UserExistencePredicateImplementation.class); + + Multibinder.newSetBinder(binder(), UsernameChangeTaskStep.class) + .addBinding().to(DelegationUsernameChangeTaskStep.class); + } + + }; + + private static final Module POSTGRES_USER_REPOSITORY_MODULE = new AbstractModule() { + @Override + protected void configure() { + bind(PostgresUsersDAO.class).in(Scopes.SINGLETON); + bind(UsersDAO.class).to(PostgresUsersDAO.class); + + Multibinder postgresDataDefinitions = Multibinder.newSetBinder(binder(), PostgresModule.class); + postgresDataDefinitions.addBinding().toInstance(PostgresUserModule.MODULE); + install(new PostgresUsersRepositoryModule()); + } + + @Provides + @Singleton + public PostgresUsersRepositoryConfiguration provideConfiguration(ConfigurationProvider configurationProvider) throws ConfigurationException { + return PostgresUsersRepositoryConfiguration.from( + configurationProvider.getConfiguration("usersrepository")); + } + }; + + public static void main(String[] args) throws Exception { + ExtraProperties.initialize(); + + PostgresTmailConfiguration configuration = PostgresTmailConfiguration.builder() + .useWorkingDirectoryEnvProperty() + .build(); + + LOGGER.info("Loading configuration {}", configuration.toString()); + GuiceJamesServer server = createServer(configuration) + .combineWith(new JMXServerModule()) + .overrideWith(new RunArgumentsModule(args)); + + JamesServerMain.main(server); + } + + public static GuiceJamesServer createServer(PostgresTmailConfiguration configuration) { + SearchConfiguration searchConfiguration = configuration.searchConfiguration(); + + return GuiceJamesServer.forConfiguration(configuration) + .combineWith(SearchModuleChooser.chooseModules(searchConfiguration)) + .combineWith(chooseUserRepositoryModule(configuration)) + .combineWith(chooseBlobStoreModules(configuration)) + .combineWith(chooseEventBusModules(configuration)) + .combineWith(chooseDeletedMessageVaultModules(configuration)) + .combineWith(chooseRedisRateLimiterModule(configuration)) + .combineWith(POSTGRES_MODULE_AGGREGATE); + } + + + private static final Module WEBADMIN = Modules.combine( + new WebAdminServerModule(), + new DataRoutesModules(), + new InconsistencyQuotasSolvingRoutesModule(), + new MailboxRoutesModule(), + new MailQueueRoutesModule(), + new MailRepositoriesRoutesModule(), + new ReIndexingModule(), + new SieveRoutesModule(), + new WebAdminReIndexingTaskSerializationModule()); + + private static final Module PROTOCOLS = Modules.combine( + new IMAPServerModule(), + new LMTPServerModule(), + new ManageSieveServerModule(), + new POP3ServerModule(), + new ProtocolHandlerModule(), + new SMTPServerModule(), + WEBADMIN); + + private static final Module POSTGRES_SERVER_MODULE = Modules.combine( + new ActiveMQQueueModule(), + new BlobExportMechanismModule(), + POSTGRES_DELEGATION_STORE_MODULE, + new DefaultProcessorsConfigurationProviderModule(), + new PostgresMailboxModule(), + new PostgresDeadLetterModule(), + new PostgresDataModule(), + new MailboxModule(), + new NoJwtModule(), + new RawPostDequeueDecoratorModule(), + new SievePostgresRepositoryModules(), + new TaskManagerModule(), + new MemoryEventStoreModule(), + new TikaMailboxModule()); + private static final Module POSTGRES_MODULE_AGGREGATE = Modules.combine( + new MailetProcessingModule(), POSTGRES_SERVER_MODULE, PROTOCOLS); + + private static Module chooseBlobStoreModules(PostgresTmailConfiguration configuration) { + return Modules.combine(Modules.combine(BlobStoreModulesChooser.chooseModules(configuration.blobStoreConfiguration())), + new BlobStoreCacheModulesChooser.CacheDisabledModule()); + } + + public static Module chooseEventBusModules(PostgresTmailConfiguration configuration) { + return switch (configuration.eventBusImpl()) { + case IN_MEMORY -> new DefaultEventModule(); + case RABBITMQ -> Modules.combine(new RabbitMQModule(), + Modules.override(new DefaultEventModule()) + .with(new RabbitMQEventBusModule())); + }; + } + + public static Module chooseUserRepositoryModule(PostgresTmailConfiguration configuration) { + return switch (configuration.usersRepositoryImplementation()) { + case LDAP -> new LdapUsersRepositoryModule(); + case COMBINED -> Modules.override(POSTGRES_USER_REPOSITORY_MODULE) + .with(Modules.combine(new CombinedUsersRepositoryModule(), + new AbstractModule() { + @Provides + @Singleton + @Named(CombinedUserDAO.DATABASE_INJECT_NAME) + public UsersDAO provideDatabaseUserDAO(PostgresUsersDAO postgresUsersDAO) { + return postgresUsersDAO; + } + })); + case DEFAULT -> POSTGRES_USER_REPOSITORY_MODULE; + }; + } + + private static Module chooseDeletedMessageVaultModules(PostgresTmailConfiguration configuration) { + if (configuration.deletedMessageVaultConfiguration().isEnabled()) { + return Modules.combine(new PostgresDeletedMessageVaultModule(), new DeletedMessageVaultRoutesModule()); + } + return Modules.EMPTY_MODULE; + } + + private static Module chooseRedisRateLimiterModule(PostgresTmailConfiguration configuration) { + if (configuration.hasConfigurationProperties("redis")) { + return new RedisRateLimiterModule(); + } + return Modules.EMPTY_MODULE; + } + + // TODO: RspamdModule require MessageIdManager, We should do it later + /*private static Module chooseRspamdModule(PostgresTmailConfiguration configuration) { + if (configuration.hasConfigurationProperties("rspamd")) { + return new RspamdModule(); + } + return Modules.EMPTY_MODULE; + }*/ +} diff --git a/tmail-backend/apps/postgres/src/main/scripts/james-cli b/tmail-backend/apps/postgres/src/main/scripts/james-cli new file mode 100755 index 0000000000..19a73b6fb1 --- /dev/null +++ b/tmail-backend/apps/postgres/src/main/scripts/james-cli @@ -0,0 +1,3 @@ +#!/bin/bash + +java -cp /root/resources:/root/classes:/root/libs/* org.apache.james.cli.ServerCmd "$@" \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java new file mode 100644 index 0000000000..ca1066472b --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java @@ -0,0 +1,87 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package com.linagora.tmail.james.app; + +import static org.apache.james.PostgresJamesConfiguration.EventBusImpl.IN_MEMORY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import org.apache.james.GuiceJamesServer; +import org.apache.james.JamesServerBuilder; +import org.apache.james.JamesServerExtension; +import org.apache.james.SearchConfiguration; +import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.utils.DataProbeImpl; +import org.apache.james.utils.GuiceProbe; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.inject.multibindings.Multibinder; +import com.linagora.tmail.combined.identity.CombinedUsersRepository; +import com.linagora.tmail.combined.identity.LdapExtension; +import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; +import com.linagora.tmail.combined.identity.UsersRepositoryModuleChooser; +import com.linagora.tmail.encrypted.MailboxConfiguration; +import com.linagora.tmail.encrypted.MailboxManagerClassProbe; + +class CombinedIdentityTmailServerTest { + + static PostgresExtension postgresExtension = PostgresExtension.empty(); + + @RegisterExtension + static JamesServerExtension testExtension = new JamesServerBuilder(tmpDir -> + PostgresTmailConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.builder() + .postgres() + .disableCache() + .deduplication() + .noCryptoConfig()) + .searchConfiguration(SearchConfiguration.scanning()) + .usersRepository(UsersRepositoryModuleChooser.Implementation.COMBINED) + .mailbox(new MailboxConfiguration(false)) + .eventBusImpl(IN_MEMORY) + .build()) + .server(configuration -> PostgresTmailServer.createServer(configuration) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(MailboxManagerClassProbe.class)) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(UsersRepositoryClassProbe.class))) + .extension(postgresExtension) + .extensions(new LdapExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) + .build(); + + @Test + void shouldUseCombinedUsersRepositoryWhenSpecified(GuiceJamesServer server) { + assertThat(server.getProbe(UsersRepositoryClassProbe.class).getUserRepositoryClass()) + .isEqualTo(CombinedUsersRepository.class); + } + + @Test + void shouldAllowUserSynchronisation(GuiceJamesServer server) { + assertThatCode( + () -> server.getProbe(DataProbeImpl.class) + .fluent() + .addDomain("james.org") + .addUser("james-user@james.org", "123456")) + .doesNotThrowAnyException(); + } +} diff --git a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/DistributedPostgresTmailServerTest.java b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/DistributedPostgresTmailServerTest.java new file mode 100644 index 0000000000..68d727bc48 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/DistributedPostgresTmailServerTest.java @@ -0,0 +1,147 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package com.linagora.tmail.james.app; + +import static org.apache.james.PostgresJamesConfiguration.EventBusImpl.RABBITMQ; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Durations.FIVE_HUNDRED_MILLISECONDS; +import static org.awaitility.Durations.ONE_MINUTE; +import static org.hamcrest.Matchers.equalTo; + +import org.apache.james.DockerOpenSearchExtension; +import org.apache.james.GuiceJamesServer; +import org.apache.james.JamesServerBuilder; +import org.apache.james.JamesServerConcreteContract; +import org.apache.james.JamesServerExtension; +import org.apache.james.SearchConfiguration; +import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.core.healthcheck.ResultStatus; +import org.apache.james.core.quota.QuotaSizeLimit; +import org.apache.james.modules.AwsS3BlobStoreExtension; +import org.apache.james.modules.QuotaProbesImpl; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.modules.protocols.ImapGuiceProbe; +import org.apache.james.modules.protocols.SmtpGuiceProbe; +import org.apache.james.utils.DataProbeImpl; +import org.apache.james.utils.GuiceProbe; +import org.apache.james.utils.SMTPMessageSender; +import org.apache.james.utils.TestIMAPClient; +import org.apache.james.utils.WebAdminGuiceProbe; +import org.apache.james.webadmin.WebAdminUtils; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionFactory; +import org.eclipse.jetty.http.HttpStatus; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.common.base.Strings; +import com.google.inject.multibindings.Multibinder; +import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; +import com.linagora.tmail.encrypted.MailboxConfiguration; +import com.linagora.tmail.encrypted.MailboxManagerClassProbe; + +import io.restassured.specification.RequestSpecification; + +class DistributedPostgresTmailServerTest implements JamesServerConcreteContract { + + static PostgresExtension postgresExtension = PostgresExtension.empty(); + + @RegisterExtension + static RabbitMQExtension rabbitMQExtension = new RabbitMQExtension(); + + @RegisterExtension + static JamesServerExtension testExtension = new JamesServerBuilder(tmpDir -> + PostgresTmailConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.builder() + .s3() + .disableCache() + .deduplication() + .noCryptoConfig()) + .searchConfiguration(SearchConfiguration.openSearch()) + .mailbox(new MailboxConfiguration(false)) + .eventBusImpl(RABBITMQ) + .build()) + .server(configuration -> PostgresTmailServer.createServer(configuration) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(MailboxManagerClassProbe.class)) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(UsersRepositoryClassProbe.class))) + .extension(rabbitMQExtension) + .extension(postgresExtension) + .extension(new AwsS3BlobStoreExtension()) + .extension(new DockerOpenSearchExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) + .build(); + + private static final ConditionFactory AWAIT = Awaitility.await() + .atMost(ONE_MINUTE) + .with() + .pollInterval(FIVE_HUNDRED_MILLISECONDS); + static final String DOMAIN = "james.local"; + private static final String USER = "toto@" + DOMAIN; + private static final String PASSWORD = "123456"; + + private TestIMAPClient testIMAPClient; + private SMTPMessageSender smtpMessageSender; + private RequestSpecification webAdminApi; + @BeforeEach + void setUp(GuiceJamesServer guiceJamesServer) { + this.testIMAPClient = new TestIMAPClient(); + this.smtpMessageSender = new SMTPMessageSender(DOMAIN); + this.webAdminApi = WebAdminUtils.spec(guiceJamesServer.getProbe(WebAdminGuiceProbe.class).getWebAdminPort()); + } + + @Test + void guiceServerShouldUpdateQuota(GuiceJamesServer jamesServer) throws Exception { + jamesServer.getProbe(DataProbeImpl.class) + .fluent() + .addDomain(DOMAIN) + .addUser(USER, PASSWORD); + jamesServer.getProbe(QuotaProbesImpl.class).setGlobalMaxStorage(QuotaSizeLimit.size(50 * 1024)); + + // ~ 12 KB email + int imapPort = jamesServer.getProbe(ImapGuiceProbe.class).getImapPort(); + smtpMessageSender.connect(JAMES_SERVER_HOST, jamesServer.getProbe(SmtpGuiceProbe.class).getSmtpPort()) + .authenticate(USER, PASSWORD) + .sendMessageWithHeaders(USER, USER, "header: toto\\r\\n\\r\\n" + Strings.repeat("0123456789\n", 1024)); + AWAIT.until(() -> testIMAPClient.connect(JAMES_SERVER_HOST, imapPort) + .login(USER, PASSWORD) + .select(TestIMAPClient.INBOX) + .hasAMessage()); + + AWAIT.untilAsserted(() -> assertThat(testIMAPClient.connect(JAMES_SERVER_HOST, imapPort) + .login(USER, PASSWORD) + .getQuotaRoot(TestIMAPClient.INBOX)) + .startsWith("* QUOTAROOT \"INBOX\" #private&toto@james.local\r\n" + + "* QUOTA #private&toto@james.local (STORAGE 12 50)\r\n") + .endsWith("OK GETQUOTAROOT completed.\r\n")); + } + + @Test + void healthCheckShouldBeHealthy() { + webAdminApi.when() + .get("/healthcheck") + .then() + .statusCode(HttpStatus.OK_200) + .body("status", equalTo(ResultStatus.HEALTHY.getValue())); + } +} diff --git a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java new file mode 100644 index 0000000000..0ed56eeaae --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java @@ -0,0 +1,42 @@ +package com.linagora.tmail.james.app; + +import static org.apache.james.PostgresJamesConfiguration.EventBusImpl.IN_MEMORY; + +import org.apache.james.JamesServerBuilder; +import org.apache.james.JamesServerConcreteContract; +import org.apache.james.JamesServerExtension; +import org.apache.james.SearchConfiguration; +import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.utils.GuiceProbe; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.inject.multibindings.Multibinder; +import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; +import com.linagora.tmail.encrypted.MailboxConfiguration; +import com.linagora.tmail.encrypted.MailboxManagerClassProbe; + +class PostgresTmailServerTest implements JamesServerConcreteContract { + static PostgresExtension postgresExtension = PostgresExtension.empty(); + + @RegisterExtension + static JamesServerExtension testExtension = new JamesServerBuilder(tmpDir -> + PostgresTmailConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.builder() + .postgres() + .disableCache() + .deduplication() + .noCryptoConfig()) + .searchConfiguration(SearchConfiguration.scanning()) + .mailbox(new MailboxConfiguration(false)) + .eventBusImpl(IN_MEMORY) + .build()) + .server(configuration -> PostgresTmailServer.createServer(configuration) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(MailboxManagerClassProbe.class)) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(UsersRepositoryClassProbe.class))) + .extension(postgresExtension) + .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) + .build(); +} diff --git a/tmail-backend/apps/postgres/src/test/resources/batchsizes.properties b/tmail-backend/apps/postgres/src/test/resources/batchsizes.properties new file mode 100644 index 0000000000..1784f95d79 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/batchsizes.properties @@ -0,0 +1,9 @@ +# Those properties let you configure the number of messages queried at the same time. +# IMAP FETCH command +fetch.metadata=200 +fetch.headers=200 +fetch.full=50 +# IMAP COPY command +copy=100 +# IMAP MOVE command +move=100 diff --git a/tmail-backend/apps/postgres/src/test/resources/blob.properties b/tmail-backend/apps/postgres/src/test/resources/blob.properties new file mode 100644 index 0000000000..78ea935448 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/blob.properties @@ -0,0 +1,66 @@ +# ============================================= BlobStore Implementation ================================== +# Read https://james.apache.org/server/config-blobstore.html for further details + +# Choose your BlobStore implementation +# Mandatory, allowed values are: file, s3, postgres. +implementation=postgres + +# ========================================= Deduplication ======================================== +# If you choose to enable deduplication, the mails with the same content will be stored only once. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to the deletion of all +# the mails sharing the same content once one is deleted. +# Mandatory, Allowed values are: true, false +deduplication.enable=true + +# deduplication.family needs to be incremented every time the deduplication.generation.duration is changed +# Positive integer, defaults to 1 +# deduplication.gc.generation.family=1 + +# Duration of generation. +# Deduplication only takes place within a singe generation. +# Only items two generation old can be garbage collected. (This prevent concurrent insertions issues and +# accounts for a clock skew). +# deduplication.family needs to be incremented everytime this parameter is changed. +# Duration. Default unit: days. Defaults to 30 days. +# deduplication.gc.generation.duration=30days + +# ========================================= Encryption ======================================== +# If you choose to enable encryption, the blob content will be encrypted before storing them in the BlobStore. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to all content being +# encrypted. This comes at a performance impact but presents you from leaking data if, for instance the third party +# offering you a S3 service is compromised. +# Optional, Allowed values are: true, false, defaults to false +encryption.aes.enable=false + +# Mandatory (if AES encryption is enabled) salt and password. Salt needs to be an hexadecimal encoded string +#encryption.aes.password=xxx +#encryption.aes.salt=73616c7479 +# Optional, defaults to PBKDF2WithHmacSHA512 +#encryption.aes.private.key.algorithm=PBKDF2WithHmacSHA512 + +# ============================================ Blobs Exporting ============================================== +# Read https://james.apache.org/server/config-blob-export.html for further details + +# Choosing blob exporting mechanism, allowed mechanism are: localFile, linshare +# LinShare is a file sharing service, will be explained in the below section +# Optional, default is localFile +blob.export.implementation=localFile + +# ======================================= Local File Blobs Exporting ======================================== +# Optional, directory to store exported blob, directory path follows James file system format +# default is file://var/blobExporting +blob.export.localFile.directory=file://var/blobExporting + +# ======================================= LinShare File Blobs Exporting ======================================== +# LinShare is a sharing service where you can use james, connects to an existing LinShare server and shares files to +# other mail addresses as long as those addresses available in LinShare. For example you can deploy James and LinShare +# sharing the same LDAP repository +# Mandatory if you choose LinShare, url to connect to LinShare service +# blob.export.linshare.url=http://linshare:8080 + +# ======================================= LinShare Configuration BasicAuthentication =================================== +# Authentication is mandatory if you choose LinShare, TechnicalAccount is need to connect to LinShare specific service. +# For Example: It will be formalized to 'Authorization: Basic {Credential of UUID/password}' + +# blob.export.linshare.technical.account.uuid=Technical_Account_UUID +# blob.export.linshare.technical.account.password=password \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/test/resources/dnsservice.xml b/tmail-backend/apps/postgres/src/test/resources/dnsservice.xml new file mode 100644 index 0000000000..6e4fbd2efb --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/dnsservice.xml @@ -0,0 +1,25 @@ + + + + + true + false + 50000 + diff --git a/tmail-backend/apps/postgres/src/test/resources/domainlist.xml b/tmail-backend/apps/postgres/src/test/resources/domainlist.xml new file mode 100644 index 0000000000..fe17431a1e --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/domainlist.xml @@ -0,0 +1,24 @@ + + + + + false + false + diff --git a/tmail-backend/apps/postgres/src/test/resources/imapserver.xml b/tmail-backend/apps/postgres/src/test/resources/imapserver.xml new file mode 100644 index 0000000000..c99f956ce5 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/imapserver.xml @@ -0,0 +1,56 @@ + + + + + + + imapserver + 0.0.0.0:0 + 200 + + + classpath://keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + false + false + + + imapserver-ssl + 0.0.0.0:0 + 200 + + + classpath://keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + false + + diff --git a/tmail-backend/apps/postgres/src/test/resources/keystore b/tmail-backend/apps/postgres/src/test/resources/keystore new file mode 100644 index 0000000000000000000000000000000000000000..536a6c792b0740ef4273327bf4a61ffc2d6491d8 GIT binary patch literal 2245 zcmchY={pn*7sh8LBQ%y#WJwLmHX~!-n&e4IWZ$x7nXzWyM!ymF9%GH0|)`01HpknC;&o)EW|hTnC0KzVn%TKNE#dU1v||+1tZxX zS_9GsgkCLFCv|_)LvA!S*k!K2h)$={;+p9hHH7Nb0p>KwaVg~IFb3Sc1wDRw9$A){s zjWgyn8QQ_DwD67^UN~?lj{Brp?9aL{)#!V+F@3yd+SXoy#ls2T};RV4e2y4MYI1_L5*8Y+3@jZ}Jq=k;pjN{&W6V&8CnMam*;{LK8_ zVM=cij+9`Yn?R}TQ&+mUIg*K2CR|gqXqw>>3OJI|3T0Q6?~|~GQ+Cq*Ub{W= z#tEY5JH3B7<^Ay^isK!NQlyqlK>%jK4bn-JJ1I_tg1E53mrrAfv?W-!v5v*W1PD^o zxAg%m|LiTHI$`?t4_QyHAX{D{qH>>39tRp>KI;&`pMqjM%_S@a>jO>` z6pB-cdX{xVxy#YMXTrC-^vxG;KHTzHJl8ZO(ySb{-z~l#bcPwmZz!xT*qai`@=~g7 zm%`Wwk)!3E8#0=esd0RL9=xO}l_gdqO`CGH7ked&sARd)5kT$wm= z(V}s9O156MBTz(2khxa8_$Q`dZatu&qt;^pD<4J1$qXsr6Vb23Hu=&yB~!VNc_Jq7 z>VHqD5r3dce|yB1wtClTIY>%O@DHRB{=}X}6o%-w9had83mD84mrS?s_A(A^%{Ybf zRT$$U8`bB!I?xkRBP`95KfExp?{qx}b$oLcb-j z058_v&mR{oY2ohUgL4l=i3{_fF(`FqRg~I!WempdH=@zXD*wg*_c%nL)ISY5{1;#% zkPm<&0%0H`5C}-{<*=1KBbO?SE#xkKMXvqKHKh)AwKZ^R?x7Gq zEJ*}Q`i!-;D;`bn<_(PMs?Z!Azhb;wGdEjk+VigAO}tt$&0gSSAkd^Qu!YeAVl>_P zq$(ep;B$ZZRcA%4lYiy6#UI5)x3Z~7q5Zti`7%_(oi!vm`e!I-%8fY0(DZ6xzl)3s zC8vu)lBpgh%sJWw?xJ&^Lf|}E;FK>dP{OL^>8>odoE0JSm(A1w7;@mTwWsWTaS38liiOoY7+EQJp|1|ONst!#A z0&q=oUM&(2S+u)9)NE3)LgN5Iy~&PWa%6*-3MUjfcyByu7b)f3tpKXQeTd-2|17(3qjJ zuCdt!7~*+Jj-k$)2}|B;vFe5_aZzP>x+f-|h}*dnJi&WkeY1Xb&&jLmqkgpE0spgY zybxo}kn!S$8P;k(zWJ(t|K7IXP**)mv%t;DM3PJALygR(3trmZ)bjb(P7m4wUZX6{ zTa^)O + + + + \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml b/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml new file mode 100644 index 0000000000..c899874a42 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml @@ -0,0 +1,145 @@ + + + + + + + + + + + postmaster + + + + 20 + postgres://var/mail/error/ + + + + + + + postgres://var/mail/relay-limit-exceeded/ + + + transport + + + + + + mailetContainerErrors + + + ignore + + + postgres://var/mail/error/ + propagate + + + + + + + + + + + + bcc + ignore + + + rrt-error + + + + + + local-address-error + 550 - Requested action not taken: no such user here + + + + relay + + + + + + outgoing + 5000, 100000, 500000 + 3 + 0 + 10 + true + bounces + + + + + + mailetContainerLocalAddressError + + + none + + + postgres://var/mail/address-error/ + + + + + + mailetContainerRelayDenied + + + none + + + postgres://var/mail/relay-denied/ + Warning: You are sending an e-mail to a remote server. You must be authenticated to perform such an operation + + + + + + bounces + + + false + + + + + + postgres://var/mail/rrt-error/ + true + + + + + + + + \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/test/resources/mailrepositorystore.xml b/tmail-backend/apps/postgres/src/test/resources/mailrepositorystore.xml new file mode 100644 index 0000000000..445f2727f2 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/mailrepositorystore.xml @@ -0,0 +1,35 @@ + + + + + + + + postgres + + + + postgres + + + + + + diff --git a/tmail-backend/apps/postgres/src/test/resources/rabbitmq.properties b/tmail-backend/apps/postgres/src/test/resources/rabbitmq.properties new file mode 100644 index 0000000000..5a2ca93ec0 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/rabbitmq.properties @@ -0,0 +1,10 @@ +uri=amqp://james:james@rabbitmq_host:5672 +management.uri=http://james:james@rabbitmq_host:15672/api/ + +# AMQP resources parameters for the subscriber email address contact messages. In order to synchronize contacts between Team-mail backend and 3rd. +# AQMP uri +address.contact.uri=amqp://rabbitmq:5672 +address.contact.user=guest +address.contact.password=guest +# Queue name +address.contact.queue=AddressContactQueue1 \ No newline at end of file diff --git a/tmail-backend/apps/postgres/src/test/resources/smtpserver.xml b/tmail-backend/apps/postgres/src/test/resources/smtpserver.xml new file mode 100644 index 0000000000..5c8db4f901 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/smtpserver.xml @@ -0,0 +1,102 @@ + + + + + + + smtpserver-global + 0.0.0.0:0 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + false + false + 0 + true + Apache JAMES awesome SMTP Server + + + + false + + + smtpserver-TLS + 0.0.0.0:0 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + + true + + false + 0 + true + Apache JAMES awesome SMTP Server + + + + false + + + smtpserver-authenticated + 0.0.0.0:0 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + + true + + false + 0 + true + Apache JAMES awesome SMTP Server + + + + false + + + + diff --git a/tmail-backend/apps/postgres/src/test/resources/webadmin.properties b/tmail-backend/apps/postgres/src/test/resources/webadmin.properties new file mode 100644 index 0000000000..3386a14238 --- /dev/null +++ b/tmail-backend/apps/postgres/src/test/resources/webadmin.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This template file can be used as example for James Server configuration +# DO NOT USE IT AS SUCH AND ADAPT IT TO YOUR NEEDS + +# Read https://james.apache.org/server/config-webadmin.html for further details + +enabled=true +port=0 +host=127.0.0.1 \ No newline at end of file diff --git a/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/CombinedUserDAO.java b/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/CombinedUserDAO.java index 5a1a89b205..936b0e0676 100644 --- a/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/CombinedUserDAO.java +++ b/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/CombinedUserDAO.java @@ -4,11 +4,12 @@ import java.util.Optional; import jakarta.inject.Inject; +import jakarta.inject.Named; +import jakarta.inject.Singleton; import org.apache.james.core.Username; import org.apache.james.user.api.UsersRepositoryException; import org.apache.james.user.api.model.User; -import org.apache.james.user.cassandra.CassandraUsersDAO; import org.apache.james.user.ldap.ReadOnlyLDAPUsersDAO; import org.apache.james.user.lib.UsersDAO; import org.reactivestreams.Publisher; @@ -16,21 +17,23 @@ public class CombinedUserDAO implements UsersDAO { public static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(CombinedUserDAO.class); + public static final String DATABASE_INJECT_NAME = "database"; private final ReadOnlyLDAPUsersDAO readOnlyLDAPUsersDAO; - private final CassandraUsersDAO cassandraUsersDAO; + private final UsersDAO usersDAO; @Inject + @Singleton public CombinedUserDAO(ReadOnlyLDAPUsersDAO readOnlyLDAPUsersDAO, - CassandraUsersDAO cassandraUsersDAO) { + @Named(DATABASE_INJECT_NAME) UsersDAO usersDAO) { this.readOnlyLDAPUsersDAO = readOnlyLDAPUsersDAO; - this.cassandraUsersDAO = cassandraUsersDAO; + this.usersDAO = usersDAO; } @Override public void addUser(Username username, String password) throws UsersRepositoryException { if (readOnlyLDAPUsersDAO.contains(username)) { - cassandraUsersDAO.addUser(username, password); + usersDAO.addUser(username, password); } else { throw new UsersRepositoryException("Can not add user as it does not exits in LDAP repository. " + username.asString()); } @@ -44,40 +47,40 @@ public void updateUser(User user) throws UsersRepositoryException { @Override public void removeUser(Username name) throws UsersRepositoryException { if (!readOnlyLDAPUsersDAO.contains(name)) { - cassandraUsersDAO.removeUser(name); + usersDAO.removeUser(name); } else { throw new UsersRepositoryException("Can not remove user as it does still exit in LDAP repository. " + name.asString()); } } @Override - public Optional getUserByName(Username name) { - return cassandraUsersDAO.getUserByName(name); + public Optional getUserByName(Username name) throws UsersRepositoryException { + return usersDAO.getUserByName(name); } @Override - public boolean contains(Username name) { - return cassandraUsersDAO.contains(name); + public boolean contains(Username name) throws UsersRepositoryException { + return usersDAO.contains(name); } @Override public Publisher containsReactive(Username name) { - return cassandraUsersDAO.containsReactive(name); + return usersDAO.containsReactive(name); } @Override - public int countUsers() { - return cassandraUsersDAO.countUsers(); + public int countUsers() throws UsersRepositoryException { + return usersDAO.countUsers(); } @Override - public Iterator list() { - return cassandraUsersDAO.list(); + public Iterator list() throws UsersRepositoryException { + return usersDAO.list(); } @Override public Publisher listReactive() { - return cassandraUsersDAO.listReactive(); + return usersDAO.listReactive(); } public Optional test(Username name, String password) throws UsersRepositoryException { diff --git a/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java b/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java index 8bac8f621b..1d0890871e 100644 --- a/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java +++ b/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java @@ -32,11 +32,17 @@ import org.apache.james.data.LdapUsersRepositoryModule; import org.apache.james.modules.data.CassandraUsersRepositoryModule; import org.apache.james.server.core.configuration.FileConfigurationProvider; +import org.apache.james.user.cassandra.CassandraUsersDAO; +import org.apache.james.user.lib.UsersDAO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; +import com.google.inject.AbstractModule; import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.name.Named; public class UsersRepositoryModuleChooser { @@ -45,7 +51,14 @@ public class UsersRepositoryModuleChooser { public static List chooseModules(Implementation implementation) { return switch (implementation) { case LDAP -> ImmutableList.of(new LdapUsersRepositoryModule()); - case COMBINED -> ImmutableList.of(new CombinedUsersRepositoryModule()); + case COMBINED -> ImmutableList.of(new CombinedUsersRepositoryModule(), new AbstractModule() { + @Provides + @Singleton + @Named(CombinedUserDAO.DATABASE_INJECT_NAME) + public UsersDAO provideBackportUserDAO(CassandraUsersDAO cassandraUsersDAO) { + return cassandraUsersDAO; + } + }); case DEFAULT -> ImmutableList.of(new CassandraUsersRepositoryModule()); }; } From a2879102684d2ef3ff5687d228632be29a37078f Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Thu, 11 Jan 2024 10:59:35 +0700 Subject: [PATCH 05/81] ISSUE-883 postgres-app - Add bash script for quick test imap/smtp server when docker-compose up --- tmail-backend/apps/postgres/imap_smtp_test.sh | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tmail-backend/apps/postgres/imap_smtp_test.sh diff --git a/tmail-backend/apps/postgres/imap_smtp_test.sh b/tmail-backend/apps/postgres/imap_smtp_test.sh new file mode 100644 index 0000000000..caa4a8e304 --- /dev/null +++ b/tmail-backend/apps/postgres/imap_smtp_test.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Domain Configuration +WEBADMIN_BASE_URL="http://localhost:8000" +DOMAIN_NAME="domain.local" + +# IMAP Configuration +IMAP_SERVER="localhost" +IMAP_PORT=993 +ALICE_USERNAME="alice@${DOMAIN_NAME}" +ALICE_PASSWORD="secret" + +BOB_USERNAME="bob@${DOMAIN_NAME}" +BOB_PASSWORD="secret" + +# SMTP Configuration +SMTP_SERVER="localhost" +SMTP_PORT=25 +SMTP_FROM="${BOB_USERNAME}" +SMTP_TO="${ALICE_USERNAME}" +SMTP_SUBJECT="Test Email" +SMTP_BODY="Hello Alice, this is a test email." + +# Function to create a new domain using curl +create_domain() { + echo "Creating domain: ${DOMAIN_NAME}" + curl -X PUT ${WEBADMIN_BASE_URL}/domains/${DOMAIN_NAME} +} + +# Function to create a new user using curl +create_user() { + local username="$1" + local password="$2" + echo "Creating user: ${username}" + curl -L -X PUT "${WEBADMIN_BASE_URL}/users/${username}" \ + -H 'Content-Type: application/json' \ + -d "{\"password\":\"${password}\"}" +} + +# Function to send email using telnet +send_email() { + { + sleep 2 + echo "EHLO example.com" + sleep 2 + echo "MAIL FROM:<${SMTP_FROM}>" + sleep 2 + echo "RCPT TO:<${SMTP_TO}>" + sleep 2 + echo "DATA" + sleep 2 + echo "Subject: ${SMTP_SUBJECT}" + echo "" + echo "${SMTP_BODY}" + echo "." + sleep 2 + echo "QUIT" + } | telnet ${SMTP_SERVER} ${SMTP_PORT} +} + +# Function to fetch emails using openssl +fetch_emails() { + { + sleep 2 + echo "a1 LOGIN ${ALICE_USERNAME} ${ALICE_PASSWORD}" + sleep 2 + echo "a2 SELECT INBOX" + sleep 2 + echo "a3 SEARCH UNSEEN SUBJECT \"${SMTP_SUBJECT}\" FROM \"${SMTP_FROM}\"" + sleep 2 + echo "a4 LOGOUT" + } | openssl s_client -connect ${IMAP_SERVER}:${IMAP_PORT} -quiet +} + +# Create domain and users +create_domain +create_user "${ALICE_USERNAME}" "${ALICE_PASSWORD}" +create_user "${BOB_USERNAME}" "${BOB_PASSWORD}" + +# SMTP Test +echo "Sending test email..." +send_email + +# IMAP Test +echo "Fetching emails from ${ALICE_USERNAME}..." +# Fetch emails using openssl and check if the email from Bob is present +if fetch_emails | grep -q "a3 OK"; then + echo "Email from Bob found in Alice's inbox." +else + echo "Email from Bob not found in Alice's inbox." +fi + +echo "Test script completed." From cb9f9abca2901fe903d253db2059447207c6a50a Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Fri, 12 Jan 2024 15:33:27 +0700 Subject: [PATCH 06/81] ISSUE-883 Add deployment test for postgres --- tmail-backend/deployment-tests/pom.xml | 1 + .../deployment-tests/postgres/pom.xml | 32 +++++ .../deployment/PostgresImapAndSmtpTest.java | 21 ++++ .../deployment/TmailPostgresExtension.java | 114 ++++++++++++++++++ .../test/resources/james-conf/imapserver.xml | 64 ++++++++++ .../resources/james-conf/jmxremote.password | 1 + .../test/resources/james-conf/jwt_privatekey | 28 +++++ .../test/resources/james-conf/jwt_publickey | 9 ++ .../src/test/resources/james-conf/keystore | Bin 0 -> 2245 bytes tmail-backend/pom.xml | 11 ++ 10 files changed, 281 insertions(+) create mode 100644 tmail-backend/deployment-tests/postgres/pom.xml create mode 100644 tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresImapAndSmtpTest.java create mode 100644 tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/imapserver.xml create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jmxremote.password create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_privatekey create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_publickey create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/keystore diff --git a/tmail-backend/deployment-tests/pom.xml b/tmail-backend/deployment-tests/pom.xml index 1b8abb9986..0883201a4b 100644 --- a/tmail-backend/deployment-tests/pom.xml +++ b/tmail-backend/deployment-tests/pom.xml @@ -19,6 +19,7 @@ distributed distributed-ldap memory + postgres diff --git a/tmail-backend/deployment-tests/postgres/pom.xml b/tmail-backend/deployment-tests/postgres/pom.xml new file mode 100644 index 0000000000..4c927c3889 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + com.linagora.tmail + deployment-tests + 1.0.0-SNAPSHOT + + + deployment-tests-postgres + Team-mail :: Deployment Tests :: Postgres + + + + ${project.groupId} + deployment-tests-common + + + ${project.groupId} + postgres + + + ${project.groupId} + postgres + test-jar + test + + + + \ No newline at end of file diff --git a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresImapAndSmtpTest.java b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresImapAndSmtpTest.java new file mode 100644 index 0000000000..4055045435 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresImapAndSmtpTest.java @@ -0,0 +1,21 @@ +package com.linagora.tmail.deployment; + +import org.apache.james.mpt.imapmailbox.external.james.host.external.ExternalJamesConfiguration; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.GenericContainer; + +public class PostgresImapAndSmtpTest extends ImapAndSmtpContract { + + @RegisterExtension + TmailPostgresExtension extension = new TmailPostgresExtension(); + + @Override + protected ExternalJamesConfiguration configuration() { + return extension.configuration(); + } + + @Override + protected GenericContainer container() { + return extension.getContainer(); + } +} diff --git a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java new file mode 100644 index 0000000000..2a59deffad --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java @@ -0,0 +1,114 @@ +package com.linagora.tmail.deployment; + +import static com.linagora.tmail.deployment.ThirdPartyContainers.OS_IMAGE_NAME; +import static com.linagora.tmail.deployment.ThirdPartyContainers.OS_NETWORK_ALIAS; +import static com.linagora.tmail.deployment.ThirdPartyContainers.createRabbitMQ; +import static com.linagora.tmail.deployment.ThirdPartyContainers.createS3; +import static com.linagora.tmail.deployment.ThirdPartyContainers.createSearchContainer; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import org.apache.james.mpt.imapmailbox.external.james.host.external.ExternalJamesConfiguration; +import org.apache.james.util.Port; +import org.apache.james.util.Runnables; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.utility.MountableFile; + +public class TmailPostgresExtension implements BeforeEachCallback, AfterEachCallback { + private final Network network; + private final GenericContainer postgres; + private final GenericContainer opensearch; + private final GenericContainer rabbitmq; + private final GenericContainer s3; + private final GenericContainer tmailBackend; + + public TmailPostgresExtension() { + network = Network.newNetwork(); + postgres = createPostgres(network); + opensearch = createSearchContainer(network, OS_IMAGE_NAME, OS_NETWORK_ALIAS); + rabbitmq = createRabbitMQ(network); + s3 = createS3(network); + tmailBackend = createTmailDistributed(); + } + + @SuppressWarnings("resource") + private GenericContainer createTmailDistributed() { + return new GenericContainer<>("linagora/tmail-backend-postgresql-experimental:latest") + .withNetworkAliases("tmail-postgres") + .withNetwork(network) + .dependsOn(postgres, opensearch, s3, rabbitmq) + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/imapserver.xml"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jwt_privatekey"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jwt_publickey"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/keystore"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jmxremote.password"), "/root/conf/") + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-postgres-testing" + UUID.randomUUID())) + .waitingFor(TestContainerWaitStrategy.WAIT_STRATEGY) + .withExposedPorts(25, 143, 80, 8000); + } + + @SuppressWarnings("resource") + private GenericContainer createPostgres(Network network) { + return new GenericContainer<>("postgres:16.1") + .withNetworkAliases("postgres") + .withNetwork(network) + .withEnv("POSTGRES_USER", "tmail") + .withEnv("POSTGRES_PASSWORD", "secret1") + .withEnv("POSTGRES_DB", "postgres") + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-postgres-testing" + UUID.randomUUID())) + .waitingFor((new LogMessageWaitStrategy()).withRegEx(".*database system is ready to accept connections.*\\s").withTimes(2).withStartupTimeout(Duration.of(60L, ChronoUnit.SECONDS))) + .withExposedPorts(5432); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + String dockerSaveFileUrl = new File("").getAbsolutePath().replace(Paths.get("tmail-backend", "deployment-tests", "postgres").toString(), + Paths.get("tmail-backend", "apps", "postgres", "target", "jib-image.tar").toString()); + tmailBackend.getDockerClient().loadImageCmd(Files.newInputStream(Paths.get(dockerSaveFileUrl))).exec(); + tmailBackend.start(); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + tmailBackend.stop(); + Runnables.runParallel( + postgres::stop, + opensearch::stop, + rabbitmq::stop, + s3::stop); + } + + public GenericContainer getContainer() { + return tmailBackend; + } + + ExternalJamesConfiguration configuration() { + return new ExternalJamesConfiguration() { + @Override + public String getAddress() { + return tmailBackend.getHost(); + } + + @Override + public Port getImapPort() { + return Port.of(tmailBackend.getMappedPort(143)); + } + + @Override + public Port getSmptPort() { + return Port.of(tmailBackend.getMappedPort(25)); + } + }; + } + +} diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/imapserver.xml b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/imapserver.xml new file mode 100644 index 0000000000..87b596d5da --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/imapserver.xml @@ -0,0 +1,64 @@ + + + + + + + + + imapserver + 0.0.0.0:143 + 200 + + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + 120 + SECONDS + true + false + false + + + imapserver-ssl + 0.0.0.0:993 + 200 + + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + 120 + SECONDS + true + false + + diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jmxremote.password b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jmxremote.password new file mode 100644 index 0000000000..cf0b7bf422 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jmxremote.password @@ -0,0 +1 @@ +james-admin 123456 \ No newline at end of file diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_privatekey b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_privatekey new file mode 100644 index 0000000000..a4aa851ff5 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_privatekey @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDEV3BhLSBHuApH +GNlquIczzcvDOc0HoqczATsM7CeRp2SUicioxlItL2q6tKGTsK1zCcM+uKw1ydxd +AGDxpBD6kqIel/Quu/dRd8H6P/f4Hr0OOxGCgc78FYvjY6gcbPtaOmD9u/RMRqP6 +95s/DriPxPDYzA58yRH7NlhzJymthoZEtwIL/SuZ6e2huqnJdC11wEJgkwwkKO3x +01Y84V2y2EdRCvb5j9EOkCC5C4TGpjvSffD/4mcj+Z2VNTxQV0KdrlNxhBwwFEbL +TQ2B5hIkH2INDP6ubkxJeXmZ3iXyw+iNKtBUef9ZavIk4lZLKYNjZWX2eBg2cokT +p759oRoHAgMBAAECggEAdmyrCuH2C2wVPubdFIKygeuKEHnHkehoYtpGLLgv8al+ +gB1PG4VrQXfNL0oN/w/cvntP+X/X1yWnNa0py/YCi7Bv+nX6wUl8lfXe2TtGLLEV +pQS5vfbfyqqQUpnkZyjQvo5hvAlnA67D73bze6g8Z/MItir2PgvlPZl85g/kEpX3 +zt1yDTUFe+L8Ur+8Lied/0w2M53lUlebIYtsa2W7I05YzUBAVXkCIffnaIt3QtTC +tppBcVZXUacRtqULBMcUE2zc/yUKNqHzBjlkBv4VQ7nQDOjjUfW+VtccnTr+mLnl +R7VDy+POuZQ5u0gA8IwJNJFd5qdIm7l4tG7xa69rMQKBgQDmX1FF5Zty0Q7Acp1n +G9TZBOTTehzQPYsJwMynLR/b8mUAL+FTCh7Q7OJkGhcBxVDgc34OVPvc9wlFSUuU +ac0C0GtcD3BidatEfwMqVdpwcDnSEK47N13oSmaAL21mgC6/0ypV5TBgbkMRcxSb +h1GdeBWEG1RluVx6n1TflSvw6QKBgQDaLvj3fNbcIfJubx5oP2kmA1rw5jcSShK4 +UgM4Ifj98LOmsiB6qfY+36p6D78XINV0KpS3THi8rWzf31OuTN1BoZ5UpcyOOrDb +2pNnfGpC1VBP4rfWJMNpcGstj3YUNEV5pLyd5Cd6/gV8nRgiQ9ccEDJ1I5GXVWfV ++2a7/zddbwKBgHqWWi8xoWiVqp3p36yQeNEK86E9J7wAI86K09xZ/MwTzn8s+2Au +0HsossfFwlxk3Uay7m899dB9fGdsO1W8fyVyNs8EQC+EoiCO3eZXTSfr8DjCO5Sz +P7tua+DmW/bhWv8kpTCUBwwpYHMWo+6nMVz0G67yxBRlcLqnsohPXtSRAoGBAMMD +MxJ6Kc1OJlMgzKve6YvJefJRwq19Oag33ZrBerz29IwtMCyTV36xCb3Z7zGr7j3L +hWskVdJGrEaZZUEogKaV31/HZcNGoCeSASiBIrUj1ongmfI0n9jRW2q4jJDYe7ST +UudJMySSgbL08spFmrIBpCfhJ9N8ybeP4i5smj7PAoGBANSfn+DzQPICp2rldw7y +rPSCywIM6LzdoyykRMmX04sQAFVKgJwPei+oLrg8HcUjCZ1t8KJ4tHsvqFkQ0O4s +q8eh2eli5Qppg/Qmx1zT1W7+JYxlPsiXfmViBcY2+yNjNOHQPzyJyE+pvybW0DC+ +k9EJZv81OGrKInhjB/Ep6C76 +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_publickey b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_publickey new file mode 100644 index 0000000000..ea8beb9de0 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/jwt_publickey @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxFdwYS0gR7gKRxjZariH +M83LwznNB6KnMwE7DOwnkadklInIqMZSLS9qurShk7CtcwnDPrisNcncXQBg8aQQ ++pKiHpf0Lrv3UXfB+j/3+B69DjsRgoHO/BWL42OoHGz7Wjpg/bv0TEaj+vebPw64 +j8Tw2MwOfMkR+zZYcycprYaGRLcCC/0rmentobqpyXQtdcBCYJMMJCjt8dNWPOFd +sthHUQr2+Y/RDpAguQuExqY70n3w/+JnI/mdlTU8UFdCna5TcYQcMBRGy00NgeYS +JB9iDQz+rm5MSXl5md4l8sPojSrQVHn/WWryJOJWSymDY2Vl9ngYNnKJE6e+faEa +BwIDAQAB +-----END PUBLIC KEY----- \ No newline at end of file diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/keystore b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/keystore new file mode 100644 index 0000000000000000000000000000000000000000..536a6c792b0740ef4273327bf4a61ffc2d6491d8 GIT binary patch literal 2245 zcmchY={pn*7sh8LBQ%y#WJwLmHX~!-n&e4IWZ$x7nXzWyM!ymF9%GH0|)`01HpknC;&o)EW|hTnC0KzVn%TKNE#dU1v||+1tZxX zS_9GsgkCLFCv|_)LvA!S*k!K2h)$={;+p9hHH7Nb0p>KwaVg~IFb3Sc1wDRw9$A){s zjWgyn8QQ_DwD67^UN~?lj{Brp?9aL{)#!V+F@3yd+SXoy#ls2T};RV4e2y4MYI1_L5*8Y+3@jZ}Jq=k;pjN{&W6V&8CnMam*;{LK8_ zVM=cij+9`Yn?R}TQ&+mUIg*K2CR|gqXqw>>3OJI|3T0Q6?~|~GQ+Cq*Ub{W= z#tEY5JH3B7<^Ay^isK!NQlyqlK>%jK4bn-JJ1I_tg1E53mrrAfv?W-!v5v*W1PD^o zxAg%m|LiTHI$`?t4_QyHAX{D{qH>>39tRp>KI;&`pMqjM%_S@a>jO>` z6pB-cdX{xVxy#YMXTrC-^vxG;KHTzHJl8ZO(ySb{-z~l#bcPwmZz!xT*qai`@=~g7 zm%`Wwk)!3E8#0=esd0RL9=xO}l_gdqO`CGH7ked&sARd)5kT$wm= z(V}s9O156MBTz(2khxa8_$Q`dZatu&qt;^pD<4J1$qXsr6Vb23Hu=&yB~!VNc_Jq7 z>VHqD5r3dce|yB1wtClTIY>%O@DHRB{=}X}6o%-w9had83mD84mrS?s_A(A^%{Ybf zRT$$U8`bB!I?xkRBP`95KfExp?{qx}b$oLcb-j z058_v&mR{oY2ohUgL4l=i3{_fF(`FqRg~I!WempdH=@zXD*wg*_c%nL)ISY5{1;#% zkPm<&0%0H`5C}-{<*=1KBbO?SE#xkKMXvqKHKh)AwKZ^R?x7Gq zEJ*}Q`i!-;D;`bn<_(PMs?Z!Azhb;wGdEjk+VigAO}tt$&0gSSAkd^Qu!YeAVl>_P zq$(ep;B$ZZRcA%4lYiy6#UI5)x3Z~7q5Zti`7%_(oi!vm`e!I-%8fY0(DZ6xzl)3s zC8vu)lBpgh%sJWw?xJ&^Lf|}E;FK>dP{OL^>8>odoE0JSm(A1w7;@mTwWsWTaS38liiOoY7+EQJp|1|ONst!#A z0&q=oUM&(2S+u)9)NE3)LgN5Iy~&PWa%6*-3MUjfcyByu7b)f3tpKXQeTd-2|17(3qjJ zuCdt!7~*+Jj-k$)2}|B;vFe5_aZzP>x+f-|h}*dnJi&WkeY1Xb&&jLmqkgpE0spgY zybxo}kn!S$8P;k(zWJ(t|K7IXP**)mv%t;DM3PJALygR(3trmZ)bjb(P7m4wUZX6{ zTa^)Otest-jar ${project.version} + + ${project.groupId} + postgres + ${project.version} + + + ${project.groupId} + postgres + test-jar + ${project.version} + ${project.groupId} encrypted-mailbox-guice From 090f186703e9d870caa029d820104742138656aa Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Fri, 19 Jan 2024 06:03:58 +0700 Subject: [PATCH 07/81] ISSUE-895 Refactor Guice binding for choose User Module --- .../tmail/james/app/PostgresTmailServer.java | 75 +++++-------------- 1 file changed, 17 insertions(+), 58 deletions(-) diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java index 1c75bf209b..1eb65678b3 100644 --- a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java @@ -1,12 +1,10 @@ package com.linagora.tmail.james.app; -import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.james.ExtraProperties; import org.apache.james.GuiceJamesServer; import org.apache.james.JamesServerMain; import org.apache.james.SearchConfiguration; import org.apache.james.SearchModuleChooser; -import org.apache.james.backends.postgres.PostgresModule; import org.apache.james.data.LdapUsersRepositoryModule; import org.apache.james.modules.BlobExportMechanismModule; import org.apache.james.modules.MailboxModule; @@ -15,6 +13,7 @@ import org.apache.james.modules.blobstore.BlobStoreCacheModulesChooser; import org.apache.james.modules.blobstore.BlobStoreModulesChooser; import org.apache.james.modules.data.PostgresDataModule; +import org.apache.james.modules.data.PostgresDelegationStoreModule; import org.apache.james.modules.data.PostgresUsersRepositoryModule; import org.apache.james.modules.data.SievePostgresRepositoryModules; import org.apache.james.modules.event.RabbitMQEventBusModule; @@ -48,24 +47,15 @@ import org.apache.james.modules.server.WebAdminServerModule; import org.apache.james.modules.vault.DeletedMessageVaultRoutesModule; import org.apache.james.rate.limiter.redis.RedisRateLimiterModule; -import org.apache.james.server.core.configuration.ConfigurationProvider; -import org.apache.james.user.api.DelegationStore; -import org.apache.james.user.api.DelegationUsernameChangeTaskStep; -import org.apache.james.user.api.UsernameChangeTaskStep; import org.apache.james.user.lib.UsersDAO; -import org.apache.james.user.postgres.PostgresDelegationStore; -import org.apache.james.user.postgres.PostgresUserModule; import org.apache.james.user.postgres.PostgresUsersDAO; -import org.apache.james.user.postgres.PostgresUsersRepositoryConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.inject.AbstractModule; import com.google.inject.Module; import com.google.inject.Provides; -import com.google.inject.Scopes; import com.google.inject.Singleton; -import com.google.inject.multibindings.Multibinder; import com.google.inject.name.Named; import com.google.inject.util.Modules; import com.linagora.tmail.combined.identity.CombinedUserDAO; @@ -74,38 +64,6 @@ public class PostgresTmailServer { static Logger LOGGER = LoggerFactory.getLogger("org.apache.james.CONFIGURATION"); - // TODO refactor after: https://github.com/apache/james-project/pull/1919 - private static final Module POSTGRES_DELEGATION_STORE_MODULE = new AbstractModule() { - @Override - protected void configure() { - bind(DelegationStore.class).to(PostgresDelegationStore.class); - bind(PostgresDelegationStore.UserExistencePredicate.class).to(PostgresDelegationStore.UserExistencePredicateImplementation.class); - - Multibinder.newSetBinder(binder(), UsernameChangeTaskStep.class) - .addBinding().to(DelegationUsernameChangeTaskStep.class); - } - - }; - - private static final Module POSTGRES_USER_REPOSITORY_MODULE = new AbstractModule() { - @Override - protected void configure() { - bind(PostgresUsersDAO.class).in(Scopes.SINGLETON); - bind(UsersDAO.class).to(PostgresUsersDAO.class); - - Multibinder postgresDataDefinitions = Multibinder.newSetBinder(binder(), PostgresModule.class); - postgresDataDefinitions.addBinding().toInstance(PostgresUserModule.MODULE); - install(new PostgresUsersRepositoryModule()); - } - - @Provides - @Singleton - public PostgresUsersRepositoryConfiguration provideConfiguration(ConfigurationProvider configurationProvider) throws ConfigurationException { - return PostgresUsersRepositoryConfiguration.from( - configurationProvider.getConfiguration("usersrepository")); - } - }; - public static void main(String[] args) throws Exception { ExtraProperties.initialize(); @@ -158,7 +116,7 @@ public static GuiceJamesServer createServer(PostgresTmailConfiguration configura private static final Module POSTGRES_SERVER_MODULE = Modules.combine( new ActiveMQQueueModule(), new BlobExportMechanismModule(), - POSTGRES_DELEGATION_STORE_MODULE, + new PostgresDelegationStoreModule(), new DefaultProcessorsConfigurationProviderModule(), new PostgresMailboxModule(), new PostgresDeadLetterModule(), @@ -188,20 +146,21 @@ public static Module chooseEventBusModules(PostgresTmailConfiguration configurat } public static Module chooseUserRepositoryModule(PostgresTmailConfiguration configuration) { - return switch (configuration.usersRepositoryImplementation()) { - case LDAP -> new LdapUsersRepositoryModule(); - case COMBINED -> Modules.override(POSTGRES_USER_REPOSITORY_MODULE) - .with(Modules.combine(new CombinedUsersRepositoryModule(), - new AbstractModule() { - @Provides - @Singleton - @Named(CombinedUserDAO.DATABASE_INJECT_NAME) - public UsersDAO provideDatabaseUserDAO(PostgresUsersDAO postgresUsersDAO) { - return postgresUsersDAO; - } - })); - case DEFAULT -> POSTGRES_USER_REPOSITORY_MODULE; - }; + return Modules.combine(PostgresUsersRepositoryModule.USER_CONFIGURATION_MODULE, + switch (configuration.usersRepositoryImplementation()) { + case LDAP -> new LdapUsersRepositoryModule(); + case COMBINED -> Modules.override(new PostgresUsersRepositoryModule()) + .with(Modules.combine(new CombinedUsersRepositoryModule(), + new AbstractModule() { + @Provides + @Singleton + @Named(CombinedUserDAO.DATABASE_INJECT_NAME) + public UsersDAO provideDatabaseUserDAO(PostgresUsersDAO postgresUsersDAO) { + return postgresUsersDAO; + } + })); + case DEFAULT -> new PostgresUsersRepositoryModule(); + }); } private static Module chooseDeletedMessageVaultModules(PostgresTmailConfiguration configuration) { From d003f2dc0222ea299411c83585571a1bcc4727e5 Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Fri, 19 Jan 2024 06:45:47 +0700 Subject: [PATCH 08/81] ISSUE-895 Support choose Rspamd & Introduce postgres-healthcheck-integration-tests module --- tmail-backend/apps/postgres/pom.xml | 5 +- .../tmail/james/app/PostgresTmailServer.java | 7 +- .../pom.xml | 128 ++++++++++++++++ ...tgresTMailHealthCheckIntegrationTests.java | 68 ++++++++ .../src/test/resources/batchsizes.properties | 9 ++ .../src/test/resources/blob.properties | 66 ++++++++ .../src/test/resources/dnsservice.xml | 25 +++ .../src/test/resources/domainlist.xml | 24 +++ .../src/test/resources/imapserver.xml | 56 +++++++ .../src/test/resources/keystore | Bin 0 -> 2245 bytes .../src/test/resources/listeners.xml | 22 +++ .../src/test/resources/mailetcontainer.xml | 145 ++++++++++++++++++ .../test/resources/mailrepositorystore.xml | 35 +++++ .../src/test/resources/rabbitmq.properties | 10 ++ .../src/test/resources/redis.properties | 2 + .../src/test/resources/rspamd.properties | 4 + .../src/test/resources/smtpserver.xml | 102 ++++++++++++ .../src/test/resources/webadmin.properties | 25 +++ tmail-backend/integration-tests/pom.xml | 1 + 19 files changed, 730 insertions(+), 4 deletions(-) create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/pom.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/java/com/linagora/tmail/integration/postgres/PostgresTMailHealthCheckIntegrationTests.java create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/batchsizes.properties create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/blob.properties create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/dnsservice.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/domainlist.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/imapserver.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/keystore create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/listeners.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailetcontainer.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailrepositorystore.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rabbitmq.properties create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/redis.properties create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rspamd.properties create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/smtpserver.xml create mode 100644 tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/webadmin.properties diff --git a/tmail-backend/apps/postgres/pom.xml b/tmail-backend/apps/postgres/pom.xml index 030466bc84..e92da1339a 100644 --- a/tmail-backend/apps/postgres/pom.xml +++ b/tmail-backend/apps/postgres/pom.xml @@ -120,7 +120,6 @@ org.testcontainers postgresql - 1.19.1 test @@ -143,6 +142,10 @@ test-jar test + + ${project.groupId} + tmail-healthcheck + ${project.groupId} smtp-extensions diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java index 1eb65678b3..581d1eb414 100644 --- a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java @@ -60,6 +60,7 @@ import com.google.inject.util.Modules; import com.linagora.tmail.combined.identity.CombinedUserDAO; import com.linagora.tmail.combined.identity.CombinedUsersRepositoryModule; +import com.linagora.tmail.rspamd.RspamdModule; public class PostgresTmailServer { static Logger LOGGER = LoggerFactory.getLogger("org.apache.james.CONFIGURATION"); @@ -89,6 +90,7 @@ public static GuiceJamesServer createServer(PostgresTmailConfiguration configura .combineWith(chooseEventBusModules(configuration)) .combineWith(chooseDeletedMessageVaultModules(configuration)) .combineWith(chooseRedisRateLimiterModule(configuration)) + .combineWith(chooseRspamdModule(configuration)) .combineWith(POSTGRES_MODULE_AGGREGATE); } @@ -177,11 +179,10 @@ private static Module chooseRedisRateLimiterModule(PostgresTmailConfiguration co return Modules.EMPTY_MODULE; } - // TODO: RspamdModule require MessageIdManager, We should do it later - /*private static Module chooseRspamdModule(PostgresTmailConfiguration configuration) { + private static Module chooseRspamdModule(PostgresTmailConfiguration configuration) { if (configuration.hasConfigurationProperties("rspamd")) { return new RspamdModule(); } return Modules.EMPTY_MODULE; - }*/ + } } diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/pom.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/pom.xml new file mode 100644 index 0000000000..b29bd47c99 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + + com.linagora.tmail + integration-tests + 1.0.0-SNAPSHOT + ../../pom.xml + + + postgres-healthcheck-integration-tests + Team-mail :: Integration Tests :: Healthcheck :: Postgres + + + + ${project.groupId} + postgres + + + ${project.groupId} + postgres + test-jar + + + ${project.groupId} + healthcheck-integration-tests-common + test + + + ${project.groupId} + webadmin-integration-tests-common + test + + + ${project.groupId} + tmail-guice-distributed + test-jar + test + + + ${james.groupId} + apache-james-backends-postgres + test-jar + test + + + ${james.groupId} + apache-james-backends-opensearch + test-jar + test + + + org.testcontainers + postgresql + test + + + ${james.groupId} + james-server-guice-opensearch + test-jar + test + + + ${james.groupId} + james-server-guice-opensearch + + + ${james.groupId} + apache-james-backends-rabbitmq + test-jar + test + + + ${james.groupId} + apache-james-backends-redis + test-jar + test + + + ${james.groupId} + blob-s3 + test-jar + test + + + ${james.groupId} + blob-s3-guice + test-jar + test + + + ${james.groupId} + james-server-postgres-app + test-jar + test + + + ${james.groupId} + james-server-postgres-app + test-jar + test + + + ${james.groupId} + james-server-guice-common + test-jar + + + ${james.groupId} + james-server-guice-jmap + test-jar + + + ${james.groupId} + apache-james-rspamd + test-jar + test + + + ${james.groupId} + james-server-rate-limiter-redis + test-jar + test + + + + diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/java/com/linagora/tmail/integration/postgres/PostgresTMailHealthCheckIntegrationTests.java b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/java/com/linagora/tmail/integration/postgres/PostgresTMailHealthCheckIntegrationTests.java new file mode 100644 index 0000000000..e118e821d9 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/java/com/linagora/tmail/integration/postgres/PostgresTMailHealthCheckIntegrationTests.java @@ -0,0 +1,68 @@ +package com.linagora.tmail.integration.postgres; + +import static org.apache.james.PostgresJamesConfiguration.EventBusImpl.RABBITMQ; + +import org.apache.james.DockerOpenSearchExtension; +import org.apache.james.GuiceJamesServer; +import org.apache.james.JamesServerBuilder; +import org.apache.james.JamesServerExtension; +import org.apache.james.SearchConfiguration; +import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.backends.redis.RedisExtension; +import org.apache.james.modules.AwsS3BlobStoreExtension; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.utils.GuiceProbe; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.inject.multibindings.Multibinder; +import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; +import com.linagora.tmail.encrypted.MailboxConfiguration; +import com.linagora.tmail.encrypted.MailboxManagerClassProbe; +import com.linagora.tmail.integration.TMailHealthCheckIntegrationTests; +import com.linagora.tmail.james.app.PostgresTmailConfiguration; +import com.linagora.tmail.james.app.PostgresTmailServer; +import com.linagora.tmail.rspamd.RspamdExtensionModule; + +public class PostgresTMailHealthCheckIntegrationTests extends TMailHealthCheckIntegrationTests { + + @RegisterExtension + static JamesServerExtension testExtension = new JamesServerBuilder(tmpDir -> + PostgresTmailConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.builder() + .s3() + .disableCache() + .deduplication() + .noCryptoConfig()) + .searchConfiguration(SearchConfiguration.openSearch()) + .mailbox(new MailboxConfiguration(false)) + .eventBusImpl(RABBITMQ) + .build()) + .server(configuration -> PostgresTmailServer.createServer(configuration) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(MailboxManagerClassProbe.class)) + .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(UsersRepositoryClassProbe.class))) + .extension(PostgresExtension.empty()) + .extension(new RabbitMQExtension()) + .extension(new AwsS3BlobStoreExtension()) + .extension(new DockerOpenSearchExtension()) + .extension(new RspamdExtensionModule()) + .extension(new RedisExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) + .build(); + + @Disabled("We need a TaskExecutionDetailsProjection for PostgresTmailServer") + @Test + void tmailHealthCheckShouldBeWellBinded(GuiceJamesServer jamesServer) { + + } + + @Disabled("We need a TaskExecutionDetailsProjection for PostgresTmailServer") + @Test + void tasksHealthCheckShouldReturnHealthyByDefault(GuiceJamesServer jamesServer) { + + } +} diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/batchsizes.properties b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/batchsizes.properties new file mode 100644 index 0000000000..1784f95d79 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/batchsizes.properties @@ -0,0 +1,9 @@ +# Those properties let you configure the number of messages queried at the same time. +# IMAP FETCH command +fetch.metadata=200 +fetch.headers=200 +fetch.full=50 +# IMAP COPY command +copy=100 +# IMAP MOVE command +move=100 diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/blob.properties b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/blob.properties new file mode 100644 index 0000000000..78ea935448 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/blob.properties @@ -0,0 +1,66 @@ +# ============================================= BlobStore Implementation ================================== +# Read https://james.apache.org/server/config-blobstore.html for further details + +# Choose your BlobStore implementation +# Mandatory, allowed values are: file, s3, postgres. +implementation=postgres + +# ========================================= Deduplication ======================================== +# If you choose to enable deduplication, the mails with the same content will be stored only once. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to the deletion of all +# the mails sharing the same content once one is deleted. +# Mandatory, Allowed values are: true, false +deduplication.enable=true + +# deduplication.family needs to be incremented every time the deduplication.generation.duration is changed +# Positive integer, defaults to 1 +# deduplication.gc.generation.family=1 + +# Duration of generation. +# Deduplication only takes place within a singe generation. +# Only items two generation old can be garbage collected. (This prevent concurrent insertions issues and +# accounts for a clock skew). +# deduplication.family needs to be incremented everytime this parameter is changed. +# Duration. Default unit: days. Defaults to 30 days. +# deduplication.gc.generation.duration=30days + +# ========================================= Encryption ======================================== +# If you choose to enable encryption, the blob content will be encrypted before storing them in the BlobStore. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to all content being +# encrypted. This comes at a performance impact but presents you from leaking data if, for instance the third party +# offering you a S3 service is compromised. +# Optional, Allowed values are: true, false, defaults to false +encryption.aes.enable=false + +# Mandatory (if AES encryption is enabled) salt and password. Salt needs to be an hexadecimal encoded string +#encryption.aes.password=xxx +#encryption.aes.salt=73616c7479 +# Optional, defaults to PBKDF2WithHmacSHA512 +#encryption.aes.private.key.algorithm=PBKDF2WithHmacSHA512 + +# ============================================ Blobs Exporting ============================================== +# Read https://james.apache.org/server/config-blob-export.html for further details + +# Choosing blob exporting mechanism, allowed mechanism are: localFile, linshare +# LinShare is a file sharing service, will be explained in the below section +# Optional, default is localFile +blob.export.implementation=localFile + +# ======================================= Local File Blobs Exporting ======================================== +# Optional, directory to store exported blob, directory path follows James file system format +# default is file://var/blobExporting +blob.export.localFile.directory=file://var/blobExporting + +# ======================================= LinShare File Blobs Exporting ======================================== +# LinShare is a sharing service where you can use james, connects to an existing LinShare server and shares files to +# other mail addresses as long as those addresses available in LinShare. For example you can deploy James and LinShare +# sharing the same LDAP repository +# Mandatory if you choose LinShare, url to connect to LinShare service +# blob.export.linshare.url=http://linshare:8080 + +# ======================================= LinShare Configuration BasicAuthentication =================================== +# Authentication is mandatory if you choose LinShare, TechnicalAccount is need to connect to LinShare specific service. +# For Example: It will be formalized to 'Authorization: Basic {Credential of UUID/password}' + +# blob.export.linshare.technical.account.uuid=Technical_Account_UUID +# blob.export.linshare.technical.account.password=password \ No newline at end of file diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/dnsservice.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/dnsservice.xml new file mode 100644 index 0000000000..6e4fbd2efb --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/dnsservice.xml @@ -0,0 +1,25 @@ + + + + + true + false + 50000 + diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/domainlist.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/domainlist.xml new file mode 100644 index 0000000000..fe17431a1e --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/domainlist.xml @@ -0,0 +1,24 @@ + + + + + false + false + diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/imapserver.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/imapserver.xml new file mode 100644 index 0000000000..c99f956ce5 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/imapserver.xml @@ -0,0 +1,56 @@ + + + + + + + imapserver + 0.0.0.0:0 + 200 + + + classpath://keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + false + false + + + imapserver-ssl + 0.0.0.0:0 + 200 + + + classpath://keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + + 0 + 0 + false + + diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/keystore b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/keystore new file mode 100644 index 0000000000000000000000000000000000000000..536a6c792b0740ef4273327bf4a61ffc2d6491d8 GIT binary patch literal 2245 zcmchY={pn*7sh8LBQ%y#WJwLmHX~!-n&e4IWZ$x7nXzWyM!ymF9%GH0|)`01HpknC;&o)EW|hTnC0KzVn%TKNE#dU1v||+1tZxX zS_9GsgkCLFCv|_)LvA!S*k!K2h)$={;+p9hHH7Nb0p>KwaVg~IFb3Sc1wDRw9$A){s zjWgyn8QQ_DwD67^UN~?lj{Brp?9aL{)#!V+F@3yd+SXoy#ls2T};RV4e2y4MYI1_L5*8Y+3@jZ}Jq=k;pjN{&W6V&8CnMam*;{LK8_ zVM=cij+9`Yn?R}TQ&+mUIg*K2CR|gqXqw>>3OJI|3T0Q6?~|~GQ+Cq*Ub{W= z#tEY5JH3B7<^Ay^isK!NQlyqlK>%jK4bn-JJ1I_tg1E53mrrAfv?W-!v5v*W1PD^o zxAg%m|LiTHI$`?t4_QyHAX{D{qH>>39tRp>KI;&`pMqjM%_S@a>jO>` z6pB-cdX{xVxy#YMXTrC-^vxG;KHTzHJl8ZO(ySb{-z~l#bcPwmZz!xT*qai`@=~g7 zm%`Wwk)!3E8#0=esd0RL9=xO}l_gdqO`CGH7ked&sARd)5kT$wm= z(V}s9O156MBTz(2khxa8_$Q`dZatu&qt;^pD<4J1$qXsr6Vb23Hu=&yB~!VNc_Jq7 z>VHqD5r3dce|yB1wtClTIY>%O@DHRB{=}X}6o%-w9had83mD84mrS?s_A(A^%{Ybf zRT$$U8`bB!I?xkRBP`95KfExp?{qx}b$oLcb-j z058_v&mR{oY2ohUgL4l=i3{_fF(`FqRg~I!WempdH=@zXD*wg*_c%nL)ISY5{1;#% zkPm<&0%0H`5C}-{<*=1KBbO?SE#xkKMXvqKHKh)AwKZ^R?x7Gq zEJ*}Q`i!-;D;`bn<_(PMs?Z!Azhb;wGdEjk+VigAO}tt$&0gSSAkd^Qu!YeAVl>_P zq$(ep;B$ZZRcA%4lYiy6#UI5)x3Z~7q5Zti`7%_(oi!vm`e!I-%8fY0(DZ6xzl)3s zC8vu)lBpgh%sJWw?xJ&^Lf|}E;FK>dP{OL^>8>odoE0JSm(A1w7;@mTwWsWTaS38liiOoY7+EQJp|1|ONst!#A z0&q=oUM&(2S+u)9)NE3)LgN5Iy~&PWa%6*-3MUjfcyByu7b)f3tpKXQeTd-2|17(3qjJ zuCdt!7~*+Jj-k$)2}|B;vFe5_aZzP>x+f-|h}*dnJi&WkeY1Xb&&jLmqkgpE0spgY zybxo}kn!S$8P;k(zWJ(t|K7IXP**)mv%t;DM3PJALygR(3trmZ)bjb(P7m4wUZX6{ zTa^)O + + + + \ No newline at end of file diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailetcontainer.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailetcontainer.xml new file mode 100644 index 0000000000..c899874a42 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailetcontainer.xml @@ -0,0 +1,145 @@ + + + + + + + + + + + postmaster + + + + 20 + postgres://var/mail/error/ + + + + + + + postgres://var/mail/relay-limit-exceeded/ + + + transport + + + + + + mailetContainerErrors + + + ignore + + + postgres://var/mail/error/ + propagate + + + + + + + + + + + + bcc + ignore + + + rrt-error + + + + + + local-address-error + 550 - Requested action not taken: no such user here + + + + relay + + + + + + outgoing + 5000, 100000, 500000 + 3 + 0 + 10 + true + bounces + + + + + + mailetContainerLocalAddressError + + + none + + + postgres://var/mail/address-error/ + + + + + + mailetContainerRelayDenied + + + none + + + postgres://var/mail/relay-denied/ + Warning: You are sending an e-mail to a remote server. You must be authenticated to perform such an operation + + + + + + bounces + + + false + + + + + + postgres://var/mail/rrt-error/ + true + + + + + + + + \ No newline at end of file diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailrepositorystore.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailrepositorystore.xml new file mode 100644 index 0000000000..445f2727f2 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/mailrepositorystore.xml @@ -0,0 +1,35 @@ + + + + + + + + postgres + + + + postgres + + + + + + diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rabbitmq.properties b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rabbitmq.properties new file mode 100644 index 0000000000..5a2ca93ec0 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rabbitmq.properties @@ -0,0 +1,10 @@ +uri=amqp://james:james@rabbitmq_host:5672 +management.uri=http://james:james@rabbitmq_host:15672/api/ + +# AMQP resources parameters for the subscriber email address contact messages. In order to synchronize contacts between Team-mail backend and 3rd. +# AQMP uri +address.contact.uri=amqp://rabbitmq:5672 +address.contact.user=guest +address.contact.password=guest +# Queue name +address.contact.queue=AddressContactQueue1 \ No newline at end of file diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/redis.properties b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/redis.properties new file mode 100644 index 0000000000..449e981549 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/redis.properties @@ -0,0 +1,2 @@ +redisURL=redis://redis:6379 +cluster.enabled=false \ No newline at end of file diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rspamd.properties b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rspamd.properties new file mode 100644 index 0000000000..7382f27a96 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/rspamd.properties @@ -0,0 +1,4 @@ +rspamdUrl=http://rspamd:11334 +rspamdPassword=admin +# Whether to scan/learn mails using per-user Bayes. Default to false. +perUserBayes=false \ No newline at end of file diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/smtpserver.xml b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/smtpserver.xml new file mode 100644 index 0000000000..5c8db4f901 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/smtpserver.xml @@ -0,0 +1,102 @@ + + + + + + + smtpserver-global + 0.0.0.0:0 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + false + false + 0 + true + Apache JAMES awesome SMTP Server + + + + false + + + smtpserver-TLS + 0.0.0.0:0 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + + true + + false + 0 + true + Apache JAMES awesome SMTP Server + + + + false + + + smtpserver-authenticated + 0.0.0.0:0 + 200 + + file://conf/keystore + james72laBalle + org.bouncycastle.jce.provider.BouncyCastleProvider + SunX509 + + 360 + 0 + 0 + + true + + false + 0 + true + Apache JAMES awesome SMTP Server + + + + false + + + + diff --git a/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/webadmin.properties b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/webadmin.properties new file mode 100644 index 0000000000..3386a14238 --- /dev/null +++ b/tmail-backend/integration-tests/healthcheck/postgres-healthcheck-integration-tests/src/test/resources/webadmin.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This template file can be used as example for James Server configuration +# DO NOT USE IT AS SUCH AND ADAPT IT TO YOUR NEEDS + +# Read https://james.apache.org/server/config-webadmin.html for further details + +enabled=true +port=0 +host=127.0.0.1 \ No newline at end of file diff --git a/tmail-backend/integration-tests/pom.xml b/tmail-backend/integration-tests/pom.xml index 9343a80d00..cf792c52a9 100644 --- a/tmail-backend/integration-tests/pom.xml +++ b/tmail-backend/integration-tests/pom.xml @@ -34,6 +34,7 @@ rspamd/distributed-rspamd-integration-tests healthcheck/healthcheck-integration-tests-common healthcheck/distributed-healthcheck-integration-tests + healthcheck/postgres-healthcheck-integration-tests From 6bea651b0f862319c9444187e048818b88af3d63 Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Fri, 19 Jan 2024 11:50:16 +0700 Subject: [PATCH 09/81] ISSUE-895 Refactor - generic code for user module chooser with Combined Identity --- tmail-backend/apps/distributed/pom.xml | 4 ++ .../app/DistributedJamesConfiguration.java | 1 + .../app/CombinedIdentityJamesServerTest.java | 2 +- tmail-backend/apps/postgres/pom.xml | 4 ++ .../james/app/PostgresTmailConfiguration.java | 2 +- .../tmail/james/app/PostgresTmailServer.java | 28 ++------- .../app/CombinedIdentityTmailServerTest.java | 2 +- tmail-backend/guice/common/pom.xml | 41 +++++++++++++ .../DatabaseCombinedUserRequireModule.java | 49 +++++++++++++++ .../tmail}/UsersRepositoryModuleChooser.java | 60 +++++++------------ tmail-backend/pom.xml | 6 ++ 11 files changed, 135 insertions(+), 64 deletions(-) create mode 100644 tmail-backend/guice/common/pom.xml create mode 100644 tmail-backend/guice/common/src/main/java/com/linagora/tmail/DatabaseCombinedUserRequireModule.java rename tmail-backend/{combined-identity/src/main/java/com/linagora/tmail/combined/identity => guice/common/src/main/java/com/linagora/tmail}/UsersRepositoryModuleChooser.java (60%) diff --git a/tmail-backend/apps/distributed/pom.xml b/tmail-backend/apps/distributed/pom.xml index 0c7f30c0f2..485db375bd 100644 --- a/tmail-backend/apps/distributed/pom.xml +++ b/tmail-backend/apps/distributed/pom.xml @@ -73,6 +73,10 @@ ${project.groupId} team-mailboxes-guice + + ${project.groupId} + tmail-guice-common + ${project.groupId} tmail-guice-distributed diff --git a/tmail-backend/apps/distributed/src/main/java/com/linagora/tmail/james/app/DistributedJamesConfiguration.java b/tmail-backend/apps/distributed/src/main/java/com/linagora/tmail/james/app/DistributedJamesConfiguration.java index 75ffc0d0f1..b7d8965554 100644 --- a/tmail-backend/apps/distributed/src/main/java/com/linagora/tmail/james/app/DistributedJamesConfiguration.java +++ b/tmail-backend/apps/distributed/src/main/java/com/linagora/tmail/james/app/DistributedJamesConfiguration.java @@ -22,6 +22,7 @@ import com.linagora.tmail.OpenPaasModuleChooserConfiguration; import com.linagora.tmail.blob.guice.BlobStoreConfiguration; import com.linagora.tmail.combined.identity.UsersRepositoryModuleChooser; +import com.linagora.tmail.UsersRepositoryModuleChooser; import com.linagora.tmail.encrypted.MailboxConfiguration; import com.linagora.tmail.james.jmap.firebase.FirebaseModuleChooserConfiguration; import com.linagora.tmail.james.jmap.service.discovery.LinagoraServicesDiscoveryModuleChooserConfiguration; diff --git a/tmail-backend/apps/distributed/src/test/java/com/linagora/tmail/james/app/CombinedIdentityJamesServerTest.java b/tmail-backend/apps/distributed/src/test/java/com/linagora/tmail/james/app/CombinedIdentityJamesServerTest.java index f383986e57..f707e397ad 100644 --- a/tmail-backend/apps/distributed/src/test/java/com/linagora/tmail/james/app/CombinedIdentityJamesServerTest.java +++ b/tmail-backend/apps/distributed/src/test/java/com/linagora/tmail/james/app/CombinedIdentityJamesServerTest.java @@ -14,10 +14,10 @@ import org.junit.jupiter.api.extension.RegisterExtension; import com.google.inject.multibindings.Multibinder; +import com.linagora.tmail.UsersRepositoryModuleChooser; import com.linagora.tmail.combined.identity.CombinedUsersRepository; import com.linagora.tmail.combined.identity.LdapExtension; import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; -import com.linagora.tmail.combined.identity.UsersRepositoryModuleChooser; import com.linagora.tmail.module.LinagoraTestJMAPServerModule; public class CombinedIdentityJamesServerTest { diff --git a/tmail-backend/apps/postgres/pom.xml b/tmail-backend/apps/postgres/pom.xml index e92da1339a..156a2a204f 100644 --- a/tmail-backend/apps/postgres/pom.xml +++ b/tmail-backend/apps/postgres/pom.xml @@ -146,6 +146,10 @@ ${project.groupId} tmail-healthcheck + + ${project.groupId} + tmail-guice-common + ${project.groupId} smtp-extensions diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java index 3fae8d2dc6..5c4c9c24db 100644 --- a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailConfiguration.java @@ -21,7 +21,7 @@ import org.apache.james.vault.VaultConfiguration; import com.github.fge.lambdas.Throwing; -import com.linagora.tmail.combined.identity.UsersRepositoryModuleChooser; +import com.linagora.tmail.UsersRepositoryModuleChooser; import com.linagora.tmail.encrypted.MailboxConfiguration; import com.linagora.tmail.james.jmap.firebase.FirebaseModuleChooserConfiguration; import com.linagora.tmail.james.jmap.service.discovery.LinagoraServicesDiscoveryModuleChooserConfiguration; diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java index 581d1eb414..203d54b884 100644 --- a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java @@ -5,7 +5,6 @@ import org.apache.james.JamesServerMain; import org.apache.james.SearchConfiguration; import org.apache.james.SearchModuleChooser; -import org.apache.james.data.LdapUsersRepositoryModule; import org.apache.james.modules.BlobExportMechanismModule; import org.apache.james.modules.MailboxModule; import org.apache.james.modules.MailetProcessingModule; @@ -47,19 +46,14 @@ import org.apache.james.modules.server.WebAdminServerModule; import org.apache.james.modules.vault.DeletedMessageVaultRoutesModule; import org.apache.james.rate.limiter.redis.RedisRateLimiterModule; -import org.apache.james.user.lib.UsersDAO; import org.apache.james.user.postgres.PostgresUsersDAO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.AbstractModule; import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.name.Named; import com.google.inject.util.Modules; -import com.linagora.tmail.combined.identity.CombinedUserDAO; -import com.linagora.tmail.combined.identity.CombinedUsersRepositoryModule; +import com.linagora.tmail.DatabaseCombinedUserRequireModule; +import com.linagora.tmail.UsersRepositoryModuleChooser; import com.linagora.tmail.rspamd.RspamdModule; public class PostgresTmailServer { @@ -149,20 +143,10 @@ public static Module chooseEventBusModules(PostgresTmailConfiguration configurat public static Module chooseUserRepositoryModule(PostgresTmailConfiguration configuration) { return Modules.combine(PostgresUsersRepositoryModule.USER_CONFIGURATION_MODULE, - switch (configuration.usersRepositoryImplementation()) { - case LDAP -> new LdapUsersRepositoryModule(); - case COMBINED -> Modules.override(new PostgresUsersRepositoryModule()) - .with(Modules.combine(new CombinedUsersRepositoryModule(), - new AbstractModule() { - @Provides - @Singleton - @Named(CombinedUserDAO.DATABASE_INJECT_NAME) - public UsersDAO provideDatabaseUserDAO(PostgresUsersDAO postgresUsersDAO) { - return postgresUsersDAO; - } - })); - case DEFAULT -> new PostgresUsersRepositoryModule(); - }); + new UsersRepositoryModuleChooser( + DatabaseCombinedUserRequireModule.of(PostgresUsersDAO.class), + new PostgresUsersRepositoryModule()) + .chooseModule(configuration.usersRepositoryImplementation())); } private static Module chooseDeletedMessageVaultModules(PostgresTmailConfiguration configuration) { diff --git a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java index ca1066472b..b0df3195f7 100644 --- a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java +++ b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/CombinedIdentityTmailServerTest.java @@ -35,10 +35,10 @@ import org.junit.jupiter.api.extension.RegisterExtension; import com.google.inject.multibindings.Multibinder; +import com.linagora.tmail.UsersRepositoryModuleChooser; import com.linagora.tmail.combined.identity.CombinedUsersRepository; import com.linagora.tmail.combined.identity.LdapExtension; import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; -import com.linagora.tmail.combined.identity.UsersRepositoryModuleChooser; import com.linagora.tmail.encrypted.MailboxConfiguration; import com.linagora.tmail.encrypted.MailboxManagerClassProbe; diff --git a/tmail-backend/guice/common/pom.xml b/tmail-backend/guice/common/pom.xml new file mode 100644 index 0000000000..3c8ea278b2 --- /dev/null +++ b/tmail-backend/guice/common/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + + com.linagora.tmail + tmail-backend + 1.0.0-SNAPSHOT + ../../pom.xml + + + tmail-guice-common + Team-mail :: Guice :: Common + + + + ${project.groupId} + combined-identity + + + ${james.groupId} + james-server-guice-data-ldap + + + ${james.groupId} + james-server-guice-common + + + ${james.groupId} + james-server-guice-common + test-jar + test + + + ${james.groupId} + testing-base + test + + + \ No newline at end of file diff --git a/tmail-backend/guice/common/src/main/java/com/linagora/tmail/DatabaseCombinedUserRequireModule.java b/tmail-backend/guice/common/src/main/java/com/linagora/tmail/DatabaseCombinedUserRequireModule.java new file mode 100644 index 0000000000..d7dcc57248 --- /dev/null +++ b/tmail-backend/guice/common/src/main/java/com/linagora/tmail/DatabaseCombinedUserRequireModule.java @@ -0,0 +1,49 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package com.linagora.tmail; + +import org.apache.james.user.lib.UsersDAO; + +import com.google.inject.AbstractModule; +import com.google.inject.Injector; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.name.Named; +import com.linagora.tmail.combined.identity.CombinedUserDAO; + +public class DatabaseCombinedUserRequireModule extends AbstractModule { + public static DatabaseCombinedUserRequireModule of(Class typeUserDAOClass) { + return new DatabaseCombinedUserRequireModule<>(typeUserDAOClass); + } + + final Class typeUserDAOClass; + + public DatabaseCombinedUserRequireModule(Class typeUserDAOClass) { + this.typeUserDAOClass = typeUserDAOClass; + } + + @Provides + @Singleton + @Named(CombinedUserDAO.DATABASE_INJECT_NAME) + public UsersDAO provideDatabaseCombinedUserDAO(Injector injector) { + return injector.getInstance(typeUserDAOClass); + } + +} diff --git a/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java b/tmail-backend/guice/common/src/main/java/com/linagora/tmail/UsersRepositoryModuleChooser.java similarity index 60% rename from tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java rename to tmail-backend/guice/common/src/main/java/com/linagora/tmail/UsersRepositoryModuleChooser.java index 1d0890871e..520ddf603a 100644 --- a/tmail-backend/combined-identity/src/main/java/com/linagora/tmail/combined/identity/UsersRepositoryModuleChooser.java +++ b/tmail-backend/guice/common/src/main/java/com/linagora/tmail/UsersRepositoryModuleChooser.java @@ -17,56 +17,22 @@ * under the License. * ****************************************************************/ -/** - * This class is copied & adapted from {@link org.apache.james.data.UsersRepositoryModuleChooser} - */ +package com.linagora.tmail; -package com.linagora.tmail.combined.identity; -import java.util.List; import java.util.Optional; import org.apache.commons.configuration2.HierarchicalConfiguration; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.configuration2.tree.ImmutableNode; import org.apache.james.data.LdapUsersRepositoryModule; -import org.apache.james.modules.data.CassandraUsersRepositoryModule; import org.apache.james.server.core.configuration.FileConfigurationProvider; -import org.apache.james.user.cassandra.CassandraUsersDAO; -import org.apache.james.user.lib.UsersDAO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.google.common.collect.ImmutableList; -import com.google.inject.AbstractModule; import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.name.Named; +import com.google.inject.util.Modules; +import com.linagora.tmail.combined.identity.CombinedUsersRepositoryModule; public class UsersRepositoryModuleChooser { - - private static final Logger LOGGER = LoggerFactory.getLogger(UsersRepositoryModuleChooser.class); - - public static List chooseModules(Implementation implementation) { - return switch (implementation) { - case LDAP -> ImmutableList.of(new LdapUsersRepositoryModule()); - case COMBINED -> ImmutableList.of(new CombinedUsersRepositoryModule(), new AbstractModule() { - @Provides - @Singleton - @Named(CombinedUserDAO.DATABASE_INJECT_NAME) - public UsersDAO provideBackportUserDAO(CassandraUsersDAO cassandraUsersDAO) { - return cassandraUsersDAO; - } - }); - case DEFAULT -> ImmutableList.of(new CassandraUsersRepositoryModule()); - }; - } - - public static List chooseModules(FileConfigurationProvider fileConfigurationProvider) { - return chooseModules(Implementation.parse(fileConfigurationProvider)); - } - public enum Implementation { LDAP, COMBINED, @@ -86,9 +52,25 @@ public static Implementation parse(FileConfigurationProvider configurationProvid }) .orElse(DEFAULT); } catch (ConfigurationException e) { - LOGGER.warn("Error reading usersrepository.xml, defaulting to default implementation", e); - return Implementation.DEFAULT; + throw new RuntimeException("Error reading usersrepository.xml", e); } } } + + private final DatabaseCombinedUserRequireModule databaseCombinedUserRequireModule; + + private final Module defaultModule; + + public UsersRepositoryModuleChooser(DatabaseCombinedUserRequireModule databaseCombinedUserRequireModule, Module defaultModule) { + this.databaseCombinedUserRequireModule = databaseCombinedUserRequireModule; + this.defaultModule = defaultModule; + } + + public Module chooseModule(Implementation implementation) { + return switch (implementation) { + case LDAP -> new LdapUsersRepositoryModule(); + case COMBINED -> Modules.override(defaultModule).with(Modules.combine(new CombinedUsersRepositoryModule(), databaseCombinedUserRequireModule)); + case DEFAULT -> defaultModule; + }; + } } diff --git a/tmail-backend/pom.xml b/tmail-backend/pom.xml index 18e43922fd..21b4149b5e 100644 --- a/tmail-backend/pom.xml +++ b/tmail-backend/pom.xml @@ -30,6 +30,7 @@ deployment-tests guice/blob-guice + guice/common guice/distributed guice/encrypted-mailbox guice/protocols/jmap @@ -171,6 +172,11 @@ tmail-common ${project.version} + + ${project.groupId} + tmail-guice-common + ${project.version} + ${project.groupId} tmail-guice-distributed From 8804d4385729f50a2ee1ecfcf8fe13afd61759b7 Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Tue, 23 Jan 2024 10:12:43 +0700 Subject: [PATCH 10/81] ISSUE-895 Update correct configuration file for deployment-test --- .../deployment/TmailPostgresExtension.java | 3 + .../test/resources/james-conf/blob.properties | 104 ++++++++++++++++++ .../james-conf/opensearch.properties | 101 +++++++++++++++++ .../resources/james-conf/rabbitmq.properties | 103 +++++++++++++++++ 4 files changed, 311 insertions(+) create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/blob.properties create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/opensearch.properties create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/rabbitmq.properties diff --git a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java index 2a59deffad..8f2d57e637 100644 --- a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java +++ b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java @@ -52,6 +52,9 @@ private GenericContainer createTmailDistributed() { .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jwt_publickey"), "/root/conf/") .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/keystore"), "/root/conf/") .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jmxremote.password"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/blob.properties"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/rabbitmq.properties"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/opensearch.properties"), "/root/conf/") .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-postgres-testing" + UUID.randomUUID())) .waitingFor(TestContainerWaitStrategy.WAIT_STRATEGY) .withExposedPorts(25, 143, 80, 8000); diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/blob.properties b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/blob.properties new file mode 100644 index 0000000000..a02854b231 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/blob.properties @@ -0,0 +1,104 @@ +# ============================================= BlobStore Implementation ================================== +# Read https://james.apache.org/server/config-blobstore.html for further details + +# Choose your BlobStore implementation +# Mandatory, allowed values are: file, s3, postgres. +implementation=s3 + +# ========================================= Deduplication ======================================== +# If you choose to enable deduplication, the mails with the same content will be stored only once. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to the deletion of all +# the mails sharing the same content once one is deleted. +# Mandatory, Allowed values are: true, false +deduplication.enable=true + +# deduplication.family needs to be incremented every time the deduplication.generation.duration is changed +# Positive integer, defaults to 1 +# deduplication.gc.generation.family=1 + +# Duration of generation. +# Deduplication only takes place within a singe generation. +# Only items two generation old can be garbage collected. (This prevent concurrent insertions issues and +# accounts for a clock skew). +# deduplication.family needs to be incremented everytime this parameter is changed. +# Duration. Default unit: days. Defaults to 30 days. +# deduplication.gc.generation.duration=30days + +# ========================================= Encryption ======================================== +# If you choose to enable encryption, the blob content will be encrypted before storing them in the BlobStore. +# Warning: Once this feature is enabled, there is no turning back as turning it off will lead to all content being +# encrypted. This comes at a performance impact but presents you from leaking data if, for instance the third party +# offering you a S3 service is compromised. +# Optional, Allowed values are: true, false, defaults to false +encryption.aes.enable=false + +# Mandatory (if AES encryption is enabled) salt and password. Salt needs to be an hexadecimal encoded string +#encryption.aes.password=xxx +#encryption.aes.salt=73616c7479 +# Optional, defaults to PBKDF2WithHmacSHA512 +#encryption.aes.private.key.algorithm=PBKDF2WithHmacSHA512 + +# ============================================== ObjectStorage ============================================ + +# ========================================= ObjectStorage Buckets ========================================== +# bucket names prefix +# Optional, default no prefix +# objectstorage.bucketPrefix=prod- + +# Default bucket name +# Optional, default is bucketPrefix + `default` +# objectstorage.namespace=james + +# ========================================= ObjectStorage on S3 ============================================= +# Mandatory if you choose s3 storage service, S3 authentication endpoint +objectstorage.s3.endPoint=${env:OBJECT_STORAGE_ENDPOINT:-http://s3.docker.test:8000/} + +# Mandatory if you choose s3 storage service, S3 region +#objectstorage.s3.region=eu-west-1 +objectstorage.s3.region=${env:OBJECT_STORAGE_REGION:-eu-west-1} + +# Mandatory if you choose aws-s3 storage service, access key id configured in S3 +objectstorage.s3.accessKeyId=${env:OBJECT_STORAGE_ACCESS_KEY_ID:-accessKey1} + +# Mandatory if you choose s3 storage service, secret key configured in S3 +objectstorage.s3.secretKey=${env:OBJECT_STORAGE_SECRET_KEY:-secretKey1} + +# Optional if you choose s3 storage service: The trust store file, secret, and algorithm to use +# when connecting to the storage service. If not specified falls back to Java defaults. +#objectstorage.s3.truststore.path= +#objectstorage.s3.truststore.type=JKS +#objectstorage.s3.truststore.secret= +#objectstorage.s3.truststore.algorithm=SunX509 + + +# optional: Object read in memory will be rejected if they exceed the size limit exposed here. Size, exemple `100M`. +# Supported units: K, M, G, defaults to B if no unit is specified. If unspecified, big object won't be prevented +# from being loaded in memory. This settings complements protocol limits. +# objectstorage.s3.in.read.limit=50M + +# ============================================ Blobs Exporting ============================================== +# Read https://james.apache.org/server/config-blob-export.html for further details + +# Choosing blob exporting mechanism, allowed mechanism are: localFile, linshare +# LinShare is a file sharing service, will be explained in the below section +# Optional, default is localFile +blob.export.implementation=localFile + +# ======================================= Local File Blobs Exporting ======================================== +# Optional, directory to store exported blob, directory path follows James file system format +# default is file://var/blobExporting +blob.export.localFile.directory=file://var/blobExporting + +# ======================================= LinShare File Blobs Exporting ======================================== +# LinShare is a sharing service where you can use james, connects to an existing LinShare server and shares files to +# other mail addresses as long as those addresses available in LinShare. For example you can deploy James and LinShare +# sharing the same LDAP repository +# Mandatory if you choose LinShare, url to connect to LinShare service +# blob.export.linshare.url=http://linshare:8080 + +# ======================================= LinShare Configuration BasicAuthentication =================================== +# Authentication is mandatory if you choose LinShare, TechnicalAccount is need to connect to LinShare specific service. +# For Example: It will be formalized to 'Authorization: Basic {Credential of UUID/password}' + +# blob.export.linshare.technical.account.uuid=Technical_Account_UUID +# blob.export.linshare.technical.account.password=password diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/opensearch.properties b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/opensearch.properties new file mode 100644 index 0000000000..df261c5dee --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/opensearch.properties @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This template file can be used as example for James Server configuration +# DO NOT USE IT AS SUCH AND ADAPT IT TO YOUR NEEDS + +# Configuration file for OpenSearch +# Read https://james.apache.org/server/config-opensearch.html for further details + +opensearch.masterHost=opensearch +opensearch.port=9200 + +# Optional. Only http or https are accepted, default is http +# opensearch.hostScheme=http + +# Optional, default is `default` +# Choosing the SSL check strategy when using https scheme +# default: Use the default SSL TrustStore of the system. +# ignore: Ignore SSL Validation check (not recommended). +# override: Override the SSL Context to use a custom TrustStore containing ES server's certificate. +# opensearch.hostScheme.https.sslValidationStrategy=default + +# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy +# Configure OpenSearch rest client to use this trustStore file to recognize nginx's ssl certificate. +# You need to specify both trustStorePath and trustStorePassword +# opensearch.hostScheme.https.trustStorePath=/file/to/trust/keystore.jks + +# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy +# Configure OpenSearch rest client to use this trustStore file with the specified password. +# You need to specify both trustStorePath and trustStorePassword +# opensearch.hostScheme.https.trustStorePassword=myJKSPassword + +# Optional. default is `default` +# Configure OpenSearch rest client to use host name verifier during SSL handshake +# default: using the default hostname verifier provided by apache http client. +# accept_any_hostname: accept any host (not recommended). +# opensearch.hostScheme.https.hostNameVerifier=default + +# Optional. +# Basic auth username to access opensearch. +# Ignore opensearch.user and opensearch.password to not be using authentication (default behaviour). +# Otherwise, you need to specify both properties. +# opensearch.user=elasticsearch + +# Optional. +# Basic auth password to access opensearch. +# Ignore opensearch.user and opensearch.password to not be using authentication (default behaviour). +# Otherwise, you need to specify both properties. +# opensearch.password=secret + +# You can alternatively provide a list of hosts following this format : +# opensearch.hosts=host1:9200,host2:9200 +# opensearch.clusterName=cluster + +opensearch.nb.shards=5 +opensearch.nb.replica=1 +opensearch.index.waitForActiveShards=1 +opensearch.retryConnection.maxRetries=7 +opensearch.retryConnection.minDelay=3000 +# Index or not attachments (default value: true) +# Note: Attachments not implemented yet for postgresql, false for now +opensearch.indexAttachments=false + +# Search overrides allow resolution of predefined search queries against alternative sources of data +# and allow bypassing opensearch. This is useful to handle most resynchronisation queries that +# are simple enough to be resolved against Cassandra. +# +# Possible values are: +# - `org.apache.james.mailbox.postgres.search.AllSearchOverride` Some IMAP clients uses SEARCH ALL to fully list messages in +# a mailbox and detect deletions. This is typically done by clients not supporting QRESYNC and from an IMAP perspective +# is considered an optimisation as less data is transmitted compared to a FETCH command. Resolving such requests against +# Postgresql is enabled by this search override and likely desirable. +# - `org.apache.james.mailbox.postgres.search.UidSearchOverride`. Same as above but restricted by ranges. +# - `org.apache.james.mailbox.postgres.search.DeletedSearchOverride`. Find deleted messages by looking up in the relevant Postgresql +# table. +# - `org.apache.james.mailbox.postgres.search.DeletedWithRangeSearchOverride`. Same as above but limited by ranges. +# - `org.apache.james.mailbox.postgres.search.NotDeletedWithRangeSearchOverride`. List non deleted messages in a given range. +# Lists all messages and filters out deleted message thus this is based on the following heuristic: most messages are not marked as deleted. +# - `org.apache.james.mailbox.postgres.search.UnseenSearchOverride`. List unseen messages in the corresponding Postgresql index. +# +# Please note that custom overrides can be defined here. +# +# opensearch.search.overrides=org.apache.james.mailbox.postgres.search.AllSearchOverride,org.apache.james.mailbox.postgres.search.DeletedSearchOverride, org.apache.james.mailbox.postgres.search.DeletedWithRangeSearchOverride,org.apache.james.mailbox.postgres.search.NotDeletedWithRangeSearchOverride,org.apache.james.mailbox.postgres.search.UidSearchOverride,org.apache.james.mailbox.postgres.search.UnseenSearchOverride + +# Optional. Default is `false` +# When set to true, James will attempt to reindex from the indexed message when moved. If the message is not found, it will fall back to the old behavior (The message will be indexed from the blobStore source) +# opensearch.message.index.optimize.move=false \ No newline at end of file diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/rabbitmq.properties b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/rabbitmq.properties new file mode 100644 index 0000000000..c8b3f52602 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/rabbitmq.properties @@ -0,0 +1,103 @@ +# RabbitMQ configuration + +# Read https://james.apache.org/server/config-rabbitmq.html for further details + +# Mandatory +uri=amqp://rabbitmq:5672 +# If you use a vhost, specify it as well at the end of the URI +# uri=amqp://rabbitmq:5672/vhost + +# Vhost to use for creating queues and exchanges +# Optional, only use this if you have invalid URIs containing characters like '_' +# vhost=vhost1 + +# Optional, default to the host specified as part of the URI. +# Allow creating cluster aware connections. +# hosts=ip1:5672,ip2:5672 + +# RabbitMQ Administration Management +# Mandatory +management.uri=http://rabbitmq:15672 +# Mandatory +management.user=guest +# Mandatory +management.password=guest + +# Configure retries count to retrieve a connection. Exponential backoff is performed between each retries. +# Optional integer, defaults to 10 +#connection.pool.retries=10 +# Configure initial duration (in ms) between two connection retries. Exponential backoff is performed between each retries. +# Optional integer, defaults to 100 +#connection.pool.min.delay.ms=100 +# Configure retries count to retrieve a channel. Exponential backoff is performed between each retries. +# Optional integer, defaults to 3 +#channel.pool.retries=3 +# Configure timeout duration (in ms) to obtain a rabbitmq channel. Defaults to 30 seconds. +# Optional integer, defaults to 30 seconds. +#channel.pool.max.delay.ms=30000 +# Configure the size of the channel pool. +# Optional integer, defaults to 3 +#channel.pool.size=3 + +# Boolean. Whether to activate Quorum queue usage for use cases that benefits from it (work queue). +# Quorum queues enables high availability. +# False (default value) results in the usage of classic queues. +#quorum.queues.enable=true + +# Strictly positive integer. The replication factor to use when creating quorum queues. +#quorum.queues.replication.factor + +# Parameters for the Cassandra administrative view + +# Whether the Cassandra administrative view should be activated. Boolean value defaulting to true. +# Not necessarily needed for MDA deployments, mail queue management adds significant complexity. +# cassandra.view.enabled=true + +# Period of the window. Too large values will lead to wide rows while too little values might lead to many queries. +# Use the number of mail per Cassandra row, along with your expected traffic, to determine this value +# This value can only be decreased to a value dividing the current value +# Optional, default 1h +mailqueue.view.sliceWindow=1h + +# Use to distribute the emails of a given slice within your cassandra cluster +# A good value is 2*cassandraNodeCount +# This parameter can only be increased. +# Optional, default 1 +mailqueue.view.bucketCount=1 + +# Determine the probability to update the browse start pointer +# Too little value will lead to unnecessary reads. Too big value will lead to more expensive browse. +# Choose this parameter so that it get's update one time every one-two sliceWindow +# Optional, default 1000 +mailqueue.view.updateBrowseStartPace=1000 + +# Enables or disables the gauge metric on the mail queue size +# Computing the size of the mail queue is currently implemented on top of browse operation and thus have a linear complexity +# Metrics get exported periodically as configured in opensearch.properties, thus getSize is also called periodically +# Choose to disable it when the mail queue size is getting too big +# Note that this is as well a temporary workaround until we get 'getSize' method better optimized +# Optional, default false +mailqueue.size.metricsEnabled=false + +# Whether to enable task consumption on this node. Tasks are WebAdmin triggered long running jobs. +# Disable with caution (this only makes sense in a distributed setup where other nodes consume tasks). +# Defaults to true. +task.consumption.enabled=true + +# Configure task queue consumer timeout. References: https://www.rabbitmq.com/consumers.html#acknowledgement-timeout. Required at least RabbitMQ version 3.12 to have effect. +# This is used to avoid the task queue consumer (which could run very long tasks) being disconnected by RabbitMQ after the default acknowledgement timeout 30 minutes. +# Optional. Duration (support multiple time units cf `DurationParser`), defaults to 1 day. +#task.queue.consumer.timeout=1day + +# Configure queue ttl (in ms). References: https://www.rabbitmq.com/ttl.html#queue-ttl. +# This is used only on queues used to share notification patterns, are exclusive to a node. If omitted, it will not add the TTL configure when declaring queues. +# Optional integer, defaults is 3600000. +#notification.queue.ttl=3600000 + +# AMQP resources parameters for the subscriber email address contact messages. In order to synchronize contacts between Team-mail backend and 3rd. +# AQMP uri +address.contact.uri=amqp://rabbitmq:5672 +address.contact.user=guest +address.contact.password=guest +# Queue name +address.contact.queue=AddressContactQueue1 \ No newline at end of file From d2bf2054720d3fc3397bc27722b79db770c356ac Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Tue, 23 Jan 2024 10:44:37 +0700 Subject: [PATCH 11/81] ISSUE-903 [CI build] Postgres experimental branch pushes also postgres image to docker hub on latest tag --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d084e04764..1cc41a5b5c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,7 +52,7 @@ pipeline { script { // build and push docker images dir("tmail-backend") { - sh "mvn -Pci jib:build -Djib.to.auth.username=${DOCKER_HUB_CREDENTIAL_USR} -Djib.to.auth.password=${DOCKER_HUB_CREDENTIAL_PSW} -Djib.to.image=linagora/tmail-backend:postgresql-experimental -pl apps/postgres -X" + sh "mvn -Pci jib:build -Djib.to.auth.username=${DOCKER_HUB_CREDENTIAL_USR} -Djib.to.auth.password=${DOCKER_HUB_CREDENTIAL_PSW} -Djib.to.image=linagora/tmail-backend:postgresql-experimental -Djib.to.tags=postgresql-experimental -pl apps/postgres -X" } } } From 52217defdf2ee96d4b7a378504a34a630b8d282f Mon Sep 17 00:00:00 2001 From: Tung Tran Date: Mon, 29 Jan 2024 11:31:57 +0700 Subject: [PATCH 12/81] ISSUE-883 Add deployment test for combined user with postgres --- .../PostgresLdapImapAndSmtpTest.java | 21 ++++++ .../TmailLdapPostgresExtension.java | 67 +++++++++++++++++++ .../deployment/TmailPostgresExtension.java | 20 +++--- .../resources/james-conf/usersrepository.xml | 13 ++++ .../resources/prepopulated-ldap/Dockerfile | 3 + .../resources/prepopulated-ldap/populate.ldif | 43 ++++++++++++ 6 files changed, 157 insertions(+), 10 deletions(-) create mode 100644 tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresLdapImapAndSmtpTest.java create mode 100644 tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailLdapPostgresExtension.java create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/usersrepository.xml create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/Dockerfile create mode 100644 tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/populate.ldif diff --git a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresLdapImapAndSmtpTest.java b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresLdapImapAndSmtpTest.java new file mode 100644 index 0000000000..5ee28accbd --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/PostgresLdapImapAndSmtpTest.java @@ -0,0 +1,21 @@ +package com.linagora.tmail.deployment; + +import org.apache.james.mpt.imapmailbox.external.james.host.external.ExternalJamesConfiguration; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.GenericContainer; + +public class PostgresLdapImapAndSmtpTest extends ImapAndSmtpContract { + + @RegisterExtension + TmailLdapPostgresExtension extension = new TmailLdapPostgresExtension(); + + @Override + protected ExternalJamesConfiguration configuration() { + return extension.configuration(); + } + + @Override + protected GenericContainer container() { + return extension.getContainer(); + } +} diff --git a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailLdapPostgresExtension.java b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailLdapPostgresExtension.java new file mode 100644 index 0000000000..42b77f3b5c --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailLdapPostgresExtension.java @@ -0,0 +1,67 @@ +package com.linagora.tmail.deployment; + +import java.time.Duration; +import java.util.UUID; + +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +public class TmailLdapPostgresExtension extends TmailPostgresExtension { + private GenericContainer ldap; + + public TmailLdapPostgresExtension() { + ldap = createLdap(network); + } + + @SuppressWarnings("resource") + @Override + public GenericContainer createTmailDistributed() { + if (ldap == null) { + ldap = createLdap(network); + } + + return new GenericContainer<>("linagora/tmail-backend-postgresql-experimental:latest") + .withNetworkAliases("tmail-postgres") + .withNetwork(network) + .dependsOn(postgres, opensearch, s3, rabbitmq, ldap) + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/imapserver.xml"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/usersrepository.xml"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jwt_privatekey"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jwt_publickey"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/keystore"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/jmxremote.password"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/blob.properties"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/rabbitmq.properties"), "/root/conf/") + .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/opensearch.properties"), "/root/conf/") + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-backend-postgres-testing" + UUID.randomUUID())) + .waitingFor(TestContainerWaitStrategy.WAIT_STRATEGY) + .withExposedPorts(25, 143, 80, 8000); + } + + private GenericContainer createLdap(Network network) { + return new GenericContainer<>( + new ImageFromDockerfile() + .withFileFromClasspath("populate.ldif", "prepopulated-ldap/populate.ldif") + .withFileFromClasspath("Dockerfile", "prepopulated-ldap/Dockerfile")) + .withNetworkAliases("ldap") + .withNetwork(network) + .withEnv("LDAP_DOMAIN", "james.org") + .withEnv("LDAP_ADMIN_PASSWORD", "secret") + .withCommand("--copy-service --loglevel debug") + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-openldap-testing" + UUID.randomUUID())) + .waitingFor(new LogMessageWaitStrategy().withRegEx(".*slapd starting\\n").withTimes(1) + .withStartupTimeout(Duration.ofMinutes(3))) + .withStartupCheckStrategy(new MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(10))); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + super.afterEach(extensionContext); + ldap.stop(); + } +} diff --git a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java index 8f2d57e637..3fbe7dd700 100644 --- a/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java +++ b/tmail-backend/deployment-tests/postgres/src/test/java/com/linagora/tmail/deployment/TmailPostgresExtension.java @@ -25,12 +25,12 @@ import org.testcontainers.utility.MountableFile; public class TmailPostgresExtension implements BeforeEachCallback, AfterEachCallback { - private final Network network; - private final GenericContainer postgres; - private final GenericContainer opensearch; - private final GenericContainer rabbitmq; - private final GenericContainer s3; - private final GenericContainer tmailBackend; + protected final Network network; + protected final GenericContainer postgres; + protected final GenericContainer opensearch; + protected final GenericContainer rabbitmq; + protected final GenericContainer s3; + protected final GenericContainer tmailBackend; public TmailPostgresExtension() { network = Network.newNetwork(); @@ -42,7 +42,7 @@ public TmailPostgresExtension() { } @SuppressWarnings("resource") - private GenericContainer createTmailDistributed() { + protected GenericContainer createTmailDistributed() { return new GenericContainer<>("linagora/tmail-backend-postgresql-experimental:latest") .withNetworkAliases("tmail-postgres") .withNetwork(network) @@ -55,20 +55,20 @@ private GenericContainer createTmailDistributed() { .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/blob.properties"), "/root/conf/") .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/rabbitmq.properties"), "/root/conf/") .withCopyFileToContainer(MountableFile.forClasspathResource("james-conf/opensearch.properties"), "/root/conf/") - .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-postgres-testing" + UUID.randomUUID())) + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-backend-postgres-testing" + UUID.randomUUID())) .waitingFor(TestContainerWaitStrategy.WAIT_STRATEGY) .withExposedPorts(25, 143, 80, 8000); } @SuppressWarnings("resource") - private GenericContainer createPostgres(Network network) { + protected static GenericContainer createPostgres(Network network) { return new GenericContainer<>("postgres:16.1") .withNetworkAliases("postgres") .withNetwork(network) .withEnv("POSTGRES_USER", "tmail") .withEnv("POSTGRES_PASSWORD", "secret1") .withEnv("POSTGRES_DB", "postgres") - .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-postgres-testing" + UUID.randomUUID())) + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("team-mail-postgres-database-testing" + UUID.randomUUID())) .waitingFor((new LogMessageWaitStrategy()).withRegEx(".*database system is ready to accept connections.*\\s").withTimes(2).withStartupTimeout(Duration.of(60L, ChronoUnit.SECONDS))) .withExposedPorts(5432); } diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/usersrepository.xml b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/usersrepository.xml new file mode 100644 index 0000000000..8ba7bae515 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/james-conf/usersrepository.xml @@ -0,0 +1,13 @@ + + + SHA-512 + true + true + \ No newline at end of file diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/Dockerfile b/tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/Dockerfile new file mode 100644 index 0000000000..b76bb3da24 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/Dockerfile @@ -0,0 +1,3 @@ +FROM osixia/openldap:1.5.0 + +COPY populate.ldif /container/service/slapd/assets/config/bootstrap/ldif/data.ldif diff --git a/tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/populate.ldif b/tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/populate.ldif new file mode 100644 index 0000000000..3d3b1e3421 --- /dev/null +++ b/tmail-backend/deployment-tests/postgres/src/test/resources/prepopulated-ldap/populate.ldif @@ -0,0 +1,43 @@ +dn: ou=people, dc=james,dc=org +ou: people +objectClass: organizationalUnit + +dn: ou=empty, dc=james,dc=org +ou: empty +objectClass: organizationalUnit + +dn: uid=james-user, ou=people, dc=james,dc=org +objectClass: inetOrgPerson +uid: james-user +cn: james-user +sn: james-user +mail: james-user@james.org +userPassword: secret +description: James user + +dn: uid=bob, ou=people, dc=james,dc=org +objectClass: inetOrgPerson +uid: bob +cn: bob +sn: bob +mail: bob@domain.tld +userPassword: bobpassword +description: bob + +dn: uid=alice, ou=people, dc=james,dc=org +objectClass: inetOrgPerson +uid: alice +cn: alice +sn: alice +mail: alice@domain.tld +userPassword: alicepassword +description: alice + +dn: uid=imapuser, ou=people, dc=james,dc=org +objectClass: inetOrgPerson +uid: imapuser +cn: imapuser +sn: imapuser +mail: imapuser@domain +userPassword: password +description: imapuser From c12ce1c1faf348fef9327d0e1132ded85b3c2c75 Mon Sep 17 00:00:00 2001 From: hung phan Date: Thu, 7 Mar 2024 14:57:22 +0700 Subject: [PATCH 13/81] ISSUE-918 JMAP memory binding for Postgres TMail app --- tmail-backend/apps/postgres/pom.xml | 4 + .../tmail/james/app/PostgresTmailServer.java | 113 +++++++++++++++--- .../james/app/PostgresTmailServerTest.java | 5 +- .../src/test/resources/mailetcontainer.xml | 8 ++ 4 files changed, 112 insertions(+), 18 deletions(-) diff --git a/tmail-backend/apps/postgres/pom.xml b/tmail-backend/apps/postgres/pom.xml index 156a2a204f..cc2cf3a1a6 100644 --- a/tmail-backend/apps/postgres/pom.xml +++ b/tmail-backend/apps/postgres/pom.xml @@ -142,6 +142,10 @@ test-jar test + + ${project.groupId} + mailbox-encrypted-memory + ${project.groupId} tmail-healthcheck diff --git a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java index 203d54b884..1c12ce499e 100644 --- a/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java +++ b/tmail-backend/apps/postgres/src/main/java/com/linagora/tmail/james/app/PostgresTmailServer.java @@ -1,10 +1,15 @@ package com.linagora.tmail.james.app; +import static org.apache.james.PostgresJamesServerMain.JMAP; + +import java.util.List; + import org.apache.james.ExtraProperties; import org.apache.james.GuiceJamesServer; import org.apache.james.JamesServerMain; import org.apache.james.SearchConfiguration; import org.apache.james.SearchModuleChooser; +import org.apache.james.jmap.draft.JMAPListenerModule; import org.apache.james.modules.BlobExportMechanismModule; import org.apache.james.modules.MailboxModule; import org.apache.james.modules.MailetProcessingModule; @@ -13,11 +18,12 @@ import org.apache.james.modules.blobstore.BlobStoreModulesChooser; import org.apache.james.modules.data.PostgresDataModule; import org.apache.james.modules.data.PostgresDelegationStoreModule; +import org.apache.james.modules.data.PostgresEventStoreModule; import org.apache.james.modules.data.PostgresUsersRepositoryModule; +import org.apache.james.modules.data.PostgresVacationModule; import org.apache.james.modules.data.SievePostgresRepositoryModules; import org.apache.james.modules.event.RabbitMQEventBusModule; import org.apache.james.modules.events.PostgresDeadLetterModule; -import org.apache.james.modules.eventstore.MemoryEventStoreModule; import org.apache.james.modules.mailbox.DefaultEventModule; import org.apache.james.modules.mailbox.PostgresDeletedMessageVaultModule; import org.apache.james.modules.mailbox.PostgresMailboxModule; @@ -31,14 +37,11 @@ import org.apache.james.modules.queue.activemq.ActiveMQQueueModule; import org.apache.james.modules.queue.rabbitmq.RabbitMQModule; import org.apache.james.modules.server.DataRoutesModules; -import org.apache.james.modules.server.DefaultProcessorsConfigurationProviderModule; import org.apache.james.modules.server.InconsistencyQuotasSolvingRoutesModule; import org.apache.james.modules.server.JMXServerModule; import org.apache.james.modules.server.MailQueueRoutesModule; import org.apache.james.modules.server.MailRepositoriesRoutesModule; import org.apache.james.modules.server.MailboxRoutesModule; -import org.apache.james.modules.server.NoJwtModule; -import org.apache.james.modules.server.RawPostDequeueDecoratorModule; import org.apache.james.modules.server.ReIndexingModule; import org.apache.james.modules.server.SieveRoutesModule; import org.apache.james.modules.server.TaskManagerModule; @@ -54,7 +57,39 @@ import com.google.inject.util.Modules; import com.linagora.tmail.DatabaseCombinedUserRequireModule; import com.linagora.tmail.UsersRepositoryModuleChooser; +import com.linagora.tmail.encrypted.InMemoryEncryptedEmailContentStoreModule; +import com.linagora.tmail.encrypted.KeystoreMemoryModule; +import com.linagora.tmail.james.jmap.contact.MemoryEmailAddressContactModule; +import com.linagora.tmail.james.jmap.firebase.FirebaseCommonModule; +import com.linagora.tmail.james.jmap.firebase.FirebaseModuleChooserConfiguration; +import com.linagora.tmail.james.jmap.firebase.MemoryFirebaseSubscriptionRepository; +import com.linagora.tmail.james.jmap.label.MemoryLabelRepositoryModule; +import com.linagora.tmail.james.jmap.method.CalendarEventMethodModule; +import com.linagora.tmail.james.jmap.method.ContactAutocompleteMethodModule; +import com.linagora.tmail.james.jmap.method.CustomMethodModule; +import com.linagora.tmail.james.jmap.method.EmailRecoveryActionMethodModule; +import com.linagora.tmail.james.jmap.method.EmailSendMethodModule; +import com.linagora.tmail.james.jmap.method.EncryptedEmailDetailedViewGetMethodModule; +import com.linagora.tmail.james.jmap.method.EncryptedEmailFastViewGetMethodModule; +import com.linagora.tmail.james.jmap.method.FilterGetMethodModule; +import com.linagora.tmail.james.jmap.method.FilterSetMethodModule; +import com.linagora.tmail.james.jmap.method.ForwardGetMethodModule; +import com.linagora.tmail.james.jmap.method.ForwardSetMethodModule; +import com.linagora.tmail.james.jmap.method.JmapSettingsMethodModule; +import com.linagora.tmail.james.jmap.method.KeystoreGetMethodModule; +import com.linagora.tmail.james.jmap.method.KeystoreSetMethodModule; +import com.linagora.tmail.james.jmap.method.LabelMethodModule; +import com.linagora.tmail.james.jmap.oidc.WebFingerModule; +import com.linagora.tmail.james.jmap.settings.MemoryJmapSettingsRepositoryModule; +import com.linagora.tmail.james.jmap.team.mailboxes.TeamMailboxJmapModule; +import com.linagora.tmail.james.jmap.ticket.TicketRoutesModule; +import com.linagora.tmail.rate.limiter.api.memory.MemoryRateLimitingModule; import com.linagora.tmail.rspamd.RspamdModule; +import com.linagora.tmail.team.TeamMailboxModule; +import com.linagora.tmail.webadmin.EmailAddressContactRoutesModule; +import com.linagora.tmail.webadmin.RateLimitPlanRoutesModule; +import com.linagora.tmail.webadmin.archival.InboxArchivalTaskModule; +import com.linagora.tmail.webadmin.cleanup.MailboxesCleanupModule; public class PostgresTmailServer { static Logger LOGGER = LoggerFactory.getLogger("org.apache.james.CONFIGURATION"); @@ -78,6 +113,7 @@ public static GuiceJamesServer createServer(PostgresTmailConfiguration configura SearchConfiguration searchConfiguration = configuration.searchConfiguration(); return GuiceJamesServer.forConfiguration(configuration) + .combineWith(POSTGRES_MODULE_AGGREGATE) .combineWith(SearchModuleChooser.chooseModules(searchConfiguration)) .combineWith(chooseUserRepositoryModule(configuration)) .combineWith(chooseBlobStoreModules(configuration)) @@ -85,10 +121,10 @@ public static GuiceJamesServer createServer(PostgresTmailConfiguration configura .combineWith(chooseDeletedMessageVaultModules(configuration)) .combineWith(chooseRedisRateLimiterModule(configuration)) .combineWith(chooseRspamdModule(configuration)) - .combineWith(POSTGRES_MODULE_AGGREGATE); + .combineWith(chooseFirebase(configuration.firebaseModuleChooserConfiguration())) + .overrideWith(chooseJmapModule(configuration)); } - private static final Module WEBADMIN = Modules.combine( new WebAdminServerModule(), new DataRoutesModules(), @@ -109,23 +145,54 @@ public static GuiceJamesServer createServer(PostgresTmailConfiguration configura new SMTPServerModule(), WEBADMIN); + public static final Module JMAP_LINAGORA = Modules.override( + JMAP, + new CalendarEventMethodModule(), + new ContactAutocompleteMethodModule(), + new CustomMethodModule(), + new EncryptedEmailDetailedViewGetMethodModule(), + new EncryptedEmailFastViewGetMethodModule(), + new EmailSendMethodModule(), + new FilterGetMethodModule(), + new FilterSetMethodModule(), + new ForwardGetMethodModule(), + new InMemoryEncryptedEmailContentStoreModule(), + new KeystoreMemoryModule(), + new ForwardSetMethodModule(), + new KeystoreSetMethodModule(), + new KeystoreGetMethodModule(), + new TicketRoutesModule(), + new WebFingerModule(), + new EmailRecoveryActionMethodModule(), + new LabelMethodModule(), + new JmapSettingsMethodModule(), + new MailboxesCleanupModule(), + new InboxArchivalTaskModule()) + .with(new TeamMailboxJmapModule()); + private static final Module POSTGRES_SERVER_MODULE = Modules.combine( new ActiveMQQueueModule(), new BlobExportMechanismModule(), new PostgresDelegationStoreModule(), - new DefaultProcessorsConfigurationProviderModule(), new PostgresMailboxModule(), new PostgresDeadLetterModule(), new PostgresDataModule(), new MailboxModule(), - new NoJwtModule(), - new RawPostDequeueDecoratorModule(), new SievePostgresRepositoryModules(), new TaskManagerModule(), - new MemoryEventStoreModule(), - new TikaMailboxModule()); - private static final Module POSTGRES_MODULE_AGGREGATE = Modules.combine( - new MailetProcessingModule(), POSTGRES_SERVER_MODULE, PROTOCOLS); + new PostgresEventStoreModule(), + new TikaMailboxModule(), + new PostgresVacationModule()); + + private static final Module POSTGRES_MODULE_AGGREGATE = Modules.override(Modules.combine( + new MailetProcessingModule(), POSTGRES_SERVER_MODULE, PROTOCOLS, JMAP_LINAGORA)) + .with(new TeamMailboxModule(), + new MemoryRateLimitingModule(), + new RateLimitPlanRoutesModule(), + new MemoryEmailAddressContactModule(), + new EmailAddressContactRoutesModule(), + new MemoryLabelRepositoryModule(), + new MemoryJmapSettingsRepositoryModule()); private static Module chooseBlobStoreModules(PostgresTmailConfiguration configuration) { return Modules.combine(Modules.combine(BlobStoreModulesChooser.chooseModules(configuration.blobStoreConfiguration())), @@ -150,10 +217,7 @@ public static Module chooseUserRepositoryModule(PostgresTmailConfiguration confi } private static Module chooseDeletedMessageVaultModules(PostgresTmailConfiguration configuration) { - if (configuration.deletedMessageVaultConfiguration().isEnabled()) { - return Modules.combine(new PostgresDeletedMessageVaultModule(), new DeletedMessageVaultRoutesModule()); - } - return Modules.EMPTY_MODULE; + return Modules.combine(new PostgresDeletedMessageVaultModule(), new DeletedMessageVaultRoutesModule()); } private static Module chooseRedisRateLimiterModule(PostgresTmailConfiguration configuration) { @@ -169,4 +233,19 @@ private static Module chooseRspamdModule(PostgresTmailConfiguration configuratio } return Modules.EMPTY_MODULE; } + + private static Module chooseJmapModule(PostgresTmailConfiguration configuration) { + if (configuration.jmapEnabled()) { + return new JMAPListenerModule(); + } + return binder -> { + }; + } + + private static List chooseFirebase(FirebaseModuleChooserConfiguration moduleChooserConfiguration) { + if (moduleChooserConfiguration.enable()) { + return List.of(new MemoryFirebaseSubscriptionRepository.Module(), new FirebaseCommonModule()); + } + return List.of(); + } } diff --git a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java index 0ed56eeaae..c0cc7408e9 100644 --- a/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java +++ b/tmail-backend/apps/postgres/src/test/java/com/linagora/tmail/james/app/PostgresTmailServerTest.java @@ -7,6 +7,7 @@ import org.apache.james.JamesServerExtension; import org.apache.james.SearchConfiguration; import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.jmap.draft.JmapJamesServerContract; import org.apache.james.modules.blobstore.BlobStoreConfiguration; import org.apache.james.utils.GuiceProbe; import org.junit.jupiter.api.extension.RegisterExtension; @@ -15,8 +16,9 @@ import com.linagora.tmail.combined.identity.UsersRepositoryClassProbe; import com.linagora.tmail.encrypted.MailboxConfiguration; import com.linagora.tmail.encrypted.MailboxManagerClassProbe; +import com.linagora.tmail.module.LinagoraTestJMAPServerModule; -class PostgresTmailServerTest implements JamesServerConcreteContract { +class PostgresTmailServerTest implements JamesServerConcreteContract, JmapJamesServerContract { static PostgresExtension postgresExtension = PostgresExtension.empty(); @RegisterExtension @@ -34,6 +36,7 @@ class PostgresTmailServerTest implements JamesServerConcreteContract { .eventBusImpl(IN_MEMORY) .build()) .server(configuration -> PostgresTmailServer.createServer(configuration) + .overrideWith(new LinagoraTestJMAPServerModule()) .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(MailboxManagerClassProbe.class)) .overrideWith(binder -> Multibinder.newSetBinder(binder, GuiceProbe.class).addBinding().to(UsersRepositoryClassProbe.class))) .extension(postgresExtension) diff --git a/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml b/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml index c899874a42..d109eacd45 100644 --- a/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml +++ b/tmail-backend/apps/postgres/src/test/resources/mailetcontainer.xml @@ -85,6 +85,14 @@ + + + + + + + + outgoing From 4d6fff4cebbe008b27d2e19a2636920823214043 Mon Sep 17 00:00:00 2001 From: Quan Tran Date: Wed, 6 Mar 2024 15:38:45 +0700 Subject: [PATCH 14/81] ISSUE-910 Introduce jmap-extensions-postgres module --- .../jmap/extensions-postgres/pom.xml | 60 +++++++++++++++++++ tmail-backend/pom.xml | 6 ++ 2 files changed, 66 insertions(+) create mode 100644 tmail-backend/jmap/extensions-postgres/pom.xml diff --git a/tmail-backend/jmap/extensions-postgres/pom.xml b/tmail-backend/jmap/extensions-postgres/pom.xml new file mode 100644 index 0000000000..7e3b09badb --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + com.linagora.tmail + tmail-backend + 1.0.0-SNAPSHOT + ../../pom.xml + + + jmap-extensions-postgres + Team-mail :: JMAP :: Extensions :: PostgreSQL + + + + ${project.groupId} + jmap-extensions + + + ${project.groupId} + jmap-extensions + test-jar + test + + + + ${james.groupId} + apache-james-backends-postgres + + + ${james.groupId} + apache-james-backends-postgres + test-jar + test + + + ${james.groupId} + james-server-guice-common + test-jar + test + + + ${james.groupId} + james-server-testing + test + + + ${james.groupId} + testing-base + test + + + org.testcontainers + postgresql + test + + + \ No newline at end of file diff --git a/tmail-backend/pom.xml b/tmail-backend/pom.xml index 21b4149b5e..dbf8fafac9 100644 --- a/tmail-backend/pom.xml +++ b/tmail-backend/pom.xml @@ -49,6 +49,7 @@ jmap/extensions jmap/extensions-cassandra + jmap/extensions-postgres jmap/extensions-opensearch jmap/extensions-rabbitmq @@ -261,6 +262,11 @@ jmap-extensions-cassandra ${project.version} + + ${project.groupId} + jmap-extensions-postgres + ${project.version} + ${james.groupId} james-server-data-cassandra From 55a1a70eeeb8d10102aa3b24eb2c0a185b55e712 Mon Sep 17 00:00:00 2001 From: Quan Tran Date: Thu, 7 Mar 2024 11:10:45 +0700 Subject: [PATCH 15/81] ISSUE-910 Implement PostgresFirebaseSubscriptionRepository --- .../jmap/firebase/PostgresFirebaseModule.java | 58 +++++++ .../PostgresFirebaseSubscriptionDAO.java | 145 ++++++++++++++++++ ...ostgresFirebaseSubscriptionRepository.java | 114 ++++++++++++++ ...resFirebaseSubscriptionRepositoryTest.java | 37 +++++ 4 files changed, 354 insertions(+) create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseModule.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepository.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepositoryTest.java diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseModule.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseModule.java new file mode 100644 index 0000000000..5b8834d408 --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseModule.java @@ -0,0 +1,58 @@ +package com.linagora.tmail.james.jmap.firebase; + +import java.time.OffsetDateTime; +import java.util.UUID; + +import org.apache.james.backends.postgres.PostgresCommons; +import org.apache.james.backends.postgres.PostgresIndex; +import org.apache.james.backends.postgres.PostgresModule; +import org.apache.james.backends.postgres.PostgresTable; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public interface PostgresFirebaseModule { + + interface FirebaseSubscriptionTable { + Table TABLE_NAME = DSL.table("firebase_subscription"); + String PRIMARY_KEY_CONSTRAINT = "firebase_subscription_primary_key_constraint"; + String FCM_TOKEN_UNIQUE_CONSTRAINT = "fcm_token_unique_constraint"; + + Field USER = DSL.field("username", SQLDataType.VARCHAR.notNull()); + Field DEVICE_CLIENT_ID = DSL.field("device_client_id", SQLDataType.VARCHAR.notNull()); + Field ID = DSL.field("id", SQLDataType.UUID.notNull()); + Field EXPIRES = DSL.field("expires", SQLDataType.TIMESTAMPWITHTIMEZONE(6)); + Field TYPES = DSL.field("types", PostgresCommons.DataTypes.STRING_ARRAY.notNull()); + Field FCM_TOKEN = DSL.field("fcm_token", SQLDataType.VARCHAR.notNull()); + + PostgresTable TABLE = PostgresTable.name(TABLE_NAME.getName()) + .createTableStep(((dsl, tableName) -> dsl.createTableIfNotExists(tableName) + .column(USER) + .column(DEVICE_CLIENT_ID) + .column(ID) + .column(EXPIRES) + .column(TYPES) + .column(FCM_TOKEN) + .constraint(DSL.constraint(PRIMARY_KEY_CONSTRAINT) + .primaryKey(USER, DEVICE_CLIENT_ID)) + .constraint(DSL.constraint(FCM_TOKEN_UNIQUE_CONSTRAINT) + .unique(FCM_TOKEN)))) + .supportsRowLevelSecurity() + .build(); + + + PostgresIndex USERNAME_INDEX = PostgresIndex.name("firebase_subscription_username_index") + .createIndexStep((dslContext, indexName) -> dslContext.createIndexIfNotExists(indexName) + .on(TABLE_NAME, USER)); + PostgresIndex USERNAME_ID_INDEX = PostgresIndex.name("firebase_subscription_username_id_index") + .createIndexStep((dslContext, indexName) -> dslContext.createIndexIfNotExists(indexName) + .on(TABLE_NAME, USER, ID)); + } + + PostgresModule MODULE = PostgresModule.builder() + .addTable(FirebaseSubscriptionTable.TABLE) + .addIndex(FirebaseSubscriptionTable.USERNAME_INDEX, FirebaseSubscriptionTable.USERNAME_ID_INDEX) + .build(); +} diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java new file mode 100644 index 0000000000..bb6b25e408 --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java @@ -0,0 +1,145 @@ +package com.linagora.tmail.james.jmap.firebase; + +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.DEVICE_CLIENT_ID; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.EXPIRES; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.FCM_TOKEN; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.FCM_TOKEN_UNIQUE_CONSTRAINT; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.ID; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.PRIMARY_KEY_CONSTRAINT; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.TABLE_NAME; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.TYPES; +import static com.linagora.tmail.james.jmap.firebase.PostgresFirebaseModule.FirebaseSubscriptionTable.USER; +import static org.apache.james.backends.postgres.PostgresCommons.IN_CLAUSE_MAX_SIZE; +import static org.apache.james.backends.postgres.utils.PostgresUtils.UNIQUE_CONSTRAINT_VIOLATION_PREDICATE; + +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.james.backends.postgres.utils.PostgresExecutor; +import org.apache.james.core.Username; +import org.apache.james.jmap.api.change.TypeStateFactory; +import org.apache.james.jmap.api.model.TypeName; +import org.jooq.Record; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.linagora.tmail.james.jmap.model.DeviceClientIdInvalidException; +import com.linagora.tmail.james.jmap.model.FirebaseSubscription; +import com.linagora.tmail.james.jmap.model.FirebaseSubscriptionExpiredTime; +import com.linagora.tmail.james.jmap.model.FirebaseSubscriptionId; +import com.linagora.tmail.james.jmap.model.TokenInvalidException; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import scala.jdk.javaapi.CollectionConverters; + +public class PostgresFirebaseSubscriptionDAO { + private static final Function OFFSET_DATE_TIME_ZONED_DATE_TIME_FUNCTION = offsetDateTime -> + Optional.ofNullable(offsetDateTime) + .map(value -> value.atZoneSameInstant(ZoneId.of("UTC"))) + .orElse(null); + + private static final Predicate IS_PRIMARY_KEY_UNIQUE_CONSTRAINT = throwable -> throwable.getMessage().contains(PRIMARY_KEY_CONSTRAINT); + private static final Predicate IS_FCM_TOKEN_UNIQUE_CONSTRAINT = throwable -> throwable.getMessage().contains(FCM_TOKEN_UNIQUE_CONSTRAINT); + + private final PostgresExecutor postgresExecutor; + private final TypeStateFactory typeStateFactory; + + public PostgresFirebaseSubscriptionDAO(PostgresExecutor postgresExecutor, TypeStateFactory typeStateFactory) { + this.postgresExecutor = postgresExecutor; + this.typeStateFactory = typeStateFactory; + } + + public Mono save(Username username, FirebaseSubscription subscription) { + return postgresExecutor.executeVoid(dslContext -> Mono.from(dslContext.insertInto(TABLE_NAME) + .set(USER, username.asString()) + .set(DEVICE_CLIENT_ID, subscription.deviceClientId()) + .set(ID, subscription.id().value()) + .set(EXPIRES, subscription.expires().value().toOffsetDateTime()) + .set(TYPES, CollectionConverters.asJava(subscription.types()) + .stream().map(TypeName::asString).toArray(String[]::new)) + .set(FCM_TOKEN, subscription.token()))) + .onErrorMap(UNIQUE_CONSTRAINT_VIOLATION_PREDICATE.and(IS_PRIMARY_KEY_UNIQUE_CONSTRAINT), + e -> new DeviceClientIdInvalidException(subscription.deviceClientId(), "deviceClientId must be unique")) + .onErrorMap(UNIQUE_CONSTRAINT_VIOLATION_PREDICATE.and(IS_FCM_TOKEN_UNIQUE_CONSTRAINT), + e -> new TokenInvalidException("token must be unique")); + } + + public Flux listByUsername(Username username) { + return postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.selectFrom(TABLE_NAME) + .where(USER.eq(username.asString())))) + .map(this::toSubscription); + } + + public Flux getByUsernameAndIds(Username username, Collection ids) { + Function, Flux> queryPublisherFunction = idsMatching -> postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.selectFrom(TABLE_NAME) + .where(USER.eq(username.asString())) + .and(ID.in(idsMatching.stream().map(FirebaseSubscriptionId::value) + .toList())))) + .map(this::toSubscription); + + if (ids.size() <= IN_CLAUSE_MAX_SIZE) { + return queryPublisherFunction.apply(ids); + } else { + return Flux.fromIterable(Iterables.partition(ids, IN_CLAUSE_MAX_SIZE)) + .flatMap(queryPublisherFunction); + } + } + + public Mono deleteByUsername(Username username) { + return postgresExecutor.executeVoid(dslContext -> Mono.from(dslContext.deleteFrom(TABLE_NAME) + .where(USER.eq(username.asString())))); + } + + public Mono deleteByUsernameAndId(Username username, FirebaseSubscriptionId id) { + return postgresExecutor.executeVoid(dslContext -> Mono.from(dslContext.deleteFrom(TABLE_NAME) + .where(USER.eq(username.asString())) + .and(ID.eq(id.value())))); + } + + public Mono> updateType(Username username, FirebaseSubscriptionId id, Set newTypes) { + Preconditions.checkNotNull(newTypes, "newTypes should not be null"); + return postgresExecutor.executeRow(dslContext -> Mono.from(dslContext.update(TABLE_NAME) + .set(TYPES, newTypes.stream().map(TypeName::asString).toArray(String[]::new)) + .where(USER.eq(username.asString())) + .and(ID.eq(id.value())) + .returning(TYPES))) + .map(this::extractTypes); + } + + public Mono updateExpireTime(Username username, FirebaseSubscriptionId id, ZonedDateTime newExpire) { + Preconditions.checkNotNull(newExpire, "newExpire should not be null"); + return postgresExecutor.executeRow(dslContext -> Mono.from(dslContext.update(TABLE_NAME) + .set(EXPIRES, newExpire.toOffsetDateTime()) + .where(USER.eq(username.asString())) + .and(ID.eq(id.value())) + .returning(EXPIRES))) + .map(record -> OFFSET_DATE_TIME_ZONED_DATE_TIME_FUNCTION.apply(record.get(EXPIRES))); + } + + private FirebaseSubscription toSubscription(Record record) { + return new FirebaseSubscription(new FirebaseSubscriptionId(record.get(ID)), + record.get(DEVICE_CLIENT_ID), + record.get(FCM_TOKEN), + toExpires(record), + CollectionConverters.asScala(extractTypes(record)).toSeq()); + } + + private FirebaseSubscriptionExpiredTime toExpires(Record record) { + return new FirebaseSubscriptionExpiredTime(OFFSET_DATE_TIME_ZONED_DATE_TIME_FUNCTION.apply(record.get(EXPIRES))); + } + + private Set extractTypes(Record record) { + return Arrays.stream(record.get(TYPES)) + .map(string -> typeStateFactory.parse(string).right().get()) + .collect(Collectors.toSet()); + } +} \ No newline at end of file diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepository.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepository.java new file mode 100644 index 0000000000..f40699e6a3 --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepository.java @@ -0,0 +1,114 @@ +package com.linagora.tmail.james.jmap.firebase; + +import static com.linagora.tmail.james.jmap.firebase.FirebaseSubscriptionHelper.evaluateExpiresTime; +import static com.linagora.tmail.james.jmap.firebase.FirebaseSubscriptionHelper.isInThePast; + +import java.time.Clock; +import java.time.ZonedDateTime; +import java.util.Optional; +import java.util.Set; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.apache.james.backends.postgres.utils.PostgresExecutor; +import org.apache.james.core.Username; +import org.apache.james.jmap.api.change.TypeStateFactory; +import org.apache.james.jmap.api.model.TypeName; + +import com.linagora.tmail.james.jmap.model.ExpireTimeInvalidException; +import com.linagora.tmail.james.jmap.model.FirebaseSubscription; +import com.linagora.tmail.james.jmap.model.FirebaseSubscriptionCreationRequest; +import com.linagora.tmail.james.jmap.model.FirebaseSubscriptionExpiredTime; +import com.linagora.tmail.james.jmap.model.FirebaseSubscriptionId; +import com.linagora.tmail.james.jmap.model.FirebaseSubscriptionNotFoundException; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import scala.jdk.javaapi.OptionConverters; + +public class PostgresFirebaseSubscriptionRepository implements FirebaseSubscriptionRepository { + private final Clock clock; + private final TypeStateFactory typeStateFactory; + private final PostgresExecutor.Factory executorFactory; + + @Inject + @Singleton + public PostgresFirebaseSubscriptionRepository(Clock clock, TypeStateFactory typeStateFactory, PostgresExecutor.Factory executorFactory) { + this.clock = clock; + this.typeStateFactory = typeStateFactory; + this.executorFactory = executorFactory; + } + + @Override + public Mono save(Username username, FirebaseSubscriptionCreationRequest request) { + PostgresFirebaseSubscriptionDAO subscriptionDAO = firebaseSubscriptionDAO(username); + + return validateInputExpires(request) + .then(Mono.defer(() -> { + FirebaseSubscription subscription = FirebaseSubscription.from(request, + evaluateExpiresTime(OptionConverters.toJava(request.expires().map(FirebaseSubscriptionExpiredTime::value)), clock)); + return subscriptionDAO.save(username, subscription) + .thenReturn(subscription); + })); + } + + private Mono validateInputExpires(FirebaseSubscriptionCreationRequest request) { + return Mono.just(request.expires()) + .handle((inputExpires, sink) -> { + if (isInThePast(request.expires(), clock)) { + sink.error(new ExpireTimeInvalidException(request.expires().get().value(), "expires must be greater than now")); + } + }); + } + + @Override + public Mono updateExpireTime(Username username, FirebaseSubscriptionId id, ZonedDateTime newExpire) { + return Mono.just(newExpire) + .handle((inputTime, sink) -> { + if (newExpire.isBefore(ZonedDateTime.now(clock))) { + sink.error(new ExpireTimeInvalidException(inputTime, "expires must be greater than now")); + } + }) + .then(firebaseSubscriptionDAO(username) + .updateExpireTime(username, id, evaluateExpiresTime(Optional.of(newExpire), clock).value()) + .map(FirebaseSubscriptionExpiredTime::new) + .switchIfEmpty(Mono.error(() -> new FirebaseSubscriptionNotFoundException(id)))); + } + + @Override + public Mono updateTypes(Username username, FirebaseSubscriptionId id, Set types) { + return firebaseSubscriptionDAO(username) + .updateType(username, id, types) + .switchIfEmpty(Mono.error(() -> new FirebaseSubscriptionNotFoundException(id))) + .then(); + } + + @Override + public Mono revoke(Username username, FirebaseSubscriptionId id) { + return firebaseSubscriptionDAO(username) + .deleteByUsernameAndId(username, id); + } + + @Override + public Mono revoke(Username username) { + return firebaseSubscriptionDAO(username) + .deleteByUsername(username); + } + + @Override + public Flux get(Username username, Set ids) { + return firebaseSubscriptionDAO(username) + .getByUsernameAndIds(username, ids); + } + + @Override + public Flux list(Username username) { + return firebaseSubscriptionDAO(username) + .listByUsername(username); + } + + private PostgresFirebaseSubscriptionDAO firebaseSubscriptionDAO(Username username) { + return new PostgresFirebaseSubscriptionDAO(executorFactory.create(username.getDomainPart()), typeStateFactory); + } +} diff --git a/tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepositoryTest.java b/tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepositoryTest.java new file mode 100644 index 0000000000..c30dc74f12 --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionRepositoryTest.java @@ -0,0 +1,37 @@ +package com.linagora.tmail.james.jmap.firebase; + +import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.backends.postgres.PostgresModule; +import org.apache.james.jmap.api.change.TypeStateFactory; +import org.apache.james.utils.UpdatableTickingClock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.RegisterExtension; + +import scala.jdk.javaapi.CollectionConverters; + +class PostgresFirebaseSubscriptionRepositoryTest implements FirebaseSubscriptionRepositoryContract { + @RegisterExtension + static PostgresExtension postgresExtension = PostgresExtension.withRowLevelSecurity( + PostgresModule.aggregateModules(PostgresFirebaseModule.MODULE)); + + private UpdatableTickingClock clock; + private FirebaseSubscriptionRepository firebaseSubscriptionRepository; + + @BeforeEach + void setup() { + clock = new UpdatableTickingClock(FirebaseSubscriptionRepositoryContract.NOW()); + firebaseSubscriptionRepository = new PostgresFirebaseSubscriptionRepository(clock, + new TypeStateFactory(CollectionConverters.asJava(FirebaseSubscriptionRepositoryContract.TYPE_NAME_SET())), + postgresExtension.getExecutorFactory()); + } + + @Override + public UpdatableTickingClock clock() { + return clock; + } + + @Override + public FirebaseSubscriptionRepository testee() { + return firebaseSubscriptionRepository; + } +} From 6d957b5f6366f7cb570d87e966d17a24d1032977 Mon Sep 17 00:00:00 2001 From: Quan Tran Date: Fri, 8 Mar 2024 15:43:02 +0700 Subject: [PATCH 16/81] ISSUE-910 PostgresFirebaseSubscriptionDAO: avoid fetching USER column when not needed --- .../firebase/PostgresFirebaseSubscriptionDAO.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java index bb6b25e408..29f36d237e 100644 --- a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/firebase/PostgresFirebaseSubscriptionDAO.java @@ -74,16 +74,19 @@ public Mono save(Username username, FirebaseSubscription subscription) { } public Flux listByUsername(Username username) { - return postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.selectFrom(TABLE_NAME) + return postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.select(DEVICE_CLIENT_ID, ID, EXPIRES, TYPES, FCM_TOKEN) + .from(TABLE_NAME) .where(USER.eq(username.asString())))) .map(this::toSubscription); } public Flux getByUsernameAndIds(Username username, Collection ids) { - Function, Flux> queryPublisherFunction = idsMatching -> postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.selectFrom(TABLE_NAME) - .where(USER.eq(username.asString())) - .and(ID.in(idsMatching.stream().map(FirebaseSubscriptionId::value) - .toList())))) + Function, Flux> queryPublisherFunction = idsMatching -> postgresExecutor.executeRows(dslContext -> + Flux.from(dslContext.select(DEVICE_CLIENT_ID, ID, EXPIRES, TYPES, FCM_TOKEN) + .from(TABLE_NAME) + .where(USER.eq(username.asString())) + .and(ID.in(idsMatching.stream().map(FirebaseSubscriptionId::value) + .toList())))) .map(this::toSubscription); if (ids.size() <= IN_CLAUSE_MAX_SIZE) { From 5a60af1cd635015e6525825d57cb30c69691ed28 Mon Sep 17 00:00:00 2001 From: Quan Tran Date: Fri, 8 Mar 2024 14:57:22 +0700 Subject: [PATCH 17/81] ISSUE-911 Implement PostgresLabelChangeRepository --- .../jmap/extensions-postgres/pom.xml | 5 + .../jmap/label/PostgresLabelChangeDAO.java | 93 +++++++++++++++++++ .../label/PostgresLabelChangeRepository.java | 87 +++++++++++++++++ .../james/jmap/label/PostgresLabelModule.java | 50 ++++++++++ .../PostgresLabelChangeRepositoryTest.java | 42 +++++++++ tmail-backend/pom.xml | 5 + 6 files changed, 282 insertions(+) create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeDAO.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepository.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelModule.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepositoryTest.java diff --git a/tmail-backend/jmap/extensions-postgres/pom.xml b/tmail-backend/jmap/extensions-postgres/pom.xml index 7e3b09badb..d1aea2a6f0 100644 --- a/tmail-backend/jmap/extensions-postgres/pom.xml +++ b/tmail-backend/jmap/extensions-postgres/pom.xml @@ -35,6 +35,11 @@ test-jar test + + ${james.groupId} + james-server-data-jmap-postgres + test + ${james.groupId} james-server-guice-common diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeDAO.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeDAO.java new file mode 100644 index 0000000000..82da6464db --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeDAO.java @@ -0,0 +1,93 @@ +package com.linagora.tmail.james.jmap.label; + +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.ACCOUNT_ID; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.CREATED; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.CREATED_DATE; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.DESTROYED; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.STATE; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.TABLE_NAME; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.UPDATED; + +import java.time.Clock; +import java.time.ZonedDateTime; +import java.util.Arrays; + +import org.apache.james.backends.postgres.utils.PostgresExecutor; +import org.apache.james.jmap.api.change.State; +import org.apache.james.jmap.api.model.AccountId; +import org.jooq.Record; + +import com.linagora.tmail.james.jmap.model.LabelId; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import scala.collection.immutable.Set; +import scala.jdk.javaapi.CollectionConverters; + +public class PostgresLabelChangeDAO { + private final PostgresExecutor postgresExecutor; + private final Clock clock; + + public PostgresLabelChangeDAO(PostgresExecutor postgresExecutor, + Clock clock) { + this.postgresExecutor = postgresExecutor; + this.clock = clock; + } + + public Mono insert(LabelChange change) { + return postgresExecutor.executeVoid(dslContext -> Mono.from(dslContext.insertInto(TABLE_NAME) + .set(ACCOUNT_ID, change.getAccountId().getIdentifier()) + .set(STATE, change.state().getValue()) + .set(CREATED, toStringArray(change.created())) + .set(UPDATED, toStringArray(change.updated())) + .set(DESTROYED, toStringArray(change.destroyed())) + .set(CREATED_DATE, ZonedDateTime.now(clock).toOffsetDateTime()))); + } + + public Flux getAllChanges(AccountId accountId) { + return postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.select(STATE, CREATED, UPDATED, DESTROYED) + .from(TABLE_NAME) + .where(ACCOUNT_ID.eq(accountId.getIdentifier())))) + .map(record -> toLabelChange(record, accountId)); + } + + public Flux getChangesSince(AccountId accountId, State state) { + return postgresExecutor.executeRows(dslContext -> Flux.from(dslContext.select(STATE, CREATED, UPDATED, DESTROYED) + .from(TABLE_NAME) + .where(ACCOUNT_ID.eq(accountId.getIdentifier())) + .and(STATE.greaterOrEqual(state.getValue())) + .orderBy(STATE))) + .map(record -> toLabelChange(record, accountId)); + } + + public Mono getLatestState(AccountId accountId) { + return postgresExecutor.executeRow(dslContext -> Mono.from(dslContext.select(STATE) + .from(TABLE_NAME) + .where(ACCOUNT_ID.eq(accountId.getIdentifier())) + .orderBy(STATE.desc()) + .limit(1))) + .map(record -> State.of(record.get(STATE))); + } + + private String[] toStringArray(Set labelIds) { + return CollectionConverters.asJava(labelIds) + .stream() + .map(LabelId::serialize) + .toArray(String[]::new); + } + + private LabelChange toLabelChange(Record record, AccountId accountId) { + return LabelChange.apply(accountId, + toLabelIdSet(record.get(CREATED)), + toLabelIdSet(record.get(UPDATED)), + toLabelIdSet(record.get(DESTROYED)), + State.of(record.get(STATE))); + } + + private Set toLabelIdSet(String[] strings) { + return CollectionConverters.asScala(Arrays.stream(strings) + .map(LabelId::fromKeyword) + .iterator()) + .toSet(); + } +} diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepository.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepository.java new file mode 100644 index 0000000000..516c8f7464 --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepository.java @@ -0,0 +1,87 @@ +package com.linagora.tmail.james.jmap.label; + +import java.time.Clock; + +import javax.inject.Inject; + +import org.apache.james.backends.postgres.utils.PostgresExecutor; +import org.apache.james.core.Username; +import org.apache.james.jmap.api.change.Limit; +import org.apache.james.jmap.api.change.State; +import org.apache.james.jmap.api.exception.ChangeNotFoundException; +import org.apache.james.jmap.api.model.AccountId; + +import com.google.common.base.Preconditions; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import scala.Option; +import scala.collection.immutable.Set$; +import scala.jdk.javaapi.OptionConverters; + +public class PostgresLabelChangeRepository implements LabelChangeRepository { + private final PostgresExecutor.Factory executorFactory; + private final Clock clock; + + @Inject + public PostgresLabelChangeRepository(PostgresExecutor.Factory executorFactory, Clock clock) { + this.executorFactory = executorFactory; + this.clock = clock; + } + + @Override + public Mono save(LabelChange change) { + return labelChangeDAO(change.getAccountId()) + .insert(change); + } + + @Override + public Mono getSinceState(AccountId accountId, State state, Option maxChanges) { + Preconditions.checkNotNull(accountId); + Preconditions.checkNotNull(state); + + int maxIds = OptionConverters.toJava(maxChanges) + .orElse(LabelChangeRepository.DEFAULT_MAX_IDS_TO_RETURN()) + .getValue(); + Preconditions.checkArgument(maxIds > 0, "maxChanges must be a positive integer"); + + PostgresLabelChangeDAO labelChangeDAO = labelChangeDAO(accountId); + + if (state.equals(State.INITIAL)) { + return getAllChanges(accountId, maxIds, labelChangeDAO); + } + + return getChangesSince(accountId, state, maxIds, labelChangeDAO); + } + + private Mono getChangesSince(AccountId accountId, State state, int maxIds, PostgresLabelChangeDAO labelChangeDAO) { + return labelChangeDAO.getChangesSince(accountId, state) + .switchIfEmpty(Flux.error(() -> new ChangeNotFoundException(state, String.format("State '%s' could not be found", state.getValue())))) + .filter(change -> !change.state().equals(state)) + .switchIfEmpty(fallbackToTheCurrentState(accountId, state)) + .map(LabelChanges::from) + .reduce(LabelChanges.initial(), (change1, change2) -> LabelChanges.merge(maxIds, change1, change2)); + } + + private Mono fallbackToTheCurrentState(AccountId accountId, State state) { + return Mono.defer(() -> Mono.just(LabelChange.apply( + accountId, Set$.MODULE$.empty(), Set$.MODULE$.empty(), Set$.MODULE$.empty(), state))); + } + + private Mono getAllChanges(AccountId accountId, int maxIds, PostgresLabelChangeDAO labelChangeDAO) { + return labelChangeDAO.getAllChanges(accountId) + .map(LabelChanges::from) + .reduce(LabelChanges.initial(), (change1, change2) -> LabelChanges.merge(maxIds, change1, change2)); + } + + @Override + public Mono getLatestState(AccountId accountId) { + return labelChangeDAO(accountId).getLatestState(accountId) + .switchIfEmpty(Mono.just(State.INITIAL)); + } + + private PostgresLabelChangeDAO labelChangeDAO(AccountId accountId) { + return new PostgresLabelChangeDAO(executorFactory.create(Username.of(accountId.getIdentifier()).getDomainPart()), + clock); + } +} diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelModule.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelModule.java new file mode 100644 index 0000000000..6adfd5036b --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelModule.java @@ -0,0 +1,50 @@ +package com.linagora.tmail.james.jmap.label; + +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.INDEX; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelChangeTable.TABLE; + +import java.time.OffsetDateTime; +import java.util.UUID; + +import org.apache.james.backends.postgres.PostgresIndex; +import org.apache.james.backends.postgres.PostgresModule; +import org.apache.james.backends.postgres.PostgresTable; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public interface PostgresLabelModule { + interface LabelChangeTable { + Table TABLE_NAME = DSL.table("label_change"); + + Field ACCOUNT_ID = DSL.field("account_id", SQLDataType.VARCHAR.notNull()); + Field STATE = DSL.field("state", SQLDataType.UUID.notNull()); + Field CREATED = DSL.field("created", SQLDataType.VARCHAR.getArrayDataType().notNull()); + Field UPDATED = DSL.field("updated", SQLDataType.VARCHAR.getArrayDataType().notNull()); + Field DESTROYED = DSL.field("destroyed", SQLDataType.VARCHAR.getArrayDataType().notNull()); + Field CREATED_DATE = DSL.field("created_date", SQLDataType.TIMESTAMPWITHTIMEZONE.notNull()); + + PostgresTable TABLE = PostgresTable.name(TABLE_NAME.getName()) + .createTableStep(((dsl, tableName) -> dsl.createTableIfNotExists(tableName) + .column(ACCOUNT_ID) + .column(STATE) + .column(CREATED) + .column(UPDATED) + .column(DESTROYED) + .column(CREATED_DATE) + .constraint(DSL.primaryKey(ACCOUNT_ID, STATE)))) + .supportsRowLevelSecurity() + .build(); + + PostgresIndex INDEX = PostgresIndex.name("index_label_change_created_date") + .createIndexStep((dslContext, indexName) -> dslContext.createIndexIfNotExists(indexName) + .on(TABLE_NAME, CREATED_DATE)); + } + + PostgresModule MODULE = PostgresModule.builder() + .addTable(TABLE) + .addIndex(INDEX) + .build(); +} diff --git a/tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepositoryTest.java b/tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepositoryTest.java new file mode 100644 index 0000000000..e88a7a2936 --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/label/PostgresLabelChangeRepositoryTest.java @@ -0,0 +1,42 @@ +package com.linagora.tmail.james.jmap.label; + +import java.time.ZonedDateTime; + +import org.apache.james.backends.postgres.PostgresExtension; +import org.apache.james.backends.postgres.PostgresModule; +import org.apache.james.jmap.api.change.State; +import org.apache.james.jmap.postgres.change.PostgresStateFactory; +import org.apache.james.utils.UpdatableTickingClock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.linagora.tmail.james.jmap.firebase.FirebaseSubscriptionRepositoryContract; + +class PostgresLabelChangeRepositoryTest implements LabelChangeRepositoryContract { + @RegisterExtension + static PostgresExtension postgresExtension = PostgresExtension.withRowLevelSecurity( + PostgresModule.aggregateModules(PostgresLabelModule.MODULE)); + + private PostgresLabelChangeRepository labelChangeRepository; + + @BeforeEach + void setup() { + UpdatableTickingClock clock = new UpdatableTickingClock(FirebaseSubscriptionRepositoryContract.NOW()); + labelChangeRepository = new PostgresLabelChangeRepository(postgresExtension.getExecutorFactory(), clock); + } + + @Override + public LabelChangeRepository testee() { + return labelChangeRepository; + } + + @Override + public State.Factory stateFactory() { + return new PostgresStateFactory(); + } + + @Override + public void setClock(ZonedDateTime newTime) { + + } +} diff --git a/tmail-backend/pom.xml b/tmail-backend/pom.xml index dbf8fafac9..95182ef216 100644 --- a/tmail-backend/pom.xml +++ b/tmail-backend/pom.xml @@ -273,6 +273,11 @@ test-jar ${james.version} + + ${james.groupId} + james-server-data-jmap-postgres + ${james.version} + ${james.groupId} james-server-data-ldap From 4562db04519682ef6d4ae383f4d062d27716e19a Mon Sep 17 00:00:00 2001 From: Quan Tran Date: Fri, 8 Mar 2024 15:12:10 +0700 Subject: [PATCH 18/81] ISSUE-911 SQL script to clean outdated data in the `label_change` table --- tmail-backend/apps/postgres/README.md | 9 +++++++++ .../apps/postgres/clean_up_data_tmail.sql | 16 ++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tmail-backend/apps/postgres/README.md create mode 100644 tmail-backend/apps/postgres/clean_up_data_tmail.sql diff --git a/tmail-backend/apps/postgres/README.md b/tmail-backend/apps/postgres/README.md new file mode 100644 index 0000000000..25bdaefcb9 --- /dev/null +++ b/tmail-backend/apps/postgres/README.md @@ -0,0 +1,9 @@ +## Administration Operations +## Clean up data + +To clean up some data on the specific TMail data structures, that will be redundant again after a long time, you can execute the SQL queries `clean_up_data_tmail.sql`. + +The data that in: +- `label_change` table + +Note that the `clean_up_data_tmail.sql` should be merged with [the SQL clean up script on Apache James](https://github.com/apache/james-project/blob/postgresql/server/apps/postgres-app/clean_up.sql) to clean data on James tables as well. diff --git a/tmail-backend/apps/postgres/clean_up_data_tmail.sql b/tmail-backend/apps/postgres/clean_up_data_tmail.sql new file mode 100644 index 0000000000..ceeb226661 --- /dev/null +++ b/tmail-backend/apps/postgres/clean_up_data_tmail.sql @@ -0,0 +1,16 @@ +-- This is a script to delete old rows from some tables. One of the attempts to clean up the never-used data after a long time. + +DO +$$ + DECLARE + days_to_keep INTEGER; + BEGIN + -- Set the number of days dynamically + days_to_keep := 60; + + -- Delete rows older than the specified number of days in the `label_change` table + DELETE + FROM label_change + WHERE created_date < current_timestamp - interval '1 day' * days_to_keep; + END +$$; \ No newline at end of file From 9a5fd6488b210cb70cda33f5e0d7bdc5da650c2a Mon Sep 17 00:00:00 2001 From: Quan Tran Date: Mon, 11 Mar 2024 16:50:13 +0700 Subject: [PATCH 19/81] ISSUE-912 Implement PostgresLabelRepository --- .../jmap/extensions-postgres/pom.xml | 16 +++ .../james/jmap/label/PostgresLabelDAO.java | 109 ++++++++++++++++++ .../james/jmap/label/PostgresLabelModule.java | 31 ++++- .../jmap/label/PostgresLabelRepository.java | 84 ++++++++++++++ .../jmap/label/PostgresLabelDAOUtils.scala | 18 +++ .../label/PostgresLabelRepositoryTest.java | 24 ++++ 6 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelDAO.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelRepository.java create mode 100644 tmail-backend/jmap/extensions-postgres/src/main/scala/com/linagora/tmail/james/jmap/label/PostgresLabelDAOUtils.scala create mode 100644 tmail-backend/jmap/extensions-postgres/src/test/java/com/linagora/tmail/james/jmap/label/PostgresLabelRepositoryTest.java diff --git a/tmail-backend/jmap/extensions-postgres/pom.xml b/tmail-backend/jmap/extensions-postgres/pom.xml index d1aea2a6f0..b5b757803d 100644 --- a/tmail-backend/jmap/extensions-postgres/pom.xml +++ b/tmail-backend/jmap/extensions-postgres/pom.xml @@ -62,4 +62,20 @@ test + + + + + net.alchim31.maven + scala-maven-plugin + + + io.github.evis + scalafix-maven-plugin_2.13 + + ${project.parent.parent.basedir}/.scalafix.conf + + + + \ No newline at end of file diff --git a/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelDAO.java b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelDAO.java new file mode 100644 index 0000000000..893beccb4c --- /dev/null +++ b/tmail-backend/jmap/extensions-postgres/src/main/java/com/linagora/tmail/james/jmap/label/PostgresLabelDAO.java @@ -0,0 +1,109 @@ +package com.linagora.tmail.james.jmap.label; + +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelTable.COLOR; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelTable.DISPLAY_NAME; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelTable.KEYWORD; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelTable.TABLE_NAME; +import static com.linagora.tmail.james.jmap.label.PostgresLabelModule.LabelTable.USERNAME; + +import java.util.Collection; +import java.util.Optional; +import java.util.function.Function; + +import org.apache.james.backends.postgres.utils.PostgresExecutor; +import org.apache.james.core.Username; +import org.jooq.Record; +import org.jooq.UpdateSetFirstStep; +import org.jooq.UpdateSetMoreStep; + +import com.linagora.tmail.james.jmap.model.Color; +import com.linagora.tmail.james.jmap.model.DisplayName; +import com.linagora.tmail.james.jmap.model.Label; +import com.linagora.tmail.james.jmap.model.LabelId; +import com.linagora.tmail.james.jmap.model.LabelNotFoundException; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import scala.Option; +import scala.jdk.javaapi.OptionConverters; + +public class PostgresLabelDAO { + private final PostgresExecutor postgresExecutor; + + public PostgresLabelDAO(PostgresExecutor postgresExecutor) { + this.postgresExecutor = postgresExecutor; + } + + public Mono