diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst index 702e810ec5e..df19860b3c7 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst @@ -93,13 +93,9 @@ the private key. You will need two different sets certificates and the CA certif The following commands should do the trick for you, by creating a fake CA, a fake user certificate, and a fake host certificate:: cd $DEVROOT/DIRAC - git checkout release/integration - source tests/Jenkins/utilities.sh - generateCA - generateCertificates 365 - generateUserCredentials 365 + docker run ghcr.io/diracgrid/diracx/certificates-generation:latest mkdir -p ~/.globus/ - cp $DEVROOT/user/*.{pem,key} ~/.globus/ + docker cp certificates-generation:/ca/certs/client.{pem,key} ~/.globus/ mv ~/.globus/client.key ~/.globus/userkey.pem mv ~/.globus/client.pem ~/.globus/usercert.pem diff --git a/src/DIRAC/Core/DISET/private/Service.py b/src/DIRAC/Core/DISET/private/Service.py index 17d8795ae34..25b2058900c 100644 --- a/src/DIRAC/Core/DISET/private/Service.py +++ b/src/DIRAC/Core/DISET/private/Service.py @@ -100,9 +100,7 @@ def initialize(self): "validNames": self._validNames, "csPaths": [PathFinder.getServiceSection(svcName) for svcName in self._validNames], } - self.securityLogging = Operations().getValue("EnableSecurityLogging", False) and getServiceOption( - self._serviceInfoDict, "EnableSecurityLogging", False - ) + self.securityLogging = Operations().getValue("EnableSecurityLogging", False) # Initialize Monitoring # The import needs to be here because of the CS must be initialized before importing diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index 5cd1b4266ff..a2f2c6e32f2 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -1,4 +1,6 @@ volumes: + # Volume used to store the certificates of dirac + certs_data: # Volume used to store the config of diracx diracx-cs-store: # Volume used to store the pair of keys to sign the tokens @@ -101,6 +103,15 @@ services: command: /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/check_db_initialized.sh pull_policy: always + dirac-init-certificates: + image: ghcr.io/diracgrid/diracx/certificates-generation:latest + container_name: dirac-init-certificates + volumes: + - certs_data:/ca/certs/ + entrypoint: | + /entrypoint.sh + pull_policy: always + dirac-server: image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac container_name: server @@ -116,6 +127,8 @@ services: condition: service_started iam-login-service: condition: service_healthy + dirac-init-certificates: + condition: service_completed_successfully # Let the init container create the certificates diracx-init-key: condition: service_completed_successfully # Let the init container create the signing key diracx-init-cs: @@ -123,13 +136,14 @@ services: ulimits: nofile: 8192 volumes: + - certs_data:/ca/certs - diracx-cs-store:/cs_store - diracx-key-store:/signing-key environment: - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key - pull_policy: always command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. + pull_policy: always dirac-client: @@ -141,8 +155,10 @@ services: - dirac-server ulimits: nofile: 8192 - pull_policy: always + volumes: + - certs_data:/ca/certs command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. + pull_policy: always dirac-pilot: image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac @@ -152,6 +168,7 @@ services: depends_on: - dirac-server volumes: + - certs_data:/ca/certs - type: bind source: ${CVMFS_DIR} target: /cvmfs diff --git a/tests/Jenkins/config/ci/openssl_config_ca.cnf b/tests/Jenkins/config/ci/openssl_config_ca.cnf deleted file mode 100644 index df7ac9909bb..00000000000 --- a/tests/Jenkins/config/ci/openssl_config_ca.cnf +++ /dev/null @@ -1,110 +0,0 @@ -# OpenSSL root CA configuration file - -[ ca ] -# `man ca` -default_ca = CA_default - -[ CA_default ] -# Directory and file locations. -# EDIT HERE -dir = #GRIDSECURITY#/ca #PUT THE RIGHT DIR HERE! -####### -certs = $dir/certs -crl_dir = $dir/crl -new_certs_dir = $dir/newcerts -database = $dir/index.txt -serial = $dir/serial -RANDFILE = $dir/private/.rand - -# The root key and root certificate. -private_key = $dir/ca.key.pem -certificate = $dir/ca.cert.pem - -# For certificate revocation lists. -crlnumber = $dir/crlnumber -crl = $dir/crl/ca.crl.pem -crl_extensions = crl_ext -default_crl_days = 30 - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -name_opt = ca_default -cert_opt = ca_default -default_days = 375 -preserve = no -policy = policy_loose - -# This option is dangerous, but allows to -# set subjectAlternativeName on the Request -# `man ca` is your friend -copy_extensions=copy - -[ policy_strict ] -# The root CA should only sign intermediate certificates that match. -# See the POLICY FORMAT section of `man ca`. -organizationName = match -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -[ policy_loose ] -# Allow the intermediate CA to sign a more diverse range of certificates. -# See the POLICY FORMAT section of the `ca` man page. -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -[ req ] -# Options for the `req` tool (`man req`). -default_bits = 2048 -distinguished_name = req_distinguished_name -string_mask = utf8only -prompt = no - - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -# Extension to add when the -x509 option is used. -x509_extensions = v3_ca - -[ req_distinguished_name ] -# See . -# EDIT HERE OPTIONALLY -O = DIRAC CI -CN = DIRAC CI Signing Certification Authority -######## - -[ v3_ca ] -# Extensions for a typical CA (`man x509v3_config`). -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer -basicConstraints = critical, CA:true -keyUsage = critical, digitalSignature, cRLSign, keyCertSign - -[ usr_cert ] -# Extensions for client certificates (`man x509v3_config`). -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer -keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth - - -# Whatever is in the request will be copied, unless it is already defined. -# So we have some double definitions here with the other ssl config files, -# but that's allright. The basicConstraints = CA:FALSE is a protection that MUST stay -[ server_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = CA:FALSE -nsComment = "OpenSSL Generated Server Certificate" -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -# Our servers need both serverAuth and clientAuth -extendedKeyUsage = serverAuth,clientAuth diff --git a/tests/Jenkins/config/ci/openssl_config_host.cnf b/tests/Jenkins/config/ci/openssl_config_host.cnf deleted file mode 100644 index 6633552ed99..00000000000 --- a/tests/Jenkins/config/ci/openssl_config_host.cnf +++ /dev/null @@ -1,37 +0,0 @@ -# OpenSSL root CA configuration file - - -[ req ] -# Options for the `req` tool (`man req`). -default_bits = 2048 -distinguished_name = req_distinguished_name -string_mask = utf8only -prompt = no -encrypt_key = yes - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -# Extension to add in the request -req_extensions = v3_req - -[ req_distinguished_name ] -# CAUTION the name also needs to be in the alt_names as per RFC (don't remember the number) -# See . -C = ch -O = DIRAC -OU = DIRAC CI -CN = #hostname# - - -[ v3_req ] -# Extensions to ask for the cert (`man x509v3_config`). -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = serverAuth,clientAuth -subjectKeyIdentifier = hash -subjectAltName = @alt_names - - -[ alt_names ] -DNS.1 = #hostname# -DNS.2 = localhost diff --git a/tests/Jenkins/config/ci/openssl_config_user.cnf b/tests/Jenkins/config/ci/openssl_config_user.cnf deleted file mode 100644 index f767b58a8c5..00000000000 --- a/tests/Jenkins/config/ci/openssl_config_user.cnf +++ /dev/null @@ -1,20 +0,0 @@ -[ req ] -default_bits = 2048 -encrypt_key = yes -distinguished_name = req_dn -prompt = no -req_extensions = v3_req - -# Generates the following subject -# Subject: O=DIRAC CI, O=CERN, CN=ciuser -[ req_dn ] -C = ch -O = DIRAC -OU = DIRAC CI -CN = ciuser - -[ v3_req ] -# Extensions for client certificates (`man x509v3_config`). -nsComment = "OpenSSL Generated Client Certificate" -keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index b57ddb93fd5..424f9d020e2 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -90,10 +90,7 @@ source "${TESTCODE}/DIRAC/tests/Jenkins/utilities.sh" installSite() { echo "==> [installSite]" - generateCA - generateCertificates - - echo -n > "${SERVERINSTALLDIR}/dirac-ci-install.cfg" + # echo -n > "${SERVERINSTALLDIR}/dirac-ci-install.cfg" getCFGFile echo "==> Fixing install.cfg file" @@ -127,7 +124,30 @@ installSite() { bash "installer.sh" rm "installer.sh" echo "source \"$PWD/diracos/diracosrc\"" > "$PWD/bashrc" - mv "${SERVERINSTALLDIR}/etc/grid-security/"* "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + + mkdir -p "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/" + mkdir -p "${SERVERINSTALLDIR}/user/" + + echo "==> CAs and certificates" + + # Copy the CA to the list of trusted CA + cp "/ca/certs/ca.cert.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/" + + # Copy the cert and host key to the certificates directory + cp /ca/certs/hostcert.pem "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + cp /ca/certs/hostkey.pem "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + + # Generate the hash link file required by openSSL to index CA certificates + caHash=$(openssl x509 -in "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/ca.cert.pem" -noout -hash) + # We make a relative symlink on purpose (i.e. not the full path to ca.cert.pem) + # because otherwise the BundleDeliveryClient will send the full path, which + # will be wrong on the client + ln -s "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/ca.cert.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/$caHash.0" + + # Copy the user cert and key to the correct directory + cp /ca/certs/client.pem "${SERVERINSTALLDIR}/user/" + cp /ca/certs/client.key "${SERVERINSTALLDIR}/user/" + rm -rf "${SERVERINSTALLDIR}/etc" ln -s "${SERVERINSTALLDIR}/diracos/etc" "${SERVERINSTALLDIR}/etc" source diracos/diracosrc @@ -136,7 +156,6 @@ installSite() { done cd - - echo "==> Sourcing bashrc" source "${SERVERINSTALLDIR}/bashrc" @@ -212,13 +231,6 @@ fullInstallDIRAC() { cat "${SERVERINSTALLDIR}/diracos/etc/Production.cfg" fi - # Dealing with security stuff - # generateCertificates - if ! generateUserCredentials; then - echo "ERROR: generateUserCredentials failed" >&2 - exit 1 - fi - if ! diracCredentials; then echo "ERROR: diracCredentials failed" >&2 exit 1 diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index d920556dd75..89faf966f09 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -370,182 +370,6 @@ getUserProxy() { echo '==> Done getUserProxy' } - -#------------------------------------------------------------------------------- -# OPEN SSL... let's create a fake CA and certificates -#------------------------------------------------------------------------------- - - -# function generateCA() -# -# This generates the CA that will be used to sign the server and client certificates - -generateCA() { - echo '==> [generateCA]' - - mkdir -p "${SERVERINSTALLDIR}/etc/grid-security/certificates" - mkdir -p "${SERVERINSTALLDIR}/etc/grid-security/ca/" - if ! cd "${SERVERINSTALLDIR}/etc/grid-security/ca"; then - echo "ERROR: cannot change to ${SERVERINSTALLDIR}/etc/grid-security/ca" >&2 - exit 1 - fi - - # Initialize the ca - mkdir -p newcerts certs crl - touch index.txt - echo 1000 > serial - echo 1000 > crlnumber - - # Create the CA key - openssl genrsa -out ca.key.pem 2048 # for unencrypted key - chmod 400 ca.key.pem - - - # Prepare OpenSSL config file, it contains extensions to put into place, - # DN configuration, etc.. - cp "${CI_CONFIG}/openssl_config_ca.cnf" "openssl_config_ca.cnf" - sed -i "s|#GRIDSECURITY#|${SERVERINSTALLDIR}/etc/grid-security|g" openssl_config_ca.cnf - - - # Generate the CA certificate - openssl req -config openssl_config_ca.cnf \ - -key ca.key.pem \ - -new -x509 \ - -days 7300 \ - -sha256 \ - -extensions v3_ca \ - -out ca.cert.pem - - # Copy the CA to the list of trusted CA - cp ca.cert.pem "${SERVERINSTALLDIR}/etc/grid-security/certificates/" - - # Generate the hash link file required by openSSL to index CA certificates - caHash=$(openssl x509 -in ca.cert.pem -noout -hash) - # We make a relative symlink on purpose (i.e. not the full path to ca.cert.pem) - # because otherwsie the BundleDeliveryClient will send the full path, which - # will be wrong on the client - ln -s "ca.cert.pem" "${SERVERINSTALLDIR}/etc/grid-security/certificates/$caHash.0" -} - -#............................................................................. -# -# function generateCertificates -# -# This function generates a random host certificate ( certificate and key ), -# which will be stored on etc/grid-security. -# We use the self signed CA created by generateCA function -# In real, we'd copy them from -# CVMFS: -# /cvmfs/grid.cern.ch/etc/grid-security/certificates -# -# Additional info: -# http://www.openssl.org/docs/apps/req.html -# -#............................................................................. - -generateCertificates() { - echo '==> [generateCertificates]' - nDays=${1:-7} - - mkdir -p "${SERVERINSTALLDIR}/etc/grid-security/" - if ! cd "${SERVERINSTALLDIR}/etc/grid-security/"; then - echo "ERROR: cannot change to ${SERVERINSTALLDIR}/etc/grid-security/" >&2 - exit 1 - fi - - # Generate private RSA key - openssl genrsa -out hostkey.pem 2048 &> /dev/null - chmod 400 hostkey.pem - - # Prepare OpenSSL config file, it contains extensions to put into place, - # DN configuration, etc.. - cp "${CI_CONFIG}/openssl_config_host.cnf" "openssl_config_host.cnf" - - # man hostname to see why we use --all-fqdns - # Note: if there's no dns entry for the localhost, the fqdns will be empty - # so we append to it the local hostname, and we take the first one in the list - fqdn=$( (hostname --all-fqdn; hostname ) | paste -sd ' ' | awk '{print $1}') - sed -i "s/#hostname#/$fqdn/g" openssl_config_host.cnf - - # Generate X509 Certificate request based on the private key and the OpenSSL configuration - # file, valid for nDays days (default 1). - openssl req -config openssl_config_host.cnf \ - -key hostkey.pem \ - -new \ - -sha256 \ - -out request.csr.pem - - # Sign it using the self generated CA - openssl ca -config "${SERVERINSTALLDIR}/etc/grid-security/ca/openssl_config_ca.cnf" \ - -days "$nDays" \ - -extensions server_cert \ - -batch \ - -in request.csr.pem \ - -out hostcert.pem - - cd - -} - - -#............................................................................. -# -# generateUserCredentials: -# -# Given we know the "CA" certificates, we can use them to sign a randomly -# generated key / host certificate. This function is very similar to -# generateCertificates. User credentials will be stored at: -# ${SERVERINSTALLDIR}/user -# The user will be called "ciuser". Do not confuse with the admin user, -# which is "ci". -# The argument that can be passed is the validity of the certificate -# -# Additional info: -# http://acs.lbl.gov/~boverhof/openssl_certs.html -# -#............................................................................. - -generateUserCredentials() { - echo '==> [generateUserCredentials]' - - # validity of the certificate - nDays=${1:-7} - - USERCERTDIR=${SERVERINSTALLDIR}/user - # Generate directory where to store credentials - mkdir -p "${USERCERTDIR}" - if ! cd "${USERCERTDIR}"; then - echo "ERROR: cannot change to ${USERCERTDIR}" >&2 - exit 1 - fi - - # What is that ? - save=$- - if [[ $save =~ e ]]; then - set +e - fi - - cp "${CI_CONFIG}/openssl_config_user.cnf" "${USERCERTDIR}/openssl_config_user.cnf" - if [[ $save =~ e ]]; then - set -e - fi - - openssl genrsa -out client.key 2048 &> /dev/null - chmod 400 client.key - - openssl req -config "${USERCERTDIR}/openssl_config_user.cnf" \ - -key "${USERCERTDIR}/client.key" \ - -new \ - -out "$USERCERTDIR/client.req" - - openssl ca -config "${SERVERINSTALLDIR}/etc/grid-security/ca/openssl_config_ca.cnf" \ - -extensions usr_cert \ - -batch \ - -days "$nDays" \ - -in "$USERCERTDIR/client.req" \ - -out "$USERCERTDIR/client.pem" -} - - #............................................................................. # # diracCredentials: