Upgrade CKAN to 2.10 with Python 3

.docker-compose-db.yaml

@@ -1,50 +1,47 @@
-version: '3.2'
-
 services:
-
   db:
     image: viderum/ckan-cloud-docker:db-latest
     build:
       context: db
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/db-secrets.sh
+      - docker-compose/db-secrets.sh
     volumes:
-    - db:/var/lib/postgresql/data
+      - db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   jobs-db:
     image: postgres
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/db-secrets.sh
+      - docker-compose/db-secrets.sh
     volumes:
-    - jobs-db:/var/lib/postgresql/data
+      - jobs-db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   datastore-db:
     image: viderum/ckan-cloud-docker:db-latest
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/datastore-db-secrets.sh
+      - docker-compose/datastore-db-secrets.sh
     volumes:
-    - datastore-db:/var/lib/postgresql/data
+      - datastore-db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   ckan:
     depends_on:
-    - db
-    - jobs-db
-    - datastore-db
+      - db
+      - jobs-db
+      - datastore-db
 
 volumes:
   db:

.docker-compose.vital-strategies-theme.yaml

@@ -1,5 +1,3 @@
-version: '3.2'
-
 services:
 
   proxy:
@@ -21,34 +19,36 @@ services:
     build:
       context: ckan
       args:
-        CKAN_BRANCH: ckan-2.7.3
+        CKAN_BRANCH: ckan-2.10.4
         EXTRA_PACKAGES: cron
         EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/"
-        PRE_INSTALL: "sed  -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt"
+        #PRE_INSTALL: "sed  -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt"
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview && \
-          install_standard_ckan_extension_github -r okfn/ckanext-sentry && \
-          install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \
-          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b fix-null-content-type && \
+          #install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \
+          install_standard_ckan_extension_github -r datopian/ckanext-gtm && \
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l es -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l fr -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l km -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l pt_BR -f && \
-          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_CN -f
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_Hans_CN -f
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-
+    ports: # <-- Add this section to expose port 5000
+      - 5000:5000
 
   jobs:
     image: viderum/ckan-cloud-docker:ckan-latest-vital-strategies-theme
     build:
       context: ckan
       args:
-        CKAN_BRANCH: ckan-2.7.3
+        CKAN_BRANCH: ckan-2.10.4
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r keitaroinc/ckanext-s3filestore -b main &&\
-          install_standard_ckan_extension_github -r datopian/ckanext-querytool &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-

DEPLOYING.md

@@ -115,7 +115,7 @@ In addition to SSL specific configuration, there is one more line you need to ad
 
 This should be enough for the basic installation. In case you need to tweak versions or other initialization parameters for CKAN, you need these two files:
 
-* `docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini`
+* `docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini`
   This is the file used to generate the CKAN main configuration file.
 
 * `.docker-compose.{instance-id}-theme.yaml`
@@ -196,7 +196,7 @@ bash migrate_filestorage.sh $HOST $ACCESS_KEY $SECRET_KEY $BUCKET $STORAGE_PATH
 
 After migration rebuild the SOLR search index.
 ```
-sudo make shell O=<<instance-id>> S=ckan C='/usr/local/bin/ckan-paster --plugin=ckan search-index rebuild  -c /etc/ckan/production.ini'
+sudo make shell O=<<instance-id>> S=ckan C='/usr/local/bin/ckan-paster --plugin=ckan search-index rebuild  -c /etc/ckan/ckan.ini'
 ```
 
 ## Debugging
@@ -255,10 +255,10 @@ POST_INSTALL: |
   install_standard_ckan_extension_github -r datopian/ckanext-s3filestore &&\
 ```
 
-And add extension to the list of plugins in `docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template`
+And add extension to the list of plugins in `docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template`
 
 ```
-# in docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template
+# in docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template
 ckan.plugins = image_view
    ...
    stats
@@ -268,7 +268,7 @@ ckan.plugins = image_view
 Note: depending on extension you might also need to update extensions related configurations in the same file. If needed this type of information is ussually included in extension REAMDE.
 
 ```
-# in docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template
+# in docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template
 ckanext.s3filestore.aws_access_key_id = Your-Access-Key-ID
 ckanext.s3filestore.aws_secret_access_key = Your-Secret-Access-Key
 ckanext.s3filestore.aws_bucket_name = a-bucket-to-store-your-stuff

Makefile

@@ -2,20 +2,24 @@
 
 COMPOSE_FILES = -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml
 
+DATAPUSHER_TYPE ?= datapusher
+
 start:
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
 
 stop:
 	docker-compose $(COMPOSE_FILES) stop
 
 build:
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) build
 
 pull:
 	docker-compose $(COMPOSE_FILES) pull
 
 shell:
-	docker-compose $(COMPOSE_FILES) exec $S $C
+	docker-compose $(COMPOSE_FILES) exec -it $S sh -c 'if command -v bash > /dev/null 2>&1; then exec bash; else exec sh; fi'
 
 down:
 	docker-compose $(COMPOSE_FILES) down
@@ -36,10 +40,10 @@ exec:
 	docker-compose $(COMPOSE_FILES) exec $S $C
 
 user:
-	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan ckan -c /etc/ckan/ckan.ini user add $U password=$P email=$E
 
 sysadmin:
-	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan ckan -c /etc/ckan/ckan.ini sysadmin add $U
 
 secret:
 	python create_secrets.py
@@ -49,6 +53,17 @@ cron:
 
 clean-rebuild:
 	docker-compose $(COMPOSE_FILES) down -v
-	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f
+	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs -r docker rmi -f
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) build --no-cache
-	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
+	docker-compose $(COMPOSE_FILES) up -d nginx && make cron
+
+backup-db:
+	docker-compose $(COMPOSE_FILES) exec -T db pg_dump -U postgres --format=custom -d ckan > ckan.dump
+	docker-compose ${COMPOSE_FILES} exec -T ckan sh -c "cd /var/lib/ckan && tar -czf /tmp/ckan_data.tar.gz data"
+	docker cp $$(docker-compose ${COMPOSE_FILES} ps -q ckan):/tmp/ckan_data.tar.gz ckan_data.tar.gz
+	docker-compose $(COMPOSE_FILES) exec -T datastore-db pg_dump -U postgres --format=custom -d datastore > datastore.dump
+
+upgrade-db:
+	./db/migration/upgrade_databases.sh "$(COMPOSE_FILES)"

README.md

@@ -92,7 +92,7 @@ In your project specific `docker-compose` file, you must expose port 5000 for CK
 
 ### Remove unused plugins from CKAN
 
-Before building and starting the environment, make sure you only have the required plugins enabled. If you're using a pre-defined project template for local testing, you might not need some of the included extensions, such as `ckanext-googleanalytics` or `ckanext-sentry`. For example, if you want to use the `vital-strategies` project template, you should remove the following plugins from the `.ini` file (found in `docker-compose/ckan-conf-templates/vital-strategies-theme-production.ini`) to avoid issues (unless you want to properly configure them):
+Before building and starting the environment, make sure you only have the required plugins enabled. If you're using a pre-defined project template for local testing, you might not need some of the included extensions, such as `ckanext-googleanalytics` or `ckanext-sentry`. For example, if you want to use the `vital-strategies` project template, you should remove the following plugins from the `.ini` file (found in `docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini`) to avoid issues (unless you want to properly configure them):
 
 ```
 ckan.plugins = image_view
@@ -154,7 +154,7 @@ Create a CKAN admin user
 
 ```
 docker-compose exec ckan ckan-paster --plugin=ckan \
-    sysadmin add -c /etc/ckan/production.ini admin password=12345678 email=admin@localhost
+    sysadmin add -c /etc/ckan/ckan.ini admin password=12345678 email=admin@localhost
 ```
 
 Login to CKAN at http://nginx:8080 with username `admin` and password `12345678`
@@ -250,8 +250,8 @@ This allows to test different CKAN configurations and extension combinations
 Duplicate the CKAN default configuration:
 
 ```
-cp docker-compose/ckan-conf-templates/production.ini.template \
-   docker-compose/ckan-conf-templates/my-ckan-production.ini.template
+cp docker-compose/ckan-conf-templates/ckan.ini.template \
+   docker-compose/ckan-conf-templates/my-ckan-ckan.ini.template
 ```
 
 Edit the duplicated file and modify the settings, e.g. add the extensions to the `plugins` configuration and any additional required extension configurations.

TESTING.md

@@ -56,7 +56,7 @@ Once you see a successful response, create a CKAN admin user:
 ```
 $ docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.datagov-theme.yaml \
 	  exec ckan ckan-paster --plugin=ckan \
-      sysadmin add -c /etc/ckan/production.ini admin password=12345678 email=admin@localhost
+      sysadmin add -c /etc/ckan/ckan.ini admin password=12345678 email=admin@localhost
 ```
 
 You should see the following prompt:
@@ -153,7 +153,7 @@ $ paster datastore set-permissions -c test-core.ini | psql -h datastore-db -U po
 
 Solr is already configured as 'multi-core'. To verify it, you may run the following command inside the `ckan` container:
 ```
-$ grep solr_url /etc/ckan/production.ini
+$ grep solr_url /etc/ckan/ckan.ini
 # Possible outputs:
 # single-core: solr_url = http://solr:8983/solr
 # multi-core:  solr_url = http://solr:8983/solr/ckan

cca-operator/update-instance.sh

@@ -213,7 +213,7 @@ else
     CKAN_ADMIN_PASSWORD=$(python3 -c "import binascii,os;print(binascii.hexlify(os.urandom(12)).decode())")
     echo y \
         | kubectl $KUBECTL_GLOBAL_ARGS -n ${INSTANCE_NAMESPACE} exec -it ${CKAN_POD_NAME} -- bash -c \
-            "ckan-paster --plugin=ckan sysadmin -c /etc/ckan/production.ini add admin password=${CKAN_ADMIN_PASSWORD} email=${CKAN_ADMIN_EMAIL}" \
+            "ckan -c /etc/ckan/ckan.ini sysadmin add admin password=${CKAN_ADMIN_PASSWORD} email=${CKAN_ADMIN_EMAIL}" \
                 > /dev/stderr &&\
     kubectl $KUBECTL_GLOBAL_ARGS -n "${INSTANCE_NAMESPACE}" \
         create secret generic ckan-admin-password "--from-literal=CKAN_ADMIN_PASSWORD=${CKAN_ADMIN_PASSWORD}"