From 316f7ad91c67ebf8d848bc4b43cd7b0c81f75fdf Mon Sep 17 00:00:00 2001 From: NuCivic Busy Squirrel Date: Thu, 20 Jul 2017 09:06:27 -0300 Subject: [PATCH] Remake dkan_starter/dkan 1.13.5.2 (#384) * Add new config.yml param stage_file_proxy_files: REF CIVIC-6610 defaul/stage_file_porxy_files: holds a list of files that we may want to move from production site into the testing environment in order to ensure tests pass. * remake dkan with patch. 1.13.5 + 2018 * Use ahoy dkan server. --- assets/templates/circle.yml.erb | 2 +- build-dkan.make | 2 +- circle.yml | 2 +- config/config.php | 3 + config/config.yml | 1 + dkan/.ahoy/.scripts/.ht.router.php | 42 +++++++ dkan/.ahoy/.scripts/config.rb | 19 +++- dkan/.ahoy/.scripts/dkan-uli.rb | 16 +++ dkan/.ahoy/.scripts/drupal-rebuild.sh | 9 +- dkan/.ahoy/.scripts/server.sh | 12 ++ dkan/.ahoy/dkan.ahoy.yml | 33 +++--- dkan/.ahoy/docker-compose.yml | 19 +++- dkan/.ahoy/docker.ahoy.yml | 106 +++++++++++++++--- dkan/PATCHES.txt | 4 + dkan/circle.yml | 2 +- dkan/drupal-org.make | 2 +- .../dkan_dataset_groups.views_default.inc | 5 +- ...est_dashboard.features.user_permission.inc | 42 ------- .../dkan_migrate_base.module | 2 + dkan/test/README.md | 31 +++++ .../DKANExtension/Context/PODContext.php | 9 +- .../DKANExtension/Context/RawDKANContext.php | 41 +++++++ dkan/test/features/dataset_rest_api.feature | 10 +- dkan/test/features/datastore.feature | 2 +- dkan/test/features/group.all.feature | 5 +- dkan/test/features/pod.feature | 18 +-- dkan/test/features/user.admin.feature | 2 + dkan/test/features/user.all.feature | 18 +-- dkan/test/features/workflow.feature | 42 +++---- 29 files changed, 367 insertions(+), 134 deletions(-) create mode 100644 dkan/.ahoy/.scripts/.ht.router.php create mode 100644 dkan/.ahoy/.scripts/dkan-uli.rb create mode 100644 dkan/.ahoy/.scripts/server.sh create mode 100644 dkan/PATCHES.txt delete mode 100644 dkan/modules/dkan/dkan_harvest/modules/dkan_harvest_dashboard/dkan_harvest_dashboard.features.user_permission.inc diff --git a/assets/templates/circle.yml.erb b/assets/templates/circle.yml.erb index 0f7049750..7718a25f4 100644 --- a/assets/templates/circle.yml.erb +++ b/assets/templates/circle.yml.erb @@ -45,7 +45,7 @@ dependencies: - ahoy site up --db-user=ubuntu --db-pass='' --db-host=127.0.0.1 --db-port=3306 --db-name=circle_test - ahoy utils fail-when-bad-disable # Run a webserver using drush. - - 'ahoy drush --yes runserver :8888': + - 'ahoy dkan server': background: true - wget http://selenium-release.storage.googleapis.com/2.53/selenium-server-standalone-2.53.1.jar diff --git a/build-dkan.make b/build-dkan.make index 0bd1cc630..f9f343338 100644 --- a/build-dkan.make +++ b/build-dkan.make @@ -10,4 +10,4 @@ projects[dkan][download][type] = git projects[dkan][download][url] = https://github.com/NuCivic/dkan.git projects[dkan][download][tag] = 7.x-1.13.5 -projects[dkan][patch][] = https://patch-diff.githubusercontent.com/raw/NuCivic/dkan/pull/2012.diff +projects[dkan][patch][] = https://patch-diff.githubusercontent.com/raw/NuCivic/dkan/pull/2018.diff diff --git a/circle.yml b/circle.yml index 9ca7b2f9a..66c283329 100644 --- a/circle.yml +++ b/circle.yml @@ -45,7 +45,7 @@ dependencies: - ahoy site up --db-user=ubuntu --db-pass='' --db-host=127.0.0.1 --db-port=3306 --db-name=circle_test - ahoy utils fail-when-bad-disable # Run a webserver using drush. - - 'ahoy drush --yes runserver :8888': + - 'ahoy dkan server': background: true - wget http://selenium-release.storage.googleapis.com/2.53/selenium-server-standalone-2.53.1.jar diff --git a/config/config.php b/config/config.php index 31aa03d91..c32eb0144 100644 --- a/config/config.php +++ b/config/config.php @@ -80,6 +80,9 @@ array ( 'enable' => false, ), + 'stage_file_proxy_files' => + array ( + ), 'stage_file_proxy_origin' => 'changeme', ), 'gaClientTrackingCode' => 'UA-XXXXX-Y', diff --git a/config/config.yml b/config/config.yml index 6c691983e..67dfb2050 100644 --- a/config/config.yml +++ b/config/config.yml @@ -45,6 +45,7 @@ default: https_securepages: false odfe: enable: false + stage_file_proxy_files: [] stage_file_proxy_origin: changeme gaClientTrackingCode: UA-XXXXX-Y gaNuCivicTrackingCode: UA-XXXXX-Z diff --git a/dkan/.ahoy/.scripts/.ht.router.php b/dkan/.ahoy/.scripts/.ht.router.php new file mode 100644 index 000000000..9fa2cfcd8 --- /dev/null +++ b/dkan/.ahoy/.scripts/.ht.router.php @@ -0,0 +1,42 @@ += 3.0 +# TODO: replace defaults with custom deep_merge + begin CONFIG = YAML.load_file("config/config.yml") rescue Exception => msg puts "Loading of Configuration errored out with: #{msg}." puts "Using default CONFIG instead." - CONFIG = {"circle" => {"skip_features" => []}} + CONFIG = {} +end + +if not CONFIG.has_key? "circle" + CONFIG["circle"] = {} end +if not CONFIG["circle"].has_key? "skip_features" + CONFIG["circle"]["skip_features"] = [] +end +if not CONFIG.has_key? "default" + CONFIG["default"] = {} +end + +if not CONFIG["default"].has_key? "https_everywhere" + CONFIG["default"]["https_everywhere"] = false +end diff --git a/dkan/.ahoy/.scripts/dkan-uli.rb b/dkan/.ahoy/.scripts/dkan-uli.rb new file mode 100644 index 000000000..0ab500957 --- /dev/null +++ b/dkan/.ahoy/.scripts/dkan-uli.rb @@ -0,0 +1,16 @@ +require "./dkan/.ahoy/.scripts/config.rb" + +if CONFIG["default"]["https_everywhere"] + url="surl" +else + url="url" +end + +url=`ahoy drush --uri="$(ahoy docker #{url})" uli` +os =`uname` + +if os =~ /Darwin/ + `open #{url}` +else + puts url +end diff --git a/dkan/.ahoy/.scripts/drupal-rebuild.sh b/dkan/.ahoy/.scripts/drupal-rebuild.sh index 0c2271249..51fdcf4d0 100644 --- a/dkan/.ahoy/.scripts/drupal-rebuild.sh +++ b/dkan/.ahoy/.scripts/drupal-rebuild.sh @@ -12,8 +12,13 @@ fi drush --root=docroot make --concurrency=$concurrency --prepare-install dkan/drupal-org-core.make docroot --yes -drush --root=docroot -y --verbose $root si minimal --sites-subdir=default --account-pass='admin' --db-url=$1 install_configure_form.update_status_module='array(false,false)' && +drush --root=docroot -y --verbose si minimal --sites-subdir=default --account-pass='admin' --db-url=$1 install_configure_form.update_status_module='array(false,false)' && ln -s ../../dkan docroot/profiles/dkan chmod +w docroot/sites/default/settings.php -printf "// DKAN Datastore Fast Import options.\n\$databases['default']['default']['pdo'] = array(\n PDO::MYSQL_ATTR_LOCAL_INFILE => 1,\n PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => 1,\n);" >> docroot/sites/default/settings.php + +if [ "$AHOY_CMD_PROXY" = "DOCKER" ]; then + printf "// Docker Database Settings\n\$databases['default']['default'] = array(\n 'database' => 'drupal',\n 'username' => 'drupal',\n 'password' => '123',\n 'host' => 'db',\n 'port' => '',\n 'driver' => 'mysql',\n 'prefix' => '',\n);\n" >> docroot/sites/default/settings.php +fi +printf "// DKAN Datastore Fast Import options.\n\$databases['default']['default']['pdo'] = array(\n PDO::MYSQL_ATTR_LOCAL_INFILE => 1,\n PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => 1,\n);\n" >> docroot/sites/default/settings.php + chmod -w docroot/sites/default/settings.php diff --git a/dkan/.ahoy/.scripts/server.sh b/dkan/.ahoy/.scripts/server.sh new file mode 100644 index 000000000..7cab895b8 --- /dev/null +++ b/dkan/.ahoy/.scripts/server.sh @@ -0,0 +1,12 @@ + +if [ "$HOSTNAME" = "cli" ]; then + HOST=cli +else + HOST=localhost +fi + +cp ./dkan/.ahoy/.scripts/.ht.router.php ./docroot/ +cd ./docroot + +echo $HOST +php -S $HOST:8888 .ht.router.php \ No newline at end of file diff --git a/dkan/.ahoy/dkan.ahoy.yml b/dkan/.ahoy/dkan.ahoy.yml index 73a22ae0c..b27440566 100644 --- a/dkan/.ahoy/dkan.ahoy.yml +++ b/dkan/.ahoy/dkan.ahoy.yml @@ -4,11 +4,14 @@ commands: drupal-rebuild: usage: 'Builds a drupal site in ./docroot. Requires database url.' cmd: | - if [ "$AHOY_CMD_PROXY" != "DOCKER" ]; then - ARGS="{{args}}" - else + if [ "$AHOY_CMD_PROXY" = "DOCKER" ]; then ARGS=`ahoy docker mysql-url` fi + + # If no mysql URL from docker, use passed in args. + if [ "$ARGS" = "" ];then + ARGS="{{args}}" + fi if [[ "$ARGS" != mysql* ]]; then echo "You need to specify the drupal db url to use in the form mysql://root:root@localhost/db as an argument." exit 1 @@ -44,9 +47,11 @@ commands: mkdir backups fi if [ -f backups/last_install.sql ] && ahoy confirm "An existing installation backup exists at backups/last_install.sql, do you want to use that instead of reinstalling from scratch?"; then - ahoy drush sql-drop -y - ahoy drush cc all - ahoy drush sql-cli < backups/last_install.sql + + ahoy drush sql-drop -y && \ + echo "... Removed tables, restoring DB" + ahoy dkan sqlc < backups/last_install.sql + echo "Installed dkan from backup" else if [ "$AHOY_CMD_PROXY" == "DOCKER" ]; then @@ -193,15 +198,13 @@ commands: import: dkan/.ahoy/theme.ahoy.yml hide: true + # Moved to .ahoy.yml uli: - usage: log into the + usage: Log in via drush cmd: | - uli=$(ahoy drush uli {{args}} | sed 's/^http.*\/default\/\(.*$\)/\1/g') - url="$(ahoy docker url)/$uli" - os=$(uname) + ruby dkan/.ahoy/.scripts/dkan-uli.rb {{args}} - if [ "$os" = "Darwin" ]; then - open $url - else - echo $url - fi + server: + usage: Provided as an easy way to setup the php server during testing. + cmd: + ahoy cmd-proxy bash dkan/.ahoy/.scripts/server.sh diff --git a/dkan/.ahoy/docker-compose.yml b/dkan/.ahoy/docker-compose.yml index aa5fbc77f..cde5b8155 100644 --- a/dkan/.ahoy/docker-compose.yml +++ b/dkan/.ahoy/docker-compose.yml @@ -5,12 +5,17 @@ web: hostname: web image: nuams/drupal-apache-php:1.0-5.6 + # Fix for docker for mac container not starting selenium + # https://forums.docker.com/t/space-in-new-no-proxy-env-var-breaking-old-containers/14075 + environment: - VIRTUAL_HOST=dkan.docker + - no_proxy=localhost #- VIRTUAL_PROTO=https ports: - "80" - "443" + - "8888" volumes: # PHP configuration overrides - "./.docker/etc/php5/php.ini:/etc/php5/fpm/conf.d/z_php.ini" @@ -49,6 +54,7 @@ cli: environment: - XDEBUG_CONFIG=idekey=cli - PHP_IDE_CONFIG=serverName=dkan.docker + - VIRTUAL_HOST=dkan.docker volumes: # PHP configuration overrides - "./.docker/etc/php5/php-cli.ini:/etc/php5/cli/conf.d/z_php.ini" @@ -64,6 +70,8 @@ cli: - browser - memcached - solr + ports: + - "8888" # Memcached node # Uncomment the service definition section below and the link in the web service above to start using memcached. @@ -96,16 +104,21 @@ browser: - /dev/shm:/dev/shm # Project root folder mapping - *project_root + links: - web ports: - # - "4444:4444" - "5900" + # Fix for docker for mac container not starting selenium + # https://forums.docker.com/t/space-in-new-no-proxy-env-var-breaking-old-containers/14075 + environment: + - no_proxy=localhost + solr: hostname: solr image: devinci/drupal-solr:3.x - #ports: - # - "8983:8983" + ports: + - "8983" volumes: - "./.docker/etc/solr/3.x:/var/lib/solr/conf" diff --git a/dkan/.ahoy/docker.ahoy.yml b/dkan/.ahoy/docker.ahoy.yml index bb29fcc9c..5bf0eeaad 100644 --- a/dkan/.ahoy/docker.ahoy.yml +++ b/dkan/.ahoy/docker.ahoy.yml @@ -5,39 +5,98 @@ commands: cmd: | cat << EOF # To initialize your environment to use docker. - # Run: eval \$(ahoy docker env) + # Run: eval "\$(ahoy docker env)" # OR # Add the following to your ~/.bashrc which docker || (echo "you don't seem to have docker installed. Exiting."; exit 1) - which docker-machine || (echo "you don't seem to have docker-machine installed. Exiting."; exit 1) which docker-compose || (echo "you don't seem to have docker-compose installed. Exiting."; exit 1) echo "Setting up docker environment" export AHOY_CMD_PROXY=DOCKER - docker-machine start default - $(docker-machine env default) + which docker-machine || (echo "you don't seem to have docker-machine installed. Exiting."; exit 1) + if [ ! -z "$DOCKER_MACHINE_NAME" ]; then + docker-machine start default + $(docker-machine env default) + fi + ahoy docker up EOF usage: Outputs the commands needed to setup a docker environment. + hostfile: + cmd: | + ip_address="127.0.0.1" + host_name="dkan.docker" + # find existing instances in the host file and save the line numbers + matches_in_hosts="$(grep -n $host_name /etc/hosts | cut -f1 -d:)" + host_entry="${ip_address} ${host_name}" + + if [ -z "$matches_in_hosts" ];then + echo "Adding new hosts entry." + echo "$host_entry" | sudo tee -a /etc/hosts > /dev/null + fi up: - cmd: "ahoy docker compose up -d {{args}}" + cmd: | + if [ -z $AHOY_CMD_PROXY ]; then + echo "AHOY_CMD_PROXY is not set. Run 'ahoy docker env' and follow instructions." + exit 0 + fi + ahoy docker compose up -d {{args}} + ahoy docker proxy-up + ahoy docker hostfile usage: Start the docker-compose containers. proxy-up: - cmd: "docker run -d -p 80:80 -v /var/run/docker.sock:/tmp/docker.sock:ro jwilder/nginx-proxy" + cmd: | + proxy_image=jwilder/nginx-proxy + # Check if the proxy container is running. + if [ ! "$(docker ps -f name=dkan_proxy -f ancestor=$proxy_image -q)" ];then + # Check if the proxy container stopped. + if [ "$(docker ps -f name=dkan_proxy -f ancestor=$proxy_image -qa)" ];then + echo "Restarting dkan_proxy container" + docker start dkan_proxy + else + echo "Starting dkan_proxy container" + docker run -d --privileged \ + -p 80:80 -p 443:443 -p 5959:5900 \ + -v /var/run/docker.sock:/tmp/docker.sock:ro \ + --name=dkan_proxy jwilder/nginx-proxy + fi + fi usage: Run the nginx-proxy container stop: - cmd: "ahoy docker compose stop" + cmd: | + ahoy docker compose stop + proxy_image=jwilder/nginx-proxy + if [ ! -z "$(docker ps -f name=dkan_proxy -f ancestor=$proxy_image -q)" ];then + echo "Stopping dkan_proxy container" + docker stop dkan_proxy + fi usage: Stop the docker-compose containers (non-destructive). ps: cmd: "ahoy docker compose ps" usage: List the running docker-compose containers. + + # Alias for web-ip. ip: - cmd: "docker-machine ip default" - usage: Show the ip address f the default docker machine VM + cmd: | + if [ -z "$DOCKER_MACHINE_NAME" ]; then + ahoy docker web-ip + else + docker-machine ip default + fi + + usage: Show the ip address of the web container reset: - cmd: "ahoy docker compose stop && ahoy docker compose rm && ahoy docker compose up -d" + cmd: | + ahoy docker destroy + ahoy docker up usage: Destroy and then restart the docker compose containers. destroy: - cmd: "ahoy docker compose stop && ahoy docker compose rm" + cmd: | + ahoy docker stop + ahoy docker compose rm + proxy_image=jwilder/nginx-proxy + if [ ! -z "$(docker ps -f name=dkan_proxy -f ancestor=$proxy_image -all -q)" ];then + docker rm dkan_proxy + fi usage: Destroy all the docker compose containers. (use before deleting folder) exec: cmd: | @@ -82,18 +141,37 @@ commands: compose: cmd: docker-compose -f dkan/.ahoy/docker-compose.yml -p "${PWD##*/}" {{args}} usage: Abstraction for docker-compose + web-host: + cmd: | + echo `ahoy docker exec web printenv VIRTUAL_HOST | tr -d '\r'` + web-ip: + usage: Prints the web container IP address + cmd: | + echo `docker inspect --format '{{ .NetworkSettings.IPAddress }}' $(ahoy docker compose ps -q web)` url: usage: Prints the project URL cmd: | - echo "http://`docker-machine ip default`:`ahoy docker compose port web 80|cut -d ':' -f2`" + if [ -z "$DOCKER_MACHINE_NAME" ]; then + echo "http://`ahoy docker web-host`" + else + echo "http://`docker-machine ip default`:`ahoy docker compose port web 80|cut -d ':' -f2`" + fi surl: usage: Prints a secure project URL cmd: | - echo "https://`docker-machine ip default`:`ahoy docker compose port web 443|cut -d ':' -f2`" + if [ -z "$DOCKER_MACHINE_NAME" ]; then + echo "https://`ahoy docker web-host`:`ahoy docker compose port web 443|cut -d ':' -f2`" + else + echo "https://`docker-machine ip default`:`ahoy docker compose port web 443|cut -d ':' -f2`" + fi vnc: usage: Prints the project VNC URL cmd: | - echo "`docker-machine ip default`:`ahoy docker compose port browser 5900|cut -d ':' -f2`" + if [ -z "$DOCKER_MACHINE_NAME" ]; then + echo "https://`ahoy docker web-host`:`ahoy docker compose port browser 5900|cut -d ':' -f2`" + else + echo "`docker-machine ip default`:`ahoy docker compose port browser 5900|cut -d ':' -f2`" + fi cleanup: usage: Clean ups docker unused images and volumes. See http://blog.yohanliyanage.com/2015/05/docker-clean-up-after-yourself/. diff --git a/dkan/PATCHES.txt b/dkan/PATCHES.txt new file mode 100644 index 000000000..a2c16599c --- /dev/null +++ b/dkan/PATCHES.txt @@ -0,0 +1,4 @@ +The following patches have been applied to this project: +- https://patch-diff.githubusercontent.com/raw/NuCivic/dkan/pull/2018.diff + +This file was automatically generated by Drush Make (http://drupal.org/project/drush). \ No newline at end of file diff --git a/dkan/circle.yml b/dkan/circle.yml index 169f9000e..adf47e9d9 100644 --- a/dkan/circle.yml +++ b/dkan/circle.yml @@ -60,7 +60,7 @@ dependencies: - 'PATH=/home/ubuntu/.config/composer/vendor/bin:$PATH ahoy drush --yes en dkan_harvest dkan_harvest_datajson dkan_harvest_dashboard' - 'PATH=/home/ubuntu/.config/composer/vendor/bin:$PATH ahoy drush cc all' # Run a webserver using drush. - - 'PATH=/home/ubuntu/.config/composer/vendor/bin:$PATH ahoy drush --yes runserver :8888': + - 'PATH=/home/ubuntu/.config/composer/vendor/bin:$PATH ahoy dkan server': background: true # Setup display for selenium diff --git a/dkan/drupal-org.make b/dkan/drupal-org.make index 1978970d0..acab00ccb 100644 --- a/dkan/drupal-org.make +++ b/dkan/drupal-org.make @@ -407,4 +407,4 @@ libraries: directory_name: spyc defaults: projects: - subdir: contrib + subdir: contrib \ No newline at end of file diff --git a/dkan/modules/dkan/dkan_dataset/modules/dkan_dataset_groups/dkan_dataset_groups.views_default.inc b/dkan/modules/dkan/dkan_dataset/modules/dkan_dataset_groups/dkan_dataset_groups.views_default.inc index 99cb37341..f7571c015 100644 --- a/dkan/modules/dkan/dkan_dataset/modules/dkan_dataset_groups/dkan_dataset_groups.views_default.inc +++ b/dkan/modules/dkan/dkan_dataset/modules/dkan_dataset_groups/dkan_dataset_groups.views_default.inc @@ -300,7 +300,10 @@ function dkan_dataset_groups_views_default_views() { $handler = $view->new_display('default', 'Master', 'default'); $handler->display->display_options['title'] = 'DKAN OG Extras group members (User)'; $handler->display->display_options['use_more_always'] = FALSE; - $handler->display->display_options['access']['type'] = 'perm'; + $handler->display->display_options['access']['type'] = 'role'; + $handler->display->display_options['access']['role'] = array( + 2 => '2', + ); $handler->display->display_options['cache']['type'] = 'none'; $handler->display->display_options['query']['type'] = 'views_query'; $handler->display->display_options['exposed_form']['type'] = 'basic'; diff --git a/dkan/modules/dkan/dkan_harvest/modules/dkan_harvest_dashboard/dkan_harvest_dashboard.features.user_permission.inc b/dkan/modules/dkan/dkan_harvest/modules/dkan_harvest_dashboard/dkan_harvest_dashboard.features.user_permission.inc deleted file mode 100644 index 9d706b856..000000000 --- a/dkan/modules/dkan/dkan_harvest/modules/dkan_harvest_dashboard/dkan_harvest_dashboard.features.user_permission.inc +++ /dev/null @@ -1,42 +0,0 @@ - 'use Rules component rules_dkan_harvest_source_cache', - 'roles' => array( - 'administrator' => 'administrator', - ), - 'module' => 'rules', - ); - - // Exported permission: 'use Rules component rules_dkan_harvest_source_cache_and_migrate'. - $permissions['use Rules component rules_dkan_harvest_source_cache_and_migrate'] = array( - 'name' => 'use Rules component rules_dkan_harvest_source_cache_and_migrate', - 'roles' => array( - 'administrator' => 'administrator', - ), - 'module' => 'rules', - ); - - // Exported permission: 'use Rules component rules_dkan_harvest_source_migrate'. - $permissions['use Rules component rules_dkan_harvest_source_migrate'] = array( - 'name' => 'use Rules component rules_dkan_harvest_source_migrate', - 'roles' => array( - 'administrator' => 'administrator', - ), - 'module' => 'rules', - ); - - return $permissions; -} diff --git a/dkan/modules/dkan/dkan_migrate_base/dkan_migrate_base.module b/dkan/modules/dkan/dkan_migrate_base/dkan_migrate_base.module index 86e4b0aa6..d1a2f6703 100644 --- a/dkan/modules/dkan/dkan_migrate_base/dkan_migrate_base.module +++ b/dkan/modules/dkan/dkan_migrate_base/dkan_migrate_base.module @@ -5,6 +5,8 @@ * Required module file. */ +include_once 'dkan_migrate_base.migrate.inc'; + /** * We should pull out this from here to another more generic module. * diff --git a/dkan/test/README.md b/dkan/test/README.md index 1b943f46d..19dadf165 100644 --- a/dkan/test/README.md +++ b/dkan/test/README.md @@ -16,6 +16,37 @@ Assuming you have a working DKAN installation you wish to test on: Your tests should run from the VM and use your host machine as a Selenium server, meaning any Selenium tests will run in an instance of Chrome on your machine. + +### Behat Tags + - **@add_ODFE** Enables ODFE + - **@ahoyRunMe** label only + - **@api** Enables the Drupal API Driver + - **@customizable** Exclude scenario on client sites when testing customizable functionality + - **@datastore** Drops the table after testing + - **@deleteTempUsers** Delete any tempusers that were created outside of 'Given users' + - **@disablecaptcha** Disables captcha config if it is enabled, then restores config after the test + - **@dkanBug** label only + - **@enableFastImport** Enables fast import + - **@enableDKAN_Workflow** Enables dkan_workflow + - **@fixme** label only + - **@globalUser** Populates the global user with the current user + - **@javascript** switches the current Mink session to Selenium2 + - **@mail** Setup the testing mail system, then restore original mail system + - **@no-main-menu** used to skip tests that requires a link in the main menu + - **@noworkflow** label only + - **@ok** label only + - **@pod_json_valid** label only + - **@pod_json_odfe** label only + - **@remove_ODFE** Disables ODFE + - **@testBug** label only + - **@timezone** Sets the timezone for tests and restores the timezone afterwards. + - **@Topics** label only + + **Unique tag per scenario pattern** + + To allow customized sites to skip specific tests we are adding a unique tag to every scenario. The pattern is the feature name followed by a two digit numerical value. So the pod.feature scenarios are tagged like this: @pod_01, @pod_02, @pod_03, etc. + + ## PHPUnit tests Starting from 1.13 PHPUnit tests were added into DKAN core. All tests can be found inside the `/phpunit` directory separated in different test suites, one per DKAN module. diff --git a/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/PODContext.php b/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/PODContext.php index 486e9ec13..fbfa87a00 100644 --- a/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/PODContext.php +++ b/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/PODContext.php @@ -34,19 +34,12 @@ public function gatherContexts(BeforeScenarioScope $scope){ */ public function iSeeAValidDataJson($should) { - // Change /data.json path to /json during tests. The '.' on the filename breaks tests on CircleCI's server. $data_json = open_data_schema_map_api_load('data_json_1_1'); - if ($data_json->endpoint !== 'json') { - $data_json->endpoint = 'json'; - drupal_write_record('open_data_schema_map', $data_json, 'id'); - drupal_static_reset('open_data_schema_map_api_load_all'); - menu_rebuild(); - } // Get base URL. $url = $this->getMinkParameter('base_url') ? $this->getMinkParameter('base_url') : "http://127.0.0.1::8888"; // Validate POD. - $results = open_data_schema_pod_process_validate($url . '/json', TRUE); + $results = open_data_schema_pod_process_validate($url . '/data.json', TRUE); if ($results['errors'] && $should === 'should') { throw new \Exception(sprintf('Data.json is not valid.')); } diff --git a/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/RawDKANContext.php b/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/RawDKANContext.php index 1d1d30faf..5372491d7 100644 --- a/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/RawDKANContext.php +++ b/dkan/test/dkanextension/src/Drupal/DKANExtension/Context/RawDKANContext.php @@ -67,6 +67,43 @@ public static function disableAdminMenuCache(BeforeSuiteScope $scope) { variable_set('admin_menu_cache_client', FALSE); } + /** + * @BeforeSuite + */ + public static function moveStageFileProxyFiles(BeforeSuiteScope $scope) { + // Only need to run this once. + if (variable_get('stage_file_proxy_setup', FALSE)) { + return; + } + + global $conf; + if (!isset($conf['default']['stage_file_proxy_origin']) || $conf['default']['stage_file_proxy_origin'] == 'changeme') { + return; + } + + // Fix missing font files. + $font_files = array('eot', 'svg', 'ttf', 'woff'); + + // Add the file usage. + foreach ($font_files as $ext) { + $filename = 'dkan-topics'; + $theme_path = drupal_get_path('theme', 'nuboot_radix'); + $source = $theme_path . '/assets/fonts/' . $filename . '.' . $ext; + $destination = 'public://' . $filename . '.' . $ext; + copy($source, $destination); + } + + if (isset($conf['default']['stage_file_proxy_files'])) { + $proxy_files = (array) $conf['default']['stage_file_proxy_files']; + foreach ($proxy_files as $file) { + $source = $conf['default']['stage_file_proxy_origin'] . '/' . $file; + $destination = 'public://' . $file; + $copy($source, $destination); + } + } + variable_set('stage_file_proxy_setup', TRUE); + } + /** * @AfterSuite */ @@ -89,6 +126,8 @@ public function beforeCaptcha() module_load_include('inc', 'captcha', 'captcha'); variable_set('disable_captcha', TRUE); captcha_set_form_id_setting('user_login', 'none'); + captcha_set_form_id_setting('user_pass', 'none'); + captcha_set_form_id_setting('user_register_form', 'none'); captcha_set_form_id_setting('feedback_node_form', 'none'); captcha_set_form_id_setting('comment_node_feedback_form', 'none'); } @@ -131,6 +170,8 @@ public function afterCaptcha() module_load_include('inc', 'captcha', 'captcha'); variable_set('disable_captcha', FALSE); captcha_set_form_id_setting('user_login', 'default'); + captcha_set_form_id_setting('user_pass', 'none'); + captcha_set_form_id_setting('user_register_form', 'none'); captcha_set_form_id_setting('feedback_node_form', 'default'); captcha_set_form_id_setting('comment_node_feedback_form', 'default'); } diff --git a/dkan/test/features/dataset_rest_api.feature b/dkan/test/features/dataset_rest_api.feature index 25becbcf4..4b246462b 100644 --- a/dkan/test/features/dataset_rest_api.feature +++ b/dkan/test/features/dataset_rest_api.feature @@ -20,6 +20,7 @@ Feature: DKAN Dataset REST API | title | published | | Group 01 | Yes | + @dataset_rest_api_01 Scenario: Create a Resource using the 'Dataset REST API' endpoint Given I am on "Search Resources" page Then I should not see "Resource 02" @@ -30,7 +31,7 @@ Feature: DKAN Dataset REST API When I am on "Search Resources" page Then I should see "Resource 02" - @api + @dataset_rest_api_02 @api Scenario: Attach files to Resources using the 'Dataset REST API' endpoint Given I am on "Resource 01" page Then I should not see "Polling_Places_Madison" @@ -48,6 +49,7 @@ Feature: DKAN Dataset REST API And I click "Manage Datastore" Then I should not see "No imported items" + @dataset_rest_api_03 Scenario: Update a Resource using the 'Dataset REST API' endpoint Given I am on "Resource 01" page Then I should not see "The description was modified" @@ -58,6 +60,7 @@ Feature: DKAN Dataset REST API When I am on "Resource 01" page Then I should see "The description was modified" + @dataset_rest_api_04 Scenario: Delete a Resource using the 'Dataset REST API' endpoint Given I am on "Search Resources" page Then I should see "Resource 01" @@ -66,6 +69,7 @@ Feature: DKAN Dataset REST API When I am on "Search Resources" page Then I should not see "Resource 01" + @dataset_rest_api_05 Scenario: Create a Dataset using the 'Dataset REST API' endpoint Given I am on "Search Datasets" page Then I should not see "Dataset 02" @@ -78,6 +82,7 @@ Feature: DKAN Dataset REST API And I am on "/dataset/dataset-02" Then I should see "Resource 01" + @dataset_rest_api_06 Scenario: Update a Dataset using the 'Dataset REST API' endpoint Given I am on "Dataset 01" page Then I should not see "The description was modified" @@ -88,6 +93,7 @@ Feature: DKAN Dataset REST API When I am on "Dataset 01" page Then I should see "The description was modified" + @dataset_rest_api_07 Scenario: Delete a Dataset using the 'Dataset REST API' endpoint Given I am on "Search Datasets" page Then I should see "Dataset 01" @@ -95,5 +101,3 @@ Feature: DKAN Dataset REST API And I use the "dataset rest api" endpoint to delete the node "Dataset 01" When I am on "Search Datasets" page Then I should not see "Dataset 01" - - diff --git a/dkan/test/features/datastore.feature b/dkan/test/features/datastore.feature index 0c850d595..d73f746fc 100644 --- a/dkan/test/features/datastore.feature +++ b/dkan/test/features/datastore.feature @@ -53,7 +53,7 @@ Feature: Datastore Then I wait for "DKAN Datastore Link Importer: Status" When I press "Import" And I wait for "2 imported items total." - When I click "Data API" + When I click "Data API" in the "primary tabs" region Then I wait for "Example Query" When I click "Manage Datastore" Then I wait for "DKAN Datastore Link Importer: Status" diff --git a/dkan/test/features/group.all.feature b/dkan/test/features/group.all.feature index 56d41983f..a8891ba2d 100644 --- a/dkan/test/features/group.all.feature +++ b/dkan/test/features/group.all.feature @@ -75,6 +75,7 @@ Feature: Site Manager administer groups When I follow "Groups" And I should not see "Group 03" + @customizable Scenario: View the details of a published group Given I am on "Groups" page When I follow "Group 01" @@ -85,6 +86,7 @@ Feature: Site Manager administer groups Given I am on "Group 01" page Then I should see "2 datasets" in the "content" region + @customizable Scenario: View the correct count of datasets Given I am on "Groups" page Then I should see "11 datasets" @@ -92,7 +94,8 @@ Feature: Site Manager administer groups Then I should see "Displaying 1 - 10 of 11 datasets" Scenario: View the list of group members - Given I am on "Group 01" page + Given I am logged in as "Gabriel" + And I am on "Group 01" page When I click "Members" in the "group block" region Then I should see "Gabriel" in the "group members" region And I should see "Katie" in the "group members" region diff --git a/dkan/test/features/pod.feature b/dkan/test/features/pod.feature index 59646bf92..2ff1d5c43 100644 --- a/dkan/test/features/pod.feature +++ b/dkan/test/features/pod.feature @@ -5,23 +5,23 @@ Feature: Project Open Data + Open Data Federal Extras As a dataset creator I want to create datasets with POD fields and publish them with data.json - @pod_json_valid @api @noworkflow + @pod_01 @pod_json_valid @api @noworkflow Scenario: Data.json should be valid Given I am on the homepage Then I "should" see a valid data.json - @pod_json_odfe @api @noworkflow @add_ODFE @remove_ODFE + @pod_02 @pod_json_odfe @api @noworkflow @add_ODFE @remove_ODFE Scenario: Data.json validation should fail if ODFE enabled Given I am on the homepage Then I "should not" see a valid data.json - @api @noworkflow @add_ODFE @remove_ODFE + @pod_03 @api @noworkflow @add_ODFE @remove_ODFE Scenario: See Federal Extras fields on the Dataset form Given I am logged in as a user with the "editor" role When I visit "node/add/dataset" Then I should see all of the Federal Extras fields - @api @noworkflow @add_ODFE @remove_ODFE + @pod_04 @api @noworkflow @add_ODFE @remove_ODFE Scenario: See all POD required fields marked as required # Enable POD validation + Groups validation Given I "enable" the "Strict POD validation" on DKAN Dataset Forms @@ -40,7 +40,7 @@ Feature: Project Open Data + Open Data Federal Extras Then I "disable" the "Strict POD validation" on DKAN Dataset Forms Then I "disable" the "Groups validation" on DKAN Dataset Forms - @api @noworkflow @add_ODFE @remove_ODFE + @pod_05 @api @noworkflow @add_ODFE @remove_ODFE Scenario: See all POD required fields marked as required except for Groups # Enable POD validation only. Given I "enable" the "Strict POD validation" on DKAN Dataset Forms @@ -57,13 +57,13 @@ Feature: Project Open Data + Open Data Federal Extras # Cleanup configuration. Then I "disable" the "Strict POD validation" on DKAN Dataset Forms - @api @noworkflow @add_ODFE @remove_ODFE + @pod_06 @api @noworkflow @add_ODFE @remove_ODFE Scenario: See all license values if POD validation is not enabled Given I am logged in as a user with the "content creator" role When I visit "node/add/dataset" Then I should see "all" license values - @api @noworkflow @add_ODFE @remove_ODFE + @pod_07 @api @noworkflow @add_ODFE @remove_ODFE Scenario: See only POD valid licenses if POD validation is enabled # Enable POD validation only. Given I "enable" the "Strict POD validation" on DKAN Dataset Forms @@ -74,8 +74,8 @@ Feature: Project Open Data + Open Data Federal Extras #Cleanup configuration Given I "disable" the "Strict POD validation" on DKAN Dataset Forms - @api - Scenario: DEBUG Site Manager role should have access to the validation page + @pod_08 @api + Scenario: Site Manager role should have access to the validation page Given pages: | name | url | | POD Validation | /admin/config/services/odsm/validate/pod | diff --git a/dkan/test/features/user.admin.feature b/dkan/test/features/user.admin.feature index 8fae3df6a..73e9ec064 100644 --- a/dkan/test/features/user.admin.feature +++ b/dkan/test/features/user.admin.feature @@ -71,6 +71,7 @@ Feature: User And I press "Create new account" Then I should see "Created a new user account for tempuser." When I am on "Users" page + And I wait for "tempuser" Then I should see "editor" in the "tempuser" row Scenario: Block user @@ -87,6 +88,7 @@ Feature: User Scenario: Disable user Given I am logged in as "John" And I am on "Users" page + And I wait for "edit" When I click "edit" in the "Katie" row And I press "Cancel account" And I select the radio button "Disable the account and keep its content." diff --git a/dkan/test/features/user.all.feature b/dkan/test/features/user.all.feature index e94fb564b..ba0e953fe 100644 --- a/dkan/test/features/user.all.feature +++ b/dkan/test/features/user.all.feature @@ -38,7 +38,7 @@ Feature: User | Dataset 01 | Group 01 | Katie | Yes | world | Test | | Dataset 02 | Group 01 | Katie | Yes | world | Test | - @login + @user_all_01 @login Scenario: Login Given I am on the homepage When I follow "Log in" @@ -47,14 +47,14 @@ Feature: User And I press "Log in" Then I should see the "John" user page - @login + @user_all_02 @login Scenario: Logout Given I am logged in as "John" And I am on the homepage When I follow "Log out" Then I should see "Log in" - @javascript @deleteTempUsers @customizable + @user_all_03 @javascript @deleteTempUsers @customizable Scenario: Register Given I am on the homepage When I follow "Register" @@ -67,7 +67,7 @@ Feature: User Then I should see "Thank you for applying for an account." And I should see "Your account is currently pending approval by the site administrator." - @mail @login + @user_all_04 @mail @login Scenario: Request new password Given I am on the homepage When I follow "Log in" @@ -77,16 +77,17 @@ Feature: User Then user "John" should receive an email #TODO: Follow reset password link on email? + @user_all_05 Scenario: View user profile - Given I am on "Group 01" page - And I follow "Members" - When I click "Katie" - Then I should see the "Katie" user page + Given I am on the "Katie" page + Then I should see "Katie's content" + @user_all_06 Scenario: View list of published datasets created by user on user profile Given I am on "Katie" page Then I should see "2" items in the "user content" region + @user_all_07 Scenario: Search datasets created by user on user profile Given I am on "Katie" page When I fill in "Test" for "Search" in the "content search" region @@ -94,6 +95,7 @@ Feature: User Then I should see "2 results" in the "user content" region And I should see "2" items in the "user content" region + @user_all_08 Scenario: See list of user memberships on user profile Given I am logged in as "Katie" And I am on "Katie" page diff --git a/dkan/test/features/workflow.feature b/dkan/test/features/workflow.feature index 6b877d233..50cad8f69 100644 --- a/dkan/test/features/workflow.feature +++ b/dkan/test/features/workflow.feature @@ -35,7 +35,7 @@ Feature: #Non workbench roles can see the menu item My Workflow. However #they can't access to the page. - @globalUser + @workflow_01 @globalUser Scenario Outline: As a user without a Workbench role, I should not be able to access My Workbench or the My Workbench tabs Given I am logged in as a user with the "" role Then I should not see the link "My Workbench" @@ -54,7 +54,7 @@ Feature: | editor | | site manager | - @ok @globalUser + @workflow_02 @ok @globalUser Scenario Outline: As a user with any Workflow role, I should be able to access My Workbench. Given I am logged in as a user with the "" role When I am on "My Workbench" page @@ -69,7 +69,7 @@ Feature: | Workflow Moderator | | Workflow Supervisor | - @api @javascript @globalUser + @workflow_03 @api @javascript @globalUser Scenario Outline: As a user with any Workflow role, I should be able to upgrade my own draft content to needs review. Given I am logged in as "" And datasets: @@ -121,7 +121,7 @@ Feature: # | Workflow Moderator, editor | # | Workflow Supervisor, site manager | - @ok @javascript @globalUser + @workflow_04 @ok @javascript @globalUser Scenario: As a user with the Workflow Supervisor role, I should be able to publish stale 'Needs Review' content. Given I am logged in as "Contributor" And datasets: @@ -147,7 +147,7 @@ Feature: When I press "Publish" Then I wait for "Performed Publish on 3 items" - @ok @globalUser + @workflow_05 @ok @globalUser Scenario Outline: As a user with Workflow Roles, I should not be able to see draft contents I did not author in 'My Drafts' Given I am logged in as a user with the "" role Given users: @@ -169,7 +169,7 @@ Feature: | Workflow Moderator, editor | | Workflow Supervisor, site manager | - @ok @globalUser + @workflow_06 @ok @globalUser Scenario Outline: As a user with Workflow Roles, I should be able to see draft content I authored in 'My Drafts' Given I am logged in as "" And datasets: @@ -187,7 +187,7 @@ Feature: | Moderator | | Supervisor | - @ok @globalUser + @workflow_07 @ok @globalUser Scenario Outline: As a user with Workflow Roles, I should not be able to see Published content I authored in workbench pages Given I am logged in as "Contributor" And datasets: @@ -213,7 +213,7 @@ Feature: | My Drafts | Workflow Supervisor, site manager | | Needs Review | Workflow Supervisor, site manager | - @ok @globalUser + @workflow_08 @ok @globalUser Scenario Outline: As a user with Workflow Roles, I should not be able to see Needs Review resources I authored in 'My Drafts' Given I am logged in as a user with the "" role And datasets: @@ -233,7 +233,7 @@ Feature: | Workflow Moderator, editor | | Workflow Supervisor, site manager | - @ok @globalUser + @workflow_09 @ok @globalUser Scenario: As a user with the Workflow Contributor, I should be able to see Needs Review contents I authored in 'Needs Review' Given I am logged in as "Contributor" And datasets: @@ -248,7 +248,7 @@ Feature: And I should see "My Resource" And I should see "My Dataset" - @ok @globalUser + @workflow_10 @ok @globalUser Scenario: As a user with the Workflow Contributor, I should not be able to see Needs Review contents I did not author in 'Needs Review' Given I am logged in as a user with the "Workflow Contributor" role Given users: @@ -267,7 +267,7 @@ Feature: Then I should not see "Not My Resource" Then I should not see "Not My Dataset" - @ok @globalUser + @workflow_11 @ok @globalUser Scenario: As a Workflow Moderator, I should be able to see Needs Review datasets I did not author, but which belongs to my Group, in 'Needs Review' Given users: | name | roles | @@ -283,7 +283,7 @@ Feature: And I am on "Needs Review" page Then I should see the text "Not My Dataset" - @ok @globalUser + @workflow_21 @ok @globalUser Scenario: As a Workflow Moderator, I should be able to see Stale Needs Review datasets I did not author, but which belongs to my Group, in 'Needs Review' Given users: | name | roles | @@ -299,7 +299,7 @@ Feature: And I am on "Stale Reviews" page Then I should see the text "Not My Dataset" - @ok @globalUser + @workflow_12 @ok @globalUser Scenario: As a Workflow Moderator, I should not be able to see Needs Review datasets I did not author, and which do not belong to my Group, in 'Needs Review' Given users: | name | roles | @@ -317,7 +317,7 @@ Feature: And I am on "Needs Review" page Then I should not see the text "Not My Dataset" - @ok @globalUser + @workflow_13 @ok @globalUser Scenario: As a Workflow Supervisor, I should be able to see Needs Review content I did not author, regardless whether it belongs to my group or not, in 'Needs Review' Given users: | name | roles | @@ -338,7 +338,7 @@ Feature: Then I should see the text "Still Not My Dataset" Then I should see the text "Not My Dataset" - @ok @globalUser + @workflow_14 @ok @globalUser Scenario: As a Workflow Supervisor I should be able to see content in the 'Needs Review' state I did not author, regardless whether it belongs to my group or not, but which were submitted greater than 72 hours before now, in the 'Stale Reviews' Given users: | name | roles | @@ -359,7 +359,7 @@ Feature: Then I should see the text "Still Not My Dataset" Then I should see the text "Not My Dataset" - @ok @globalUser + @workflow_15 @ok @globalUser Scenario: As a Workflow Supervisor I should be able to see content in the 'Draft' state I did not author, regardless whether it belongs to my group or not, but which were submitted greater than 72 hours before now, in the 'Stale Drafts' Given users: | name | roles | @@ -380,7 +380,7 @@ Feature: Then I should see the text "Still Not My Dataset" Then I should see the text "Not My Dataset" - @api @ahoyRunMe @javascript @globalUser + @workflow_16 @api @ahoyRunMe @javascript @globalUser Scenario: When administering users, role pairings with core roles should be enforced Given I am logged in as a user with the "administrator" role And I visit the "Create User" page @@ -400,7 +400,7 @@ Feature: And I click "Edit" Then the checkbox "content creator" should be checked - @api @globalUser + @workflow_17 @api @globalUser Scenario: Modify user workflow roles as site manager Given users: | name | roles | mail | @@ -420,7 +420,7 @@ Feature: When I am on "Users" page Then I should see "Workflow Contributor" in the "content-creator" row - @api @ahoyRunMe @javascript @globalUser + @workflow_18 @api @ahoyRunMe @javascript @globalUser Scenario: Role pairings should also work for site managers. Given users: | name | roles | @@ -444,7 +444,7 @@ Feature: And I click "Edit" Then the checkbox "editor" should be checked - @api @javascript @harvest + @workflow_20 @api @javascript @harvest Scenario: Check harvested datasets are published by default even when dkan_workflow is enabled. Given users: | name | mail | status | roles | @@ -452,7 +452,7 @@ Feature: And The "source_one" source is harvested And the content "Gold Prices in London 1950-2008 (Monthly) Harvest" should be "published" - @ok + @workflow_19 @ok # https://jira.govdelivery.com/browse/CIVIC-5348 Scenario: "View draft" should display the draft dataset and not the published revision. Given users: