Remake dkan_starter/dkan 1.13.5.2 (#384)

* Add new config.yml param stage_file_proxy_files:

REF CIVIC-6610
defaul/stage_file_porxy_files: holds a list of files that we may want to move
from production site into the testing environment in order to ensure tests
pass.

* remake dkan with patch. 1.13.5 + 2018

* Use ahoy dkan server.

assets/templates/circle.yml.erb

@@ -45,7 +45,7 @@ dependencies:
     - ahoy site up --db-user=ubuntu --db-pass='' --db-host=127.0.0.1 --db-port=3306 --db-name=circle_test
     - ahoy utils fail-when-bad-disable
     # Run a webserver using drush.
-    - 'ahoy drush --yes runserver :8888':
+    - 'ahoy dkan server':
         background: true
 
     - wget http://selenium-release.storage.googleapis.com/2.53/selenium-server-standalone-2.53.1.jar

build-dkan.make

@@ -10,4 +10,4 @@ projects[dkan][download][type] = git
 projects[dkan][download][url] = https://github.com/NuCivic/dkan.git
 projects[dkan][download][tag] = 7.x-1.13.5
 
-projects[dkan][patch][] = https://patch-diff.githubusercontent.com/raw/NuCivic/dkan/pull/2012.diff
+projects[dkan][patch][] = https://patch-diff.githubusercontent.com/raw/NuCivic/dkan/pull/2018.diff

circle.yml

@@ -45,7 +45,7 @@ dependencies:
     - ahoy site up --db-user=ubuntu --db-pass='' --db-host=127.0.0.1 --db-port=3306 --db-name=circle_test
     - ahoy utils fail-when-bad-disable
     # Run a webserver using drush.
-    - 'ahoy drush --yes runserver :8888':
+    - 'ahoy dkan server':
         background: true
 
     - wget http://selenium-release.storage.googleapis.com/2.53/selenium-server-standalone-2.53.1.jar

config/config.php

@@ -80,6 +80,9 @@
     array (
       'enable' => false,
     ),
+    'stage_file_proxy_files' => 
+    array (
+    ),
     'stage_file_proxy_origin' => 'changeme',
   ),
   'gaClientTrackingCode' => 'UA-XXXXX-Y',

config/config.yml

@@ -45,6 +45,7 @@ default:
   https_securepages: false
   odfe:
     enable: false
+  stage_file_proxy_files: []
   stage_file_proxy_origin: changeme
 gaClientTrackingCode: UA-XXXXX-Y
 gaNuCivicTrackingCode: UA-XXXXX-Z

dkan/.ahoy/.scripts/.ht.router.php

@@ -0,0 +1,42 @@
+<?php
+
+/**
+ * @file
+ * Routing-script for the built-in PHP web server.
+ *
+ * The built-in webserver should only be used for development and testing as it
+ * has a number of limitations that makes running Drupal on it highly insecure
+ * and somewhat limited.
+ *
+ * In particular be aware that:
+ *   - The server is single-threaded, any requests made during the execution of
+ *     the main request will hang until the main request has been completed.
+ *   - The webserver does not enforce any of the settings in .htaccess in
+ *     particular a remote user will be able to download files that normally
+ *     would be protected from direct access such as .module files.
+ *
+ * Usage:
+ * php -S localhost:8888 .ht.router.php
+ *
+ * @see http://php.net/manual/en/features.commandline.webserver.php
+ */
+
+$url = parse_url($_SERVER['REQUEST_URI']);
+if (file_exists('.' . $url['path'])) {
+  // Serve the requested resource as-is.
+  return FALSE;
+}
+// The use of a router-script means that a number of $_SERVER variables has to
+// be updated to point to the index-file.
+$index_file_absolute = $_SERVER['DOCUMENT_ROOT'] . DIRECTORY_SEPARATOR . 'index.php';
+$index_file_relative = DIRECTORY_SEPARATOR . 'index.php';
+// SCRIPT_FILENAME will point to the router-script itself, it should point to
+// the full path to index.php.
+$_SERVER['SCRIPT_FILENAME'] = $index_file_absolute;
+// SCRIPT_NAME and PHP_SELF will either point to /index.php or contain the full
+// virtual path being requested depending on the url being requested. They
+// should always point to index.php relative to document root.
+$_SERVER['SCRIPT_NAME'] = $index_file_relative;
+$_SERVER['PHP_SELF'] = $index_file_relative;
+// Require the main index-file and let core take over.
+require $index_file_absolute;

dkan/.ahoy/.scripts/config.rb

@@ -1,11 +1,28 @@
 require 'yaml'
 
+# Deep merge only available in ruby >= 3.0
+# TODO: replace defaults with custom deep_merge
+
 begin
   CONFIG = YAML.load_file("config/config.yml")
 rescue Exception => msg
   puts "Loading of Configuration errored out with: #{msg}."
   puts "Using default CONFIG instead."
-  CONFIG = {"circle" => {"skip_features" => []}}
+  CONFIG = {}
+end
+
+if not CONFIG.has_key? "circle"
+  CONFIG["circle"] = {}
 end
 
+if not CONFIG["circle"].has_key? "skip_features"
+  CONFIG["circle"]["skip_features"] = []
+end
 
+if not CONFIG.has_key? "default"
+  CONFIG["default"] = {}
+end
+
+if not CONFIG["default"].has_key? "https_everywhere"
+  CONFIG["default"]["https_everywhere"] = false
+end

dkan/.ahoy/.scripts/dkan-uli.rb

@@ -0,0 +1,16 @@
+require "./dkan/.ahoy/.scripts/config.rb"
+
+if CONFIG["default"]["https_everywhere"]
+  url="surl"
+else
+  url="url"
+end
+
+url=`ahoy drush --uri="$(ahoy docker #{url})" uli`
+os =`uname`
+
+if os =~ /Darwin/
+  `open #{url}`
+else
+  puts url
+end

dkan/.ahoy/.scripts/drupal-rebuild.sh

@@ -12,8 +12,13 @@ fi
 
 drush --root=docroot  make --concurrency=$concurrency --prepare-install dkan/drupal-org-core.make docroot --yes
 
-drush --root=docroot -y --verbose $root si minimal --sites-subdir=default --account-pass='admin' --db-url=$1 install_configure_form.update_status_module='array(false,false)' &&
+drush --root=docroot -y --verbose si minimal --sites-subdir=default --account-pass='admin' --db-url=$1 install_configure_form.update_status_module='array(false,false)' &&
   ln -s ../../dkan docroot/profiles/dkan
 chmod +w docroot/sites/default/settings.php
-printf "// DKAN Datastore Fast Import options.\n\$databases['default']['default']['pdo'] = array(\n  PDO::MYSQL_ATTR_LOCAL_INFILE => 1,\n  PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => 1,\n);" >> docroot/sites/default/settings.php
+
+if [ "$AHOY_CMD_PROXY" = "DOCKER" ]; then
+  printf "// Docker Database Settings\n\$databases['default']['default'] = array(\n  'database' => 'drupal',\n  'username' => 'drupal',\n  'password' => '123',\n  'host' => 'db',\n  'port' => '',\n  'driver' => 'mysql',\n  'prefix' => '',\n);\n" >> docroot/sites/default/settings.php
+fi
+printf "// DKAN Datastore Fast Import options.\n\$databases['default']['default']['pdo'] = array(\n  PDO::MYSQL_ATTR_LOCAL_INFILE => 1,\n  PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => 1,\n);\n" >> docroot/sites/default/settings.php
+
 chmod -w docroot/sites/default/settings.php

dkan/.ahoy/.scripts/server.sh

@@ -0,0 +1,12 @@
+
+if [ "$HOSTNAME" = "cli" ]; then
+  HOST=cli
+else
+  HOST=localhost
+fi
+
+cp ./dkan/.ahoy/.scripts/.ht.router.php ./docroot/
+cd ./docroot
+
+echo $HOST
+php -S $HOST:8888 .ht.router.php

dkan/.ahoy/dkan.ahoy.yml

@@ -4,11 +4,14 @@ commands:
   drupal-rebuild:
     usage: 'Builds a drupal site in ./docroot. Requires database url.'
     cmd: |
-      if [ "$AHOY_CMD_PROXY" != "DOCKER" ]; then
-        ARGS="{{args}}"
-      else
+      if [ "$AHOY_CMD_PROXY" = "DOCKER" ]; then
         ARGS=`ahoy docker mysql-url`
       fi
+
+      # If no mysql URL from docker, use passed in args.
+      if [ "$ARGS" = "" ];then
+        ARGS="{{args}}"
+      fi
       if [[ "$ARGS" != mysql* ]]; then
           echo "You need to specify the drupal db url to use in the form mysql://root:root@localhost/db as an argument."
           exit 1
@@ -44,9 +47,11 @@ commands:
         mkdir backups
       fi
       if [ -f backups/last_install.sql ] && ahoy confirm "An existing installation backup exists at backups/last_install.sql, do you want to use that instead of reinstalling from scratch?"; then
-        ahoy drush sql-drop -y
-        ahoy drush cc all
-        ahoy drush sql-cli <  backups/last_install.sql
+
+        ahoy drush sql-drop -y && \
+        echo "... Removed tables, restoring DB"
+        ahoy dkan sqlc < backups/last_install.sql
+
         echo "Installed dkan from backup"
       else
         if [ "$AHOY_CMD_PROXY" == "DOCKER" ]; then
@@ -193,15 +198,13 @@ commands:
     import: dkan/.ahoy/theme.ahoy.yml
     hide: true
 
+  # Moved to .ahoy.yml
   uli:
-    usage: log into the
+    usage: Log in via drush
     cmd: |
-      uli=$(ahoy drush uli {{args}} | sed 's/^http.*\/default\/\(.*$\)/\1/g')
-      url="$(ahoy docker url)/$uli"
-      os=$(uname)
+      ruby dkan/.ahoy/.scripts/dkan-uli.rb {{args}}
 
-      if [ "$os" = "Darwin" ]; then
-        open $url
-      else
-        echo $url
-      fi
+  server:
+    usage: Provided as an easy way to setup the php server during testing.
+    cmd:
+      ahoy cmd-proxy bash dkan/.ahoy/.scripts/server.sh

dkan/.ahoy/docker-compose.yml

@@ -5,12 +5,17 @@
 web:
   hostname: web
   image: nuams/drupal-apache-php:1.0-5.6
+  # Fix for docker for mac container not starting selenium
+  # https://forums.docker.com/t/space-in-new-no-proxy-env-var-breaking-old-containers/14075
+
   environment:
     - VIRTUAL_HOST=dkan.docker
+    - no_proxy=localhost
     #- VIRTUAL_PROTO=https
   ports:
     - "80"
     - "443"
+    - "8888"
   volumes:
     # PHP configuration overrides
     - "./.docker/etc/php5/php.ini:/etc/php5/fpm/conf.d/z_php.ini"
@@ -49,6 +54,7 @@ cli:
   environment:
     - XDEBUG_CONFIG=idekey=cli
     - PHP_IDE_CONFIG=serverName=dkan.docker
+    - VIRTUAL_HOST=dkan.docker
   volumes:
     # PHP configuration overrides
     - "./.docker/etc/php5/php-cli.ini:/etc/php5/cli/conf.d/z_php.ini"
@@ -64,6 +70,8 @@ cli:
     - browser
     - memcached
     - solr
+  ports:
+    - "8888"
 
 # Memcached node
 # Uncomment the service definition section below and the link in the web service above to start using memcached.
@@ -96,16 +104,21 @@ browser:
     - /dev/shm:/dev/shm
     # Project root folder mapping
     - *project_root
+
   links:
     - web
   ports:
-  #  - "4444:4444"
     - "5900"
 
+  # Fix for docker for mac container not starting selenium
+  # https://forums.docker.com/t/space-in-new-no-proxy-env-var-breaking-old-containers/14075
+  environment:
+    - no_proxy=localhost
+
 solr:
   hostname: solr
   image: devinci/drupal-solr:3.x
-  #ports:
-  #  - "8983:8983"
+  ports:
+    - "8983"
   volumes:
     - "./.docker/etc/solr/3.x:/var/lib/solr/conf"