diff --git a/README.md b/README.md
index 997a8e97e..92abbe150 100644
--- a/README.md
+++ b/README.md
@@ -59,20 +59,23 @@ To run the API with your custom config, specify the location of the config file
The API recognizes the following properties under the top-level `api` key in your `pelias.json` config file:
-|parameter|required|default|description|
-|---|---|---|---|
-|`services`|*no*||Service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal), [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. For a description of when different Pelias services are recommended or required, see our [services documentation](https://github.com/pelias/documentation/blob/master/services.md).|
-|`defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat`|no | |default coordinates for focus point
-|`targets.auto_discover`|no|true|Should `sources` and `layers` be automatically discovered by querying Elasticsearch at process startup. (See more info in the [Custom sources and layers](#custom-sources-and-layers) section below).
-|`targets.auto_discover_required`|no|false|If set to `true`, type mapping discovery failures will result in a fatal error. This means a valid connection to a working Elasticsearch cluster with a Pelias index is _required_ on startup. Setting this to true can help prevent issues such as incorrect configuration in production environments
-|`targets.layers_by_source`
`targets.source_aliases`
`targets.layer_aliases`|no | |custom values for which `sources` and `layers` the API accepts (See more info in the [Custom sources and layers](#custom-sources-and-layers) section below). We recommend using the `targets.auto_discover:true` configuration instead of setting these manually.
-|`customBoosts` | no | `{}` | Allows configuring boosts for specific sources and layers, in order to influence result order. See [Configurable Boosts](#custom-boosts) below for details |
-|`autocomplete.exclude_address_length` | no | 0 | As a performance optimization, this optional filter excludes results from the 'address' layer for any queries where the character length of the 'subject' portion of the parsed_text is equal to or less than this many characters in length. Addresses are usually the bulk of the records in Elasticsearch, and searching across all of them for very short text inputs can be slow, with little benefit. Consider setting this to 1 or 2 if you have several million addresses in Pelias. |
-|`indexName`|*no*|*pelias*|name of the Elasticsearch index to be used when building queries|
-|`attributionURL`|no| (autodetected)|The full URL to use for the attribution link returned in all Pelias responses. Pelias will attempt to autodetect this host, but it will often be incorrect if, for example, there is a proxy between Pelias and its users. This parameter allows setting a specific URL to avoid any such issues|
-|`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.|
-|`relativeScores`|*no*|true|if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable
-|`exposeInternalDebugTools`|*no*|true|Exposes several debugging tools, such as the ability to enable Elasticsearch explain mode, that may come at a performance cost or expose sensitive infrastructure details. Not recommended if the Pelias API is open to the public.
+| parameter | required | default | description |
+|---------------------------------------------------------------------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `services` | *no* || Service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal), [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. For a description of when different Pelias services are recommended or required, see our [services documentation](https://github.com/pelias/documentation/blob/master/services.md). | |
+| `defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat` | no | | default coordinates for focus point |
+| `targets.auto_discover` | no | true | Should `sources` and `layers` be automatically discovered by querying Elasticsearch at process startup. (See more info in the [Custom sources and layers](#custom-sources-and-layers) section below). |
+| `targets.auto_discover_required` | no | false | If set to `true`, type mapping discovery failures will result in a fatal error. This means a valid connection to a working Elasticsearch cluster with a Pelias index is _required_ on startup. Setting this to true can help prevent issues such as incorrect configuration in production environments |
+| `targets.layers_by_source`
`targets.source_aliases`
`targets.layer_aliases` | no | | custom values for which `sources` and `layers` the API accepts (See more info in the [Custom sources and layers](#custom-sources-and-layers) section below). We recommend using the `targets.auto_discover:true` configuration instead of setting these manually. |
+| `customBoosts` | no | `{}` | Allows configuring boosts for specific sources and layers, in order to influence result order. See [Configurable Boosts](#custom-boosts) below for details |
+| `autocomplete.exclude_address_length` | no | 0 | As a performance optimization, this optional filter excludes results from the 'address' layer for any queries where the character length of the 'subject' portion of the parsed_text is equal to or less than this many characters in length. Addresses are usually the bulk of the records in Elasticsearch, and searching across all of them for very short text inputs can be slow, with little benefit. Consider setting this to 1 or 2 if you have several million addresses in Pelias. |
+| `indexName` | *no* | *pelias* | name of the Elasticsearch index to be used when building queries |
+| `attributionURL` | no | (autodetected) | The full URL to use for the attribution link returned in all Pelias responses. Pelias will attempt to autodetect this host, but it will often be incorrect if, for example, there is a proxy between Pelias and its users. This parameter allows setting a specific URL to avoid any such issues |
+| `accessLog` | *no* || name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely. | |
+| `relativeScores` | *no* | true | if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable |
+| `exposeInternalDebugTools` | *no* | true | Exposes several debugging tools, such as the ability to enable Elasticsearch explain mode, that may come at a performance cost or expose sensitive infrastructure details. Not recommended if the Pelias API is open to the public. |
+| `autocomplete.default_overrides` | *no* | `{}` | Allows overriding defaults for autocomplete queries (ex: to change the weights af a component) |
+| `search.default_overrides` | *no* | `{}` | Allows overriding defaults for search queries (ex: to change the weights af a component) |
+| `reverse.default_overrides` | *no* | `{}` | Allows overriding defaults for reverse queries (ex: to change the weights af a component) |
A good starting configuration file includes this section (fill in the service and Elasticsearch hosts as needed):
diff --git a/bin/generate-docs b/bin/generate-docs
index 5b337e3e2..a6e25b3ac 100755
--- a/bin/generate-docs
+++ b/bin/generate-docs
@@ -1,4 +1,5 @@
-#!/bin/bash -ex
+#!/usr/bin/env bash
+set -ex
rm -r docs || true
diff --git a/bin/start b/bin/start
index 84794d53c..1e17fc902 100755
--- a/bin/start
+++ b/bin/start
@@ -1,2 +1,2 @@
-#!/bin/bash
+#!/usr/bin/env bash
exec node index.js
diff --git a/bin/units b/bin/units
index 977e125b9..f5e2a6375 100755
--- a/bin/units
+++ b/bin/units
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
# run tests with pipefail to avoid false passes
# see https://github.com/pelias/pelias/issues/744
diff --git a/middleware/sendJSON.js b/middleware/sendJSON.js
index 32d5b0e2e..d364596b8 100644
--- a/middleware/sendJSON.js
+++ b/middleware/sendJSON.js
@@ -3,11 +3,16 @@ const es = require('elasticsearch');
const logger = require( 'pelias-logger' ).get( 'api' );
const PeliasParameterError = require('../sanitizer/PeliasParameterError');
const PeliasTimeoutError = require('../sanitizer/PeliasTimeoutError');
+const PeliasServiceError = require('../sanitizer/PeliasServiceError');
function isParameterError(error) {
return error instanceof PeliasParameterError;
}
+function isServiceError(error) {
+ return error instanceof PeliasServiceError;
+}
+
function isTimeoutError(error) {
return error instanceof PeliasTimeoutError ||
error instanceof es.errors.RequestTimeout;
@@ -36,7 +41,7 @@ function sendJSONResponse(req, res, next) {
const errorCodes = errors.map(function(error) {
if (isParameterError(error)) {
return 400;
- } else if (isTimeoutError(error) || isElasticsearchError(error)) {
+ } else if (isTimeoutError(error) || isElasticsearchError(error) || isServiceError(error)) {
return 502;
} else {
return 500;
diff --git a/query/autocomplete.js b/query/autocomplete.js
index 6e980a75c..dbc703a8c 100644
--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@@ -67,6 +67,8 @@ query.filter( views.focus_point_filter );
// --------------------------------
+const overrides = config.get('api.autocomplete.default_overrides');
+
/**
map request variables to query variables for all inputs
provided by this HTTP request.
@@ -75,6 +77,10 @@ function generateQuery( clean ){
const vs = new peliasQuery.Vars( defaults );
+ if (_.isObject(overrides)) {
+ vs.set(overrides);
+ }
+
// sources
if( _.isArray(clean.sources) && !_.isEmpty(clean.sources) ){
vs.var( 'sources', clean.sources );
diff --git a/query/reverse.js b/query/reverse.js
index 6f145680b..853ef1cc7 100644
--- a/query/reverse.js
+++ b/query/reverse.js
@@ -1,5 +1,6 @@
const _ = require('lodash');
const peliasQuery = require('pelias-query');
+const config = require('pelias-config').generate();
const defaults = require('./reverse_defaults');
//------------------------------
@@ -23,10 +24,16 @@ query.filter( peliasQuery.view.boundary_gid );
// --------------------------------
+const overrides = config.get('api.reverse.default_overrides');
+
function generateQuery( clean ){
const vs = new peliasQuery.Vars( defaults );
+ if (_.isObject(overrides)) {
+ vs.set(overrides);
+ }
+
// set size
if( clean.querySize ){
vs.var( 'size', clean.querySize);
diff --git a/query/search.js b/query/search.js
index fd1a048e7..154ce0a0b 100644
--- a/query/search.js
+++ b/query/search.js
@@ -1,6 +1,7 @@
const _ = require('lodash');
const peliasQuery = require('pelias-query');
const defaults = require('./search_defaults');
+const config = require('pelias-config').generate();
const textParser = require('./text_parser');
//------------------------------
@@ -24,6 +25,8 @@ fallbackQuery.filter( peliasQuery.view.categories );
fallbackQuery.filter( peliasQuery.view.boundary_gid );
// --------------------------------
+const overrides = config.get('api.search.default_overrides');
+
/**
map request variables to query variables for all inputs
provided by this HTTP request.
@@ -32,6 +35,9 @@ function generateQuery( clean ){
const vs = new peliasQuery.Vars( defaults );
+ if (_.isObject(overrides)) {
+ vs.set(overrides);
+ }
// input text
vs.var( 'input:name', clean.text );
diff --git a/sanitizer/PeliasServiceError.js b/sanitizer/PeliasServiceError.js
new file mode 100644
index 000000000..f85cb75d8
--- /dev/null
+++ b/sanitizer/PeliasServiceError.js
@@ -0,0 +1,15 @@
+class PeliasServiceError extends Error {
+ constructor(message = '') {
+ super(message);
+ }
+
+ toString() {
+ return this.message;
+ }
+
+ toJSON() {
+ return this.message;
+ }
+}
+
+module.exports = PeliasServiceError;
diff --git a/sanitizer/sanitizeAll.js b/sanitizer/sanitizeAll.js
index 50bff0146..b54cabbe2 100644
--- a/sanitizer/sanitizeAll.js
+++ b/sanitizer/sanitizeAll.js
@@ -1,9 +1,14 @@
const PeliasParameterError = require('./PeliasParameterError');
const PeliasTimeoutError = require('../sanitizer/PeliasTimeoutError');
+const PeliasServiceError = require('../sanitizer/PeliasServiceError');
function getCorrectErrorType(message) {
- if (message.includes( 'Timeout')) {
+ if (message.includes( 'Timeout') || message.includes('timeout')) {
return new PeliasTimeoutError(message);
+ } else if ( message.includes('parse response') ||
+ message.includes('ECONN') ||
+ message.includes('ENOTFOUND')) {
+ return new PeliasServiceError(message);
}
// most errors are parameter errors
@@ -15,7 +20,7 @@ function getCorrectErrorType(message) {
// on Error objects here
function ensureInstanceOfError(error) {
if (error instanceof Error) {
- // preserve the message and stack trace of existing Error objecs
+ // preserve the message and stack trace of existing Error objects
const newError = getCorrectErrorType(error.message);
newError.stack = error.stack;
return newError;
diff --git a/test/unit/helper/diffPlaces.js b/test/unit/helper/diffPlaces.js
index 3ad68b95a..f16734528 100644
--- a/test/unit/helper/diffPlaces.js
+++ b/test/unit/helper/diffPlaces.js
@@ -690,7 +690,7 @@ module.exports.tests.normalizeString = function (test, common) {
test('diacritics', function (t) {
t.equal(normalizeString('Malmö'), 'malmo');
- t.equal(normalizeString('Grolmanstraße'), 'grolmanstraße');
+ t.equal(normalizeString('Grolmanstraße'), 'grolmanstrasse');
t.equal(normalizeString('àáâãäåấắæầằçḉèéêëếḗềḕ'), 'aaaaaaaaaeaacceeeeeeee');
t.end();
});
diff --git a/test/unit/middleware/sendJSON.js b/test/unit/middleware/sendJSON.js
index 3e5c28836..d025b8a56 100644
--- a/test/unit/middleware/sendJSON.js
+++ b/test/unit/middleware/sendJSON.js
@@ -1,6 +1,7 @@
const es = require('elasticsearch');
const middleware = require('../../../middleware/sendJSON');
const PeliasTimeoutError = require('../../../sanitizer/PeliasTimeoutError');
+const PeliasServiceError = require('../../../sanitizer/PeliasServiceError');
module.exports.tests = {};
@@ -243,6 +244,24 @@ module.exports.tests.unknown_exception = function(test, common) {
});
};
+module.exports.tests.econnrefused = function(test, common) {
+ test('connection refused', function(t) {
+ var res = { body: { geocoding: {
+ errors: [ new PeliasServiceError('Some service error') ]
+ }}};
+
+ res.status = function( code ){
+ return { json: function( body ){
+ t.equal( code, 502, 'Bad Gateway' );
+ t.deepEqual( body, res.body, 'body set' );
+ t.end();
+ }};
+ };
+
+ middleware(null, res);
+ });
+};
+
module.exports.all = function (tape, common) {
function test(name, testFunction) {
diff --git a/test/unit/sanitizer/sanitizeAll.js b/test/unit/sanitizer/sanitizeAll.js
index 685043b75..c7e5c969e 100644
--- a/test/unit/sanitizer/sanitizeAll.js
+++ b/test/unit/sanitizer/sanitizeAll.js
@@ -1,6 +1,7 @@
var sanitizeAll = require('../../../sanitizer/sanitizeAll');
const PeliasParameterError = require('../../../sanitizer/PeliasParameterError');
const PeliasTimeoutError = require('../../../sanitizer/PeliasTimeoutError');
+const PeliasServiceError = require('../../../sanitizer/PeliasServiceError');
module.exports.tests = {};
@@ -154,6 +155,38 @@ module.exports.tests.all = function(test, common) {
t.end();
});
+ test('Various connection errors should be converted to a PeliasServiceError type', function(t) {
+ var req = {};
+ const error_messages = [
+ 'connect ECONNREFUSED 127.0.0.1:12345',
+ 'getaddrinfo ENOTFOUND foobar',
+ 'http://127.0.0.1:12345/parse?address=3400%20Kane%20Hill%20Rd could not parse response: foobar',
+ ];
+ var sanitizers = {
+ 'first': {
+ sanitize: function(){
+ req.clean.a = 'first sanitizer';
+ return {
+ errors: error_messages.map(msg => new Error(msg)),
+ warnings: []
+ };
+ }
+ }
+ };
+
+ var expected_req = {
+ clean: {
+ a: 'first sanitizer'
+ },
+ errors: error_messages.map(msg => new PeliasServiceError(msg)),
+ warnings: []
+ };
+
+ sanitizeAll.runAllChecks(req, sanitizers);
+ t.deepEquals(req, expected_req);
+ t.end();
+ });
+
test('req.query should be passed to individual sanitizers when available', function(t) {
var req = {
query: {