diff --git a/README.md b/README.md index 5488a7e..170d728 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,29 @@ the slave table(s). See the heading below entitled "Authentication and Authorization to AWS DynamoDB API" for more information related to credentials. +### Syncing Tables to a Local DynamoDB Instance + +When developing locally using the DynamoDB emulator or Docker image, it can be useful to pull +data from upstream environments. + +See the [AWS documentation][local-setup] on how to set up the emulator or Docker image for +local development. + +To synchronize from a remote table to a local table, set the `--slave-endpoint` parameter +to the URL of the local endpoint, and set the rest of the parameters as you normally +would. Note that slave credential parameters will be ignored when using the +`--slave-endpoint` param. For example: + +```bash +node src/cli.js \ + --master us-east-1:my-dynamodb-table \ + --slave-endpoint http://localhost:8000 \ + --slave us-west-2:my-dynamodb-table \ + --write-missing \ + --write-differing +``` + +Note: the region does not matter on the `--slave`; feel free to use any valid region. ### "Dry Run" Mode @@ -354,3 +377,4 @@ details. [contributing]: https://github.com/silvermine/silvermine-info#contributing [envvars]: http://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html [credsfile]: http://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html +[local-setup]: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html diff --git a/src/Synchronizer.js b/src/Synchronizer.js index 4d33241..b196837 100644 --- a/src/Synchronizer.js +++ b/src/Synchronizer.js @@ -41,7 +41,9 @@ module.exports = Class.extend({ this._master = _.extend({}, master, { id: (master.region + ':' + master.name), docs: this._makeDocClient(master) }); this._slaves = _.map(slaves, function(def) { - return _.extend({}, def, { id: (def.region + ':' + def.name), docs: this._makeDocClient(def, opts.slaveCredentials) }); + var client = this._makeDocClient(def, opts.slaveCredentials, opts.slaveEndpoint); + + return _.extend({}, def, { id: (def.region + ':' + def.name), docs: client }); }.bind(this)); this._abortScanning = false; @@ -507,7 +509,8 @@ module.exports = Class.extend({ _compareTableDescriptions: function() { var def = Q.defer(), describeMaster = this._describeTable(this._master), - describeSlaves = Q.all(_.map(this._slaves, _.partial(this._describeTable.bind(this), _, this._opts.slaveCredentials))); + slaveDescribeFn = _.partial(this._describeTable.bind(this), _, this._opts.slaveCredentials, this._opts.slaveEndpoint), + describeSlaves = Q.all(_.map(this._slaves, slaveDescribeFn)); function logDescription(title, tableDef, tableDesc) { console.log('%s table %s', title, tableDef.id); @@ -560,8 +563,14 @@ module.exports = Class.extend({ return def.promise; }, - _describeTable: function(tableDef, creds) { - var dyn = new AWS.DynamoDB({ region: tableDef.region, credentials: creds || AWS.config.credentials }); + _describeTable: function(tableDef, creds, endpoint) { + var dyn; + + dyn = new AWS.DynamoDB({ + region: tableDef.region, + endpoint: endpoint, + credentials: (endpoint ? undefined : (creds || AWS.config.credentials)), + }); return Q.ninvoke(dyn, 'describeTable', { TableName: tableDef.name }) .then(function(resp) { @@ -569,10 +578,11 @@ module.exports = Class.extend({ }); }, - _makeDocClient: function(def, creds) { + _makeDocClient: function(def, creds, endpoint) { return new AWS.DynamoDB.DocumentClient({ region: def.region, - credentials: creds || AWS.config.credentials, + endpoint: endpoint, + credentials: (endpoint ? undefined : (creds || AWS.config.credentials)), maxRetries: this._opts.maxRetries, retryDelayOptions: { base: this._opts.retryDelayBase, diff --git a/src/cli.js b/src/cli.js index ff0bc87..bf81cf5 100755 --- a/src/cli.js +++ b/src/cli.js @@ -19,6 +19,7 @@ argvOpts = { 'role-arn', 'mfa-serial', 'mfa-token', + 'slave-endpoint', 'slave-profile', 'slave-role-arn', 'slave-mfa-serial', @@ -160,6 +161,10 @@ if (!_.isEmpty(argv['slave-profile'])) { options.slaveCredentials = new AWS.SharedIniFileCredentials({ profile: argv['slave-profile'] }); } +if (!_.isEmpty(argv['slave-endpoint'])) { + options.slaveEndpoint = argv['slave-endpoint']; +} + options.slaveCredentials = setupRoleRelatedCredentials('slave-', 'for slaves', options.slaveCredentials || AWS.config.credentials); startupPromise