feat: Add sync for remote DDB to local instance (#27)

README.md

@@ -238,6 +238,29 @@ the slave table(s).
 See the heading below entitled "Authentication and Authorization to AWS DynamoDB API" for
 more information related to credentials.
 
+### Syncing Tables to a Local DynamoDB Instance
+
+When developing locally using the DynamoDB emulator or Docker image, it can be useful to pull
+data from upstream environments.
+
+See the [AWS documentation][local-setup] on how to set up the emulator or Docker image for
+local development.
+
+To synchronize from a remote table to a local table, set the `--slave-endpoint` parameter
+to the URL of the local endpoint, and set the rest of the parameters as you normally
+would. Note that slave credential parameters will be ignored when using the
+`--slave-endpoint` param. For example:
+
+```bash
+node src/cli.js \
+   --master us-east-1:my-dynamodb-table \
+   --slave-endpoint http://localhost:8000 \
+   --slave us-west-2:my-dynamodb-table \
+   --write-missing \
+   --write-differing
+```
+
+Note: the region does not matter on the `--slave`; feel free to use any valid region.
 
 ### "Dry Run" Mode
 
@@ -354,3 +377,4 @@ details.
 [contributing]: https://github.com/silvermine/silvermine-info#contributing
 [envvars]: http://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
 [credsfile]: http://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html
+[local-setup]: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html

src/Synchronizer.js

@@ -41,7 +41,9 @@ module.exports = Class.extend({
       this._master = _.extend({}, master, { id: (master.region + ':' + master.name), docs: this._makeDocClient(master) });
 
       this._slaves = _.map(slaves, function(def) {
-         return _.extend({}, def, { id: (def.region + ':' + def.name), docs: this._makeDocClient(def, opts.slaveCredentials) });
+         var client = this._makeDocClient(def, opts.slaveCredentials, opts.slaveEndpoint);
+
+         return _.extend({}, def, { id: (def.region + ':' + def.name), docs: client });
       }.bind(this));
 
       this._abortScanning = false;
@@ -507,7 +509,8 @@ module.exports = Class.extend({
    _compareTableDescriptions: function() {
       var def = Q.defer(),
           describeMaster = this._describeTable(this._master),
-          describeSlaves = Q.all(_.map(this._slaves, _.partial(this._describeTable.bind(this), _, this._opts.slaveCredentials)));
+          slaveDescribeFn = _.partial(this._describeTable.bind(this), _, this._opts.slaveCredentials, this._opts.slaveEndpoint),
+          describeSlaves = Q.all(_.map(this._slaves, slaveDescribeFn));
 
       function logDescription(title, tableDef, tableDesc) {
          console.log('%s table %s', title, tableDef.id);
@@ -560,19 +563,26 @@ module.exports = Class.extend({
       return def.promise;
    },
 
-   _describeTable: function(tableDef, creds) {
-      var dyn = new AWS.DynamoDB({ region: tableDef.region, credentials: creds || AWS.config.credentials });
+   _describeTable: function(tableDef, creds, endpoint) {
+      var dyn;
+
+      dyn = new AWS.DynamoDB({
+         region: tableDef.region,
+         endpoint: endpoint,
+         credentials: (endpoint ? undefined : (creds || AWS.config.credentials)),
+      });
 
       return Q.ninvoke(dyn, 'describeTable', { TableName: tableDef.name })
          .then(function(resp) {
             return resp.Table;
          });
    },
 
-   _makeDocClient: function(def, creds) {
+   _makeDocClient: function(def, creds, endpoint) {
       return new AWS.DynamoDB.DocumentClient({
          region: def.region,
-         credentials: creds || AWS.config.credentials,
+         endpoint: endpoint,
+         credentials: (endpoint ? undefined : (creds || AWS.config.credentials)),
          maxRetries: this._opts.maxRetries,
          retryDelayOptions: {
             base: this._opts.retryDelayBase,

src/cli.js

@@ -19,6 +19,7 @@ argvOpts = {
       'role-arn',
       'mfa-serial',
       'mfa-token',
+      'slave-endpoint',
       'slave-profile',
       'slave-role-arn',
       'slave-mfa-serial',
@@ -160,6 +161,10 @@ if (!_.isEmpty(argv['slave-profile'])) {
    options.slaveCredentials = new AWS.SharedIniFileCredentials({ profile: argv['slave-profile'] });
 }
 
+if (!_.isEmpty(argv['slave-endpoint'])) {
+   options.slaveEndpoint = argv['slave-endpoint'];
+}
+
 options.slaveCredentials = setupRoleRelatedCredentials('slave-', 'for slaves', options.slaveCredentials || AWS.config.credentials);
 
 startupPromise