From 75930537460f16ac06296e4ab65b537997891169 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 13 Jan 2023 10:16:43 -0500 Subject: [PATCH 1/9] fix: make it work like both function or object --- src/CyclicS3FSPromises.js | 25 ++++++++++++++++++++----- src/index.js | 19 +++++++++++++++++-- src/promises.js | 17 +++++++++-------- src/sync_interface.js | 2 ++ 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/CyclicS3FSPromises.js b/src/CyclicS3FSPromises.js index b3ad367..93537e0 100644 --- a/src/CyclicS3FSPromises.js +++ b/src/CyclicS3FSPromises.js @@ -22,11 +22,26 @@ function streamToBuffer(stream) { }); } -class CyclicS3FSPromises{ - constructor(bucketName, config={}) { - this.bucket = bucketName - this.config = config - this.s3 = new S3Client({...config}); +class CyclicS3FSPromises extends Function{ + constructor(bucket, config={}) { + super('...args', 'return this._bound._call(...args)') + this._bound = this.bind(this) + if(process.env.CYCLIC_BUCKET_NAME){ + this._bound.bucket = process.env.CYCLIC_BUCKET_NAME + } + if(bucket){ + this._bound.bucket = bucket + } + this._bound.s3 = new S3Client({...config}); + + return this._bound + } + + _call(bucketName, config) { + let client = new CyclicS3FSPromises(bucketName, config) + // client.s3 = new S3Client({...config}) + // client.bucket = bucketName + return client } async readFile(fileName ,options){ diff --git a/src/index.js b/src/index.js index 5971fd7..219c770 100644 --- a/src/index.js +++ b/src/index.js @@ -21,7 +21,19 @@ function makeCallback(cb) { class CyclicS3FS extends CyclicS3FSPromises { constructor(bucketName, config={}) { - super(bucketName, config={}) + super() + if(process.env.CYCLIC_BUCKET_NAME){ + this.bucket = process.env.CYCLIC_BUCKET_NAME + } + this.config = config + if(bucketName){ + this.bucket = bucketName + } + } + + _call(bucketName, config) { + let client = new CyclicS3FS(bucketName, config) + return client } @@ -194,4 +206,7 @@ const client = function(bucketName, config={}){ return new CyclicS3FS(bucketName, config) } -module.exports = client + + + +module.exports = new CyclicS3FS() diff --git a/src/promises.js b/src/promises.js index 2a0b3e6..ff27b8c 100644 --- a/src/promises.js +++ b/src/promises.js @@ -1,11 +1,12 @@ const fs = require('fs/promises') const CyclicS3FSPromises = require('./CyclicS3FSPromises') -const client = function(bucketName, config={}){ - if(!process.env.AWS_SECRET_ACCESS_KEY){ - console.warn('[s3fs] WARNING: AWS credentials are not set. Using local file system') - return fs - } - return new CyclicS3FSPromises(bucketName, config) -} +// const client = function(bucketName, config={}){ +// if(!process.env.AWS_SECRET_ACCESS_KEY){ +// console.warn('[s3fs] WARNING: AWS credentials are not set. Using local file system') +// return fs +// } +// return new CyclicS3FSPromises(bucketName, config) +// } -module.exports = client +// module.exports = client +module.exports = new CyclicS3FSPromises() diff --git a/src/sync_interface.js b/src/sync_interface.js index 2da9252..c22d045 100644 --- a/src/sync_interface.js +++ b/src/sync_interface.js @@ -48,6 +48,7 @@ module.exports = { runSync, } const run = async function(bucket, config, method, args){ + const fs = new CyclicS3FSPromises(bucket, config) let result = await fs[method](...args) if(typeof result !== 'undefined'){ @@ -62,6 +63,7 @@ if (require.main === module) { let _argv = process.argv.slice(2,) let bucket = _argv[0] + // console.error(process.argv) let config = JSON.parse(_argv[1]) let method = _argv[2] var buf = ''; From 3cd1c4f951d7d9b3f50270f68610f45dac5ca4f7 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 13 Jan 2023 10:17:16 -0500 Subject: [PATCH 2/9] fix: make it work like both function or object --- src/sync_interface.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sync_interface.js b/src/sync_interface.js index c22d045..c566256 100644 --- a/src/sync_interface.js +++ b/src/sync_interface.js @@ -63,7 +63,6 @@ if (require.main === module) { let _argv = process.argv.slice(2,) let bucket = _argv[0] - // console.error(process.argv) let config = JSON.parse(_argv[1]) let method = _argv[2] var buf = ''; From 8f21ad4e174ed332a3c3e6a44f5b2bd80f5e6ad9 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 13 Jan 2023 10:18:04 -0500 Subject: [PATCH 3/9] fix: make it work like both function or object --- src/CyclicS3FSPromises.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CyclicS3FSPromises.js b/src/CyclicS3FSPromises.js index 93537e0..5711065 100644 --- a/src/CyclicS3FSPromises.js +++ b/src/CyclicS3FSPromises.js @@ -39,8 +39,6 @@ class CyclicS3FSPromises extends Function{ _call(bucketName, config) { let client = new CyclicS3FSPromises(bucketName, config) - // client.s3 = new S3Client({...config}) - // client.bucket = bucketName return client } From dde0130a773f278ff2990fff6620e9ae95361aa6 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 13 Jan 2023 10:27:20 -0500 Subject: [PATCH 4/9] fix: make it work like both function or object --- test/index.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/index.test.js b/test/index.test.js index 677d4bd..118d00b 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -116,7 +116,7 @@ describe("Basic smoke tests", () => { test("readFileSync(jpeg)", async () => { const d = require("fs").readFileSync('test/_read.jpeg') - const fs = new s3fs(BUCKET) + const fs = s3fs(BUCKET) const _d = fs.readFileSync('test/_read.jpeg') expect(Buffer.compare(d, _d)).toEqual(0) @@ -127,7 +127,7 @@ describe("Basic smoke tests", () => { [Date.now()]: Date.now(), }) - const fs = new s3fs(BUCKET) + const fs = s3fs(BUCKET) fs.writeFileSync('test/_write.json', content) let x = fs.readFileSync('test/_write.json') @@ -137,7 +137,7 @@ describe("Basic smoke tests", () => { test("writeFileSync(big_text)", async () => { const big_text = require("fs").readFileSync('test/_read_big.txt') - const fs = new s3fs(BUCKET) + const fs = s3fs(BUCKET) fs.writeFileSync('test/_write_big.txt', big_text) let x = fs.readFileSync('test/_write_big.txt') @@ -147,7 +147,7 @@ describe("Basic smoke tests", () => { test("writeFileSync(jpeg)", async () => { const jpeg = require("fs").readFileSync('test/_read.jpeg') - const fs = new s3fs(BUCKET) + const fs = s3fs(BUCKET) fs.writeFileSync('test/_write.jpeg', jpeg) let jpeg_s3 = fs.readFileSync('test/_write.jpeg') From 3ab0f58d31c0517ed397dacfca9809eabe6d236a Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 13 Jan 2023 10:32:04 -0500 Subject: [PATCH 5/9] fix: make it work like both function or object --- test/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/index.test.js b/test/index.test.js index 118d00b..d5edca4 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -7,7 +7,7 @@ const { } = require("@aws-sdk/client-s3"); const s3 = new S3Client({}); -const s3fs = require("../src") +const s3fs = require("../src") const s3fs_promises = require("../src/promises") afterAll(async () => { From eb1abb27370a3c56874b8599ddf2516eef6dfe79 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 18 Jan 2023 09:24:05 -0500 Subject: [PATCH 6/9] fix: fallback to fs --- src/index.js | 23 ++++++++++++++++------- src/promises.js | 28 +++++++++++++++++++--------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/index.js b/src/index.js index 219c770..49a5421 100644 --- a/src/index.js +++ b/src/index.js @@ -198,15 +198,24 @@ class CyclicS3FS extends CyclicS3FSPromises { } -const client = function(bucketName, config={}){ - if(!process.env.AWS_SECRET_ACCESS_KEY){ - console.warn('[s3fs] WARNING: AWS credentials are not set. Using local file system') - return fs +var client = new CyclicS3FS() +class FSFallback extends Function{ + constructor() { + super('...args', 'return this._bound._call(...args)') + this._bound = this.bind(this) + Object.assign(this._bound,{...fs}) + return this._bound + } + _call() { + let client = new FSFallback() + return client } - return new CyclicS3FS(bucketName, config) } +if(!process.env.AWS_SECRET_ACCESS_KEY){ + console.warn('[s3fs] WARNING: AWS credentials are not set. Using local file system') + client = new FSFallback() +} - -module.exports = new CyclicS3FS() +module.exports = client diff --git a/src/promises.js b/src/promises.js index ff27b8c..6581c40 100644 --- a/src/promises.js +++ b/src/promises.js @@ -1,12 +1,22 @@ const fs = require('fs/promises') const CyclicS3FSPromises = require('./CyclicS3FSPromises') -// const client = function(bucketName, config={}){ -// if(!process.env.AWS_SECRET_ACCESS_KEY){ -// console.warn('[s3fs] WARNING: AWS credentials are not set. Using local file system') -// return fs -// } -// return new CyclicS3FSPromises(bucketName, config) -// } +var client = new CyclicS3FSPromises() +class FSPromisesFallback extends Function{ + constructor() { + super('...args', 'return this._bound._call(...args)') + this._bound = this.bind(this) + Object.assign(this._bound,{...fs}) + return this._bound + } + _call() { + let client = new FSPromisesFallback() + return client + } +} -// module.exports = client -module.exports = new CyclicS3FSPromises() +if(!process.env.AWS_SECRET_ACCESS_KEY){ + console.warn('[s3fs] WARNING: AWS credentials are not set. Using local file system') + client = new FSPromisesFallback() +} + +module.exports = client From bbc435b188e1a30c39f52aae6e6e59494595e7a9 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 18 Jan 2023 09:34:14 -0500 Subject: [PATCH 7/9] fix: test --- test/index.test.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/index.test.js b/test/index.test.js index d5edca4..9e9550b 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -509,6 +509,20 @@ describe("Basic smoke tests", () => { }) + + test("object mode / function mode", async () => { + let as_object = Object.getOwnPropertyNames(Object.getPrototypeOf(require('../src'))) + let as_function = Object.getOwnPropertyNames(Object.getPrototypeOf(require('../src')(BUCKET))) + expect(as_function).toEqual(as_object) + }) + + test("object mode / function mode - promises", async () => { + let as_object = Object.getOwnPropertyNames(Object.getPrototypeOf(require('../src/promises'))) + let as_function = Object.getOwnPropertyNames(Object.getPrototypeOf(require('../src/promises')(BUCKET))) + expect(as_function).toEqual(as_object) + }) + + test("empty_bucket", async () => { const fs = s3fs(BUCKET) let dir_name = `/nested/dir_${Date.now()}` @@ -522,6 +536,8 @@ describe("Basic smoke tests", () => { }) + + }) From 45b5e10ae40a7cc435a45163324d047f997e13da Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 18 Jan 2023 09:40:54 -0500 Subject: [PATCH 8/9] fix: readme --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index b81e63d..5f4cee4 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,15 @@ Require in the same format as Node.js `fs`, specifying an S3 Bucket: const fs = require('@cyclic.sh/s3fs/promises')(S3_BUCKET_NAME) ``` +> Alternatively, if the enviornment variable `CYCLIC_BUCKET_NAME` is set to an S3 bucket name, initialization can happen without specifying a bucket: +> ```js +> const fs = require('@cyclic.sh/s3fs') +> ``` +> or +> ```js +> const fs = require('@cyclic.sh/s3fs/promises') +> ``` + ### Authentication Authenticating the client: From 2468c17cf40fd3a2cc944bff41452c79fe9a9294 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 18 Jan 2023 09:43:11 -0500 Subject: [PATCH 9/9] fix: readme --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5f4cee4..c0236e4 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,15 @@ Require in the same format as Node.js `fs`, specifying an S3 Bucket: const fs = require('@cyclic.sh/s3fs/promises')(S3_BUCKET_NAME) ``` -> Alternatively, if the enviornment variable `CYCLIC_BUCKET_NAME` is set to an S3 bucket name, initialization can happen without specifying a bucket: -> ```js -> const fs = require('@cyclic.sh/s3fs') -> ``` -> or -> ```js -> const fs = require('@cyclic.sh/s3fs/promises') -> ``` +- On cyclic.sh + - Alternatively, when using with cyclic.sh or if the environment variable `CYCLIC_BUCKET_NAME` is set to an S3 bucket name, initialization can happen without specifying a bucket: + ```js + const fs = require('@cyclic.sh/s3fs') + ``` + or + ```js + const fs = require('@cyclic.sh/s3fs/promises') + ``` ### Authentication