From 83aa8ebe60bc64918dcb5d7872907c97149371f5 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 16:15:16 +0530 Subject: [PATCH 01/18] initial code --- node/storage-cleaner/.gitignore | 130 +++++++++++++++ node/storage-cleaner/.prettierrc.json | 6 + node/storage-cleaner/README.md | 48 ++++++ node/storage-cleaner/package-lock.json | 210 +++++++++++++++++++++++++ node/storage-cleaner/package.json | 16 ++ node/storage-cleaner/src/appwrite.js | 62 ++++++++ node/storage-cleaner/src/main.js | 13 ++ node/storage-cleaner/src/util.js | 6 + 8 files changed, 491 insertions(+) create mode 100644 node/storage-cleaner/.gitignore create mode 100644 node/storage-cleaner/.prettierrc.json create mode 100644 node/storage-cleaner/README.md create mode 100644 node/storage-cleaner/package-lock.json create mode 100644 node/storage-cleaner/package.json create mode 100644 node/storage-cleaner/src/appwrite.js create mode 100644 node/storage-cleaner/src/main.js create mode 100644 node/storage-cleaner/src/util.js diff --git a/node/storage-cleaner/.gitignore b/node/storage-cleaner/.gitignore new file mode 100644 index 00000000..6a7d6d8e --- /dev/null +++ b/node/storage-cleaner/.gitignore @@ -0,0 +1,130 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* \ No newline at end of file diff --git a/node/storage-cleaner/.prettierrc.json b/node/storage-cleaner/.prettierrc.json new file mode 100644 index 00000000..0a725205 --- /dev/null +++ b/node/storage-cleaner/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": true, + "singleQuote": true +} diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md new file mode 100644 index 00000000..10c6c29f --- /dev/null +++ b/node/storage-cleaner/README.md @@ -0,0 +1,48 @@ +# โšก Node.js Starter Function + +A simple starter function. Edit `src/main.js` to get started and create something awesome! ๐Ÿš€ + +## ๐Ÿงฐ Usage + +### GET / + +- Returns a "Hello, World!" message. + +**Response** + +Sample `200` Response: + +```text +Hello, World! +``` + +### POST, PUT, PATCH, DELETE / + +- Returns a "Learn More" JSON response. + +**Response** + +Sample `200` Response: + +```json +{ + "motto": "Build Fast. Scale Big. All in One Place.", + "learn": "https://appwrite.io/docs", + "connect": "https://appwrite.io/discord", + "getInspired": "https://builtwith.appwrite.io" +} +``` + +## โš™๏ธ Configuration + +| Setting | Value | +|-------------------|---------------| +| Runtime | Node (18.0) | +| Entrypoint | `src/main.js` | +| Build Commands | `npm install` | +| Permissions | `any` | +| Timeout (Seconds) | 15 | + +## ๐Ÿ”’ Environment Variables + +No environment variables required. diff --git a/node/storage-cleaner/package-lock.json b/node/storage-cleaner/package-lock.json new file mode 100644 index 00000000..d4bc4a3b --- /dev/null +++ b/node/storage-cleaner/package-lock.json @@ -0,0 +1,210 @@ +{ + "name": "starter-template", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "starter-template", + "version": "1.0.0", + "dependencies": { + "node-appwrite": "^11.0.0" + }, + "devDependencies": { + "prettier": "^3.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-appwrite": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/node-appwrite/-/node-appwrite-11.0.0.tgz", + "integrity": "sha512-l+O0d1kCvY56NdlmsiWw91IlUzJszNW7FZodGCyvNlJeR4+JIpy9Mica3T4M+zvZGsbRqLhpghCysQmPBwwhQA==", + "dependencies": { + "axios": "^1.4.0", + "form-data": "^4.0.0" + } + }, + "node_modules/prettier": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", + "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + } + }, + "dependencies": { + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "requires": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, + "follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==" + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, + "node-appwrite": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/node-appwrite/-/node-appwrite-11.0.0.tgz", + "integrity": "sha512-l+O0d1kCvY56NdlmsiWw91IlUzJszNW7FZodGCyvNlJeR4+JIpy9Mica3T4M+zvZGsbRqLhpghCysQmPBwwhQA==", + "requires": { + "axios": "^1.4.0", + "form-data": "^4.0.0" + } + }, + "prettier": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", + "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", + "dev": true + }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + } + } +} diff --git a/node/storage-cleaner/package.json b/node/storage-cleaner/package.json new file mode 100644 index 00000000..2a58507d --- /dev/null +++ b/node/storage-cleaner/package.json @@ -0,0 +1,16 @@ +{ + "name": "starter-template", + "version": "1.0.0", + "description": "", + "main": "src/main.js", + "type": "module", + "scripts": { + "format": "prettier --write ." + }, + "dependencies": { + "node-appwrite": "^11.0.0" + }, + "devDependencies": { + "prettier": "^3.0.0" + } +} diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js new file mode 100644 index 00000000..100bbc5d --- /dev/null +++ b/node/storage-cleaner/src/appwrite.js @@ -0,0 +1,62 @@ +import { Client, Storage, Query } from 'node-appwrite'; +import { getExpiryDate } from './util.js'; + +class AppwriteService { + defaultQuery = [Query.orderAsc('$createdAt')]; + + constructor() { + const client = new Client() + .setEndpoint( + process.env.APPWRITE_ENDPOINT ?? 'https://cloud.appwrite.io/v1' + ) + .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID) + .setKey(process.env.APPWRITE_API_KEY); + this.storage = new Storage(client); + } + + async cleanAllBuckets() { + const buckets = await this.getBuckets(); + for (const bucket of buckets) { + await this.cleanBucket(bucket.$id); + } + } + + async getBuckets(queries = this.defaultQuery, cursor = null) { + const currentQueries = [...queries, Query.limit(100)]; + + if (cursor) { + currentQueries.push(Query.cursorAfter(cursor.$id)); + } + const response = await this.storage.listBuckets(queries); + if (response.buckets.length < 100) { + return response.buckets; + } + + const nextCursor = response.buckets[response.buckets.length - 1]; + return [ + ...response.buckets, + ...(await getBuckets(collectionId, queries, nextCursor)), + ]; + } + + async cleanBucket(bucketId) { + const queries = [Query.orderAsc('$createdAt'), Query.limit(100)]; + outerLoop: while (true) { + const fileList = await this.storage.listFiles(bucketId, queries); + const files = fileList.files; + if (files.length == 0) { + break outerLoop; + } + const expiryDate = getExpiryDate(); // get the expiryDate using the RETENTION_PERIOD + for (const file of files) { + if (new Date(file.$createdAt) < expiryDate) { + await this.storage.deleteFile(bucketId, file.$id); + } else { + break outerLoop; // this will break out of both the loops + } + } + } + } +} + +export default AppwriteService; diff --git a/node/storage-cleaner/src/main.js b/node/storage-cleaner/src/main.js new file mode 100644 index 00000000..1123e976 --- /dev/null +++ b/node/storage-cleaner/src/main.js @@ -0,0 +1,13 @@ +import AppwriteService from './appwrite.js'; + +export default async ({ req, res, log, error }) => { + const appwrite = new AppwriteService(); + + try { + await appwrite.cleanAllBuckets(); + } catch (error) { + error(err.message); + } + + return res.send('Buckets cleaned', 200); +}; diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js new file mode 100644 index 00000000..18245352 --- /dev/null +++ b/node/storage-cleaner/src/util.js @@ -0,0 +1,6 @@ +export function getExpiryDate() { + const retentionPeriod = process.env.RETENTION_PERIOD_DAYS ?? 30; + const expiryDate = new Date(); + expiryDate.setDate(expiryDate.getDate() - retentionPeriod); + return expiryDate; +} From e159a644dfc1ee965195d83275150e753386d93a Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 18:05:47 +0530 Subject: [PATCH 02/18] updated the README file --- node/storage-cleaner/README.md | 58 ++++++++++++++++------------------ 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md index 10c6c29f..db2add48 100644 --- a/node/storage-cleaner/README.md +++ b/node/storage-cleaner/README.md @@ -1,48 +1,46 @@ -# โšก Node.js Starter Function +# Node.js Storage Cleaner Function -A simple starter function. Edit `src/main.js` to get started and create something awesome! ๐Ÿš€ +Storage cleaner function to remove all files from the buckets older than X number of days. ## ๐Ÿงฐ Usage ### GET / -- Returns a "Hello, World!" message. +Remove the files older than X number of days from all buckets. **Response** -Sample `200` Response: +Sample `200` Response: Buckets cleaned -```text -Hello, World! -``` - -### POST, PUT, PATCH, DELETE / +## โš™๏ธ Configuration -- Returns a "Learn More" JSON response. +| Setting | Value | +| ----------------- | ---------------- | +| Runtime | Node (18.0) | +| Entrypoint | `src/main.js` | +| Build Commands | `npm run build` | +| Permissions | `any` | +| Events | `users.*.create` | +| CRON | `0 1 * * *` | +| Timeout (Seconds) | 15 | -**Response** +## ๐Ÿ”’ Environment Variables -Sample `200` Response: +### RETENTION_PERIOD_DAYS -```json -{ - "motto": "Build Fast. Scale Big. All in One Place.", - "learn": "https://appwrite.io/docs", - "connect": "https://appwrite.io/discord", - "getInspired": "https://builtwith.appwrite.io" -} -``` +The number of days you want to retain a file. -## โš™๏ธ Configuration +| Question | Answer | +| ------------ | ------ | +| Required | Yes | +| Sample Value | `1` | -| Setting | Value | -|-------------------|---------------| -| Runtime | Node (18.0) | -| Entrypoint | `src/main.js` | -| Build Commands | `npm install` | -| Permissions | `any` | -| Timeout (Seconds) | 15 | +### APPWRITE_API_KEY -## ๐Ÿ”’ Environment Variables +API Key to talk to Appwrite backend APIs. -No environment variables required. +| Question | Answer | +| ------------- | -------------------------------------------------------------------------------------------------- | +| Required | Yes | +| Sample Value | `d1efb...aec35` | +| Documentation | [Appwrite: Getting Started for Server](https://appwrite.io/docs/getting-started-for-server#apiKey) | From b59b8f8a992aed2dcabce8e06c9ad7b7b2898aee Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 18:23:44 +0530 Subject: [PATCH 03/18] update Readme + added throwIfMissing function --- node/storage-cleaner/README.md | 26 +++++++++++++++++--------- node/storage-cleaner/src/main.js | 6 ++++++ node/storage-cleaner/src/util.js | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md index db2add48..88682626 100644 --- a/node/storage-cleaner/README.md +++ b/node/storage-cleaner/README.md @@ -14,15 +14,14 @@ Sample `200` Response: Buckets cleaned ## โš™๏ธ Configuration -| Setting | Value | -| ----------------- | ---------------- | -| Runtime | Node (18.0) | -| Entrypoint | `src/main.js` | -| Build Commands | `npm run build` | -| Permissions | `any` | -| Events | `users.*.create` | -| CRON | `0 1 * * *` | -| Timeout (Seconds) | 15 | +| Setting | Value | +| ----------------- | ------------- | +| Runtime | Node (18.0) | +| Entrypoint | `src/main.js` | +| Build Commands | `npm install` | +| Permissions | `any` | +| CRON | `0 1 * * *` | +| Timeout (Seconds) | 15 | ## ๐Ÿ”’ Environment Variables @@ -44,3 +43,12 @@ API Key to talk to Appwrite backend APIs. | Required | Yes | | Sample Value | `d1efb...aec35` | | Documentation | [Appwrite: Getting Started for Server](https://appwrite.io/docs/getting-started-for-server#apiKey) | + +### APPWRITE_ENDPOINT + +The URL endpoint of the Appwrite server. If not provided, it defaults to the Appwrite Cloud server: `https://cloud.appwrite.io/v1`. + +| Question | Answer | +| ------------ | ------------------------------ | +| Required | No | +| Sample Value | `https://cloud.appwrite.io/v1` | diff --git a/node/storage-cleaner/src/main.js b/node/storage-cleaner/src/main.js index 1123e976..11255652 100644 --- a/node/storage-cleaner/src/main.js +++ b/node/storage-cleaner/src/main.js @@ -1,6 +1,12 @@ import AppwriteService from './appwrite.js'; +import { throwIfMissing } from './util.js'; export default async ({ req, res, log, error }) => { + throwIfMissing(process.env, [ + 'APPWRITE_FUNCTION_PROJECT_ID', + 'APPWRITE_API_KEY', + ]); + const appwrite = new AppwriteService(); try { diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index 18245352..a71dbeb7 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -4,3 +4,21 @@ export function getExpiryDate() { expiryDate.setDate(expiryDate.getDate() - retentionPeriod); return expiryDate; } + +/** + * Throws an error if any of the keys are missing from the object + * @param {*} obj + * @param {string[]} keys + * @throws {Error} + */ +export function throwIfMissing(obj, keys) { + const missing = []; + for (let key of keys) { + if (!(key in obj) || !obj[key]) { + missing.push(key); + } + } + if (missing.length > 0) { + throw new Error(`Missing required fields: ${missing.join(', ')}`); + } +} From 31d353ba64cab581a0a423a34c1849973398d7e0 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 18:36:07 +0530 Subject: [PATCH 04/18] added comments for methods --- node/storage-cleaner/src/appwrite.js | 25 +++++++++++++++++++++++++ node/storage-cleaner/src/util.js | 4 ++++ 2 files changed, 29 insertions(+) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index 100bbc5d..81010352 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -14,6 +14,15 @@ class AppwriteService { this.storage = new Storage(client); } + /** + * Clean up all storage buckets by removing files older than a specified retention period. + * + * This function retrieves a list of all storage buckets, and for each bucket, + * it calls the cleanBucket function to remove files that are older than the calculated expiry date + * based on the retention period. + * + * @returns {Promise} A Promise that resolves when all buckets are cleaned. + */ async cleanAllBuckets() { const buckets = await this.getBuckets(); for (const bucket of buckets) { @@ -21,6 +30,13 @@ class AppwriteService { } } + /** + * Retrieves the list of buckets. + * + * @param {Query[]} queries - An array of queries to filter and sort the results. Defaults to this.defaultQuery. + * @param {Cursor|null} cursor - A cursor for pagination. Pass null to start from the beginning. + * @returns {Promise} An array of buckets. + */ async getBuckets(queries = this.defaultQuery, cursor = null) { const currentQueries = [...queries, Query.limit(100)]; @@ -39,6 +55,15 @@ class AppwriteService { ]; } + /** + * Clean up files from the storage bucket by removing files older than a specified retention period. + * + * This function retrieves files from the specified bucket, ordered by creation date in ascending order, + * and deletes files that are older than the calculated expiry date based on the retention period. + * + * @param {string} bucketId - The ID of the storage bucket to clean. + * @returns {Promise} A Promise that resolves when the bucket is cleaned. + */ async cleanBucket(bucketId) { const queries = [Query.orderAsc('$createdAt'), Query.limit(100)]; outerLoop: while (true) { diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index a71dbeb7..df6a3b76 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -1,3 +1,7 @@ +/** + * Calculate and return the expiry date based on the retention period. + * @returns {Date} The calculated expiry date. + */ export function getExpiryDate() { const retentionPeriod = process.env.RETENTION_PERIOD_DAYS ?? 30; const expiryDate = new Date(); From 710fc145b3130e967d16b0338a5ab31baf46142c Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 22:48:32 +0530 Subject: [PATCH 05/18] update getBuckets code --- node/storage-cleaner/src/appwrite.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index 81010352..f7ca913c 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -43,16 +43,13 @@ class AppwriteService { if (cursor) { currentQueries.push(Query.cursorAfter(cursor.$id)); } - const response = await this.storage.listBuckets(queries); + const response = await this.storage.listBuckets(currentQueries); if (response.buckets.length < 100) { return response.buckets; } const nextCursor = response.buckets[response.buckets.length - 1]; - return [ - ...response.buckets, - ...(await getBuckets(collectionId, queries, nextCursor)), - ]; + return [...response.buckets, ...(await getBuckets(queries, nextCursor))]; } /** From deb098526c29113e6532234939f29623887e7095 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 22:50:43 +0530 Subject: [PATCH 06/18] err to error --- node/storage-cleaner/src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/storage-cleaner/src/main.js b/node/storage-cleaner/src/main.js index 11255652..ce3d7577 100644 --- a/node/storage-cleaner/src/main.js +++ b/node/storage-cleaner/src/main.js @@ -12,7 +12,7 @@ export default async ({ req, res, log, error }) => { try { await appwrite.cleanAllBuckets(); } catch (error) { - error(err.message); + error(error.message); } return res.send('Buckets cleaned', 200); From 017c36166e4b1d12cc4aac717d67a3e93e171a02 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 22:52:18 +0530 Subject: [PATCH 07/18] use exception instead of error --- node/storage-cleaner/src/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/storage-cleaner/src/main.js b/node/storage-cleaner/src/main.js index ce3d7577..d3bef00a 100644 --- a/node/storage-cleaner/src/main.js +++ b/node/storage-cleaner/src/main.js @@ -11,8 +11,8 @@ export default async ({ req, res, log, error }) => { try { await appwrite.cleanAllBuckets(); - } catch (error) { - error(error.message); + } catch (exception) { + error(exception.message); } return res.send('Buckets cleaned', 200); From 786f1defe885188e3de1b13985f7d3476065c98a Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 22:53:31 +0530 Subject: [PATCH 08/18] add this.getBuckets --- node/storage-cleaner/src/appwrite.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index f7ca913c..216dc957 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -49,7 +49,10 @@ class AppwriteService { } const nextCursor = response.buckets[response.buckets.length - 1]; - return [...response.buckets, ...(await getBuckets(queries, nextCursor))]; + return [ + ...response.buckets, + ...(await this.getBuckets(queries, nextCursor)), + ]; } /** From 9cbf00dd140d4b98090daac9232b1448cf71fbee Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 23:02:58 +0530 Subject: [PATCH 09/18] applying batching in the cleanBucket --- node/storage-cleaner/src/appwrite.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index 216dc957..f3c3e6fe 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -66,6 +66,7 @@ class AppwriteService { */ async cleanBucket(bucketId) { const queries = [Query.orderAsc('$createdAt'), Query.limit(100)]; + outerLoop: while (true) { const fileList = await this.storage.listFiles(bucketId, queries); const files = fileList.files; @@ -73,12 +74,26 @@ class AppwriteService { break outerLoop; } const expiryDate = getExpiryDate(); // get the expiryDate using the RETENTION_PERIOD + + const batchPromises = []; // will use batchPromises to send multiple delete request simultaneously + const batchSize = 50; + for (const file of files) { if (new Date(file.$createdAt) < expiryDate) { - await this.storage.deleteFile(bucketId, file.$id); + batchPromises.push(this.storage.deleteFile(bucketId, file.$id)); } else { break outerLoop; // this will break out of both the loops } + + if (batchPromises.length >= batchSize) { + await Promise.all(batchPromises); + batchPromises.length = 0; + } + } + + // Delete any remaining files in the batch. + if (batchPromises.length > 0) { + await Promise.all(batchPromises); } } } From f5ed1a567eba97f8f650b60bfd9f8f4ad94dfdee Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 23:07:29 +0530 Subject: [PATCH 10/18] modify the batching code --- node/storage-cleaner/src/appwrite.js | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index f3c3e6fe..255c8042 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -66,7 +66,7 @@ class AppwriteService { */ async cleanBucket(bucketId) { const queries = [Query.orderAsc('$createdAt'), Query.limit(100)]; - + const batchPromises = []; // will use batchPromises to send multiple delete request simultaneously outerLoop: while (true) { const fileList = await this.storage.listFiles(bucketId, queries); const files = fileList.files; @@ -75,27 +75,17 @@ class AppwriteService { } const expiryDate = getExpiryDate(); // get the expiryDate using the RETENTION_PERIOD - const batchPromises = []; // will use batchPromises to send multiple delete request simultaneously - const batchSize = 50; - for (const file of files) { if (new Date(file.$createdAt) < expiryDate) { batchPromises.push(this.storage.deleteFile(bucketId, file.$id)); } else { break outerLoop; // this will break out of both the loops } - - if (batchPromises.length >= batchSize) { - await Promise.all(batchPromises); - batchPromises.length = 0; - } - } - - // Delete any remaining files in the batch. - if (batchPromises.length > 0) { - await Promise.all(batchPromises); } } + if (batchPromises.length > 0) { + await Promise.all(batchPromises); + } } } From e9f046f627095fb9bd1387228959654799d02bc7 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Mon, 16 Oct 2023 23:50:57 +0530 Subject: [PATCH 11/18] update cleanBucket code --- node/storage-cleaner/README.md | 2 +- node/storage-cleaner/src/appwrite.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md index 88682626..283dbce1 100644 --- a/node/storage-cleaner/README.md +++ b/node/storage-cleaner/README.md @@ -21,7 +21,7 @@ Sample `200` Response: Buckets cleaned | Build Commands | `npm install` | | Permissions | `any` | | CRON | `0 1 * * *` | -| Timeout (Seconds) | 15 | +| Timeout (Seconds) | 900 | ## ๐Ÿ”’ Environment Variables diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index 255c8042..bc0cc898 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -81,8 +81,14 @@ class AppwriteService { } else { break outerLoop; // this will break out of both the loops } + if (batchPromises.length > 0) { + await Promise.all(batchPromises); + batchPromises.length = 0; + } } } + // remove the remaining files if any there in batchPromises + // this is for the case when we get out of loop using outerloop if (batchPromises.length > 0) { await Promise.all(batchPromises); } From 3c849164ecf21649136234c7f74e3a819580fa3b Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Thu, 19 Oct 2023 17:27:05 +0530 Subject: [PATCH 12/18] applied requested changes --- node/storage-cleaner/README.md | 11 +++- node/storage-cleaner/package.json | 2 +- node/storage-cleaner/src/appwrite.js | 84 +++++----------------------- node/storage-cleaner/src/main.js | 9 +-- node/storage-cleaner/src/util.js | 2 +- 5 files changed, 29 insertions(+), 79 deletions(-) diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md index 283dbce1..1b4febf1 100644 --- a/node/storage-cleaner/README.md +++ b/node/storage-cleaner/README.md @@ -1,4 +1,4 @@ -# Node.js Storage Cleaner Function +# ๐Ÿงน Node.js Storage Cleaner Function Storage cleaner function to remove all files from the buckets older than X number of days. @@ -34,6 +34,15 @@ The number of days you want to retain a file. | Required | Yes | | Sample Value | `1` | +### APPWRITE_BUCKET_ID + +The ID of the bucket from which the files are to be deleted. + +| Question | Answer | +| ------------ | -------------- | +| Required | Yes | +| Sample Value | `652d...b4daf` | + ### APPWRITE_API_KEY API Key to talk to Appwrite backend APIs. diff --git a/node/storage-cleaner/package.json b/node/storage-cleaner/package.json index 2a58507d..aba07a36 100644 --- a/node/storage-cleaner/package.json +++ b/node/storage-cleaner/package.json @@ -1,5 +1,5 @@ { - "name": "starter-template", + "name": "storage-cleaner", "version": "1.0.0", "description": "", "main": "src/main.js", diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index bc0cc898..cae35f0c 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -2,8 +2,6 @@ import { Client, Storage, Query } from 'node-appwrite'; import { getExpiryDate } from './util.js'; class AppwriteService { - defaultQuery = [Query.orderAsc('$createdAt')]; - constructor() { const client = new Client() .setEndpoint( @@ -14,47 +12,6 @@ class AppwriteService { this.storage = new Storage(client); } - /** - * Clean up all storage buckets by removing files older than a specified retention period. - * - * This function retrieves a list of all storage buckets, and for each bucket, - * it calls the cleanBucket function to remove files that are older than the calculated expiry date - * based on the retention period. - * - * @returns {Promise} A Promise that resolves when all buckets are cleaned. - */ - async cleanAllBuckets() { - const buckets = await this.getBuckets(); - for (const bucket of buckets) { - await this.cleanBucket(bucket.$id); - } - } - - /** - * Retrieves the list of buckets. - * - * @param {Query[]} queries - An array of queries to filter and sort the results. Defaults to this.defaultQuery. - * @param {Cursor|null} cursor - A cursor for pagination. Pass null to start from the beginning. - * @returns {Promise} An array of buckets. - */ - async getBuckets(queries = this.defaultQuery, cursor = null) { - const currentQueries = [...queries, Query.limit(100)]; - - if (cursor) { - currentQueries.push(Query.cursorAfter(cursor.$id)); - } - const response = await this.storage.listBuckets(currentQueries); - if (response.buckets.length < 100) { - return response.buckets; - } - - const nextCursor = response.buckets[response.buckets.length - 1]; - return [ - ...response.buckets, - ...(await this.getBuckets(queries, nextCursor)), - ]; - } - /** * Clean up files from the storage bucket by removing files older than a specified retention period. * @@ -65,33 +22,20 @@ class AppwriteService { * @returns {Promise} A Promise that resolves when the bucket is cleaned. */ async cleanBucket(bucketId) { - const queries = [Query.orderAsc('$createdAt'), Query.limit(100)]; - const batchPromises = []; // will use batchPromises to send multiple delete request simultaneously - outerLoop: while (true) { - const fileList = await this.storage.listFiles(bucketId, queries); - const files = fileList.files; - if (files.length == 0) { - break outerLoop; - } - const expiryDate = getExpiryDate(); // get the expiryDate using the RETENTION_PERIOD - - for (const file of files) { - if (new Date(file.$createdAt) < expiryDate) { - batchPromises.push(this.storage.deleteFile(bucketId, file.$id)); - } else { - break outerLoop; // this will break out of both the loops - } - if (batchPromises.length > 0) { - await Promise.all(batchPromises); - batchPromises.length = 0; - } - } - } - // remove the remaining files if any there in batchPromises - // this is for the case when we get out of loop using outerloop - if (batchPromises.length > 0) { - await Promise.all(batchPromises); - } + let hasNextPage = true; + const queries = [ + Query.smallerThan('$createdAt', getExpiryDate()), + Query.limit(100), + ]; + do { + const response = await this.storage.listFiles(bucketId, queries); + await Promise.all( + response.files.map((file) => + this.storage.deleteFile(bucketId, file.$id) + ) + ); + hasNextPage = response.files.length > 0; + } while (hasNextPage); } } diff --git a/node/storage-cleaner/src/main.js b/node/storage-cleaner/src/main.js index d3bef00a..2b940084 100644 --- a/node/storage-cleaner/src/main.js +++ b/node/storage-cleaner/src/main.js @@ -3,17 +3,14 @@ import { throwIfMissing } from './util.js'; export default async ({ req, res, log, error }) => { throwIfMissing(process.env, [ - 'APPWRITE_FUNCTION_PROJECT_ID', 'APPWRITE_API_KEY', + 'RETENTION_PERIOD_DAYS', + 'APPWRITE_BUCKET_ID', ]); const appwrite = new AppwriteService(); - try { - await appwrite.cleanAllBuckets(); - } catch (exception) { - error(exception.message); - } + await appwrite.cleanBucket(process.env.APPWRITE_BUCKET_ID); return res.send('Buckets cleaned', 200); }; diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index df6a3b76..470c7cfa 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -3,7 +3,7 @@ * @returns {Date} The calculated expiry date. */ export function getExpiryDate() { - const retentionPeriod = process.env.RETENTION_PERIOD_DAYS ?? 30; + const retentionPeriod = +(process.env.RETENTION_PERIOD_DAYS ?? 30); const expiryDate = new Date(); expiryDate.setDate(expiryDate.getDate() - retentionPeriod); return expiryDate; From a0cf8f03b72c1381b17ec838cd7ab3ac22c56f5b Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Thu, 19 Oct 2023 18:28:25 +0530 Subject: [PATCH 13/18] update throwIfmissing --- node/storage-cleaner/src/appwrite.js | 5 +---- node/storage-cleaner/src/util.js | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index cae35f0c..9c1d29eb 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -15,16 +15,13 @@ class AppwriteService { /** * Clean up files from the storage bucket by removing files older than a specified retention period. * - * This function retrieves files from the specified bucket, ordered by creation date in ascending order, - * and deletes files that are older than the calculated expiry date based on the retention period. - * * @param {string} bucketId - The ID of the storage bucket to clean. * @returns {Promise} A Promise that resolves when the bucket is cleaned. */ async cleanBucket(bucketId) { let hasNextPage = true; const queries = [ - Query.smallerThan('$createdAt', getExpiryDate()), + Query.lessThan('$createdAt', getExpiryDate()), Query.limit(100), ]; do { diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index 470c7cfa..333dd9e8 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -18,7 +18,8 @@ export function getExpiryDate() { export function throwIfMissing(obj, keys) { const missing = []; for (let key of keys) { - if (!(key in obj) || !obj[key]) { + // using obj[key] != 0 for case when the value of key will be 0 + if (!(key in obj) || (!obj[key] && obj[key] != 0)) { missing.push(key); } } From 7c80522e36c0d257c7fac3cd1fbc08091a5e5c36 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Thu, 19 Oct 2023 18:47:03 +0530 Subject: [PATCH 14/18] return ISO string instead of date --- node/storage-cleaner/src/util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index 333dd9e8..36edc715 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -1,12 +1,12 @@ /** * Calculate and return the expiry date based on the retention period. - * @returns {Date} The calculated expiry date. + * @returns {string} The calculated expiry date in ISO format */ export function getExpiryDate() { const retentionPeriod = +(process.env.RETENTION_PERIOD_DAYS ?? 30); const expiryDate = new Date(); expiryDate.setDate(expiryDate.getDate() - retentionPeriod); - return expiryDate; + return expiryDate.toISOString(); } /** From 28309d62dfb94ad21f5917c302e2339aa698d9d0 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Thu, 19 Oct 2023 19:14:00 +0530 Subject: [PATCH 15/18] update readme --- node/storage-cleaner/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md index 1b4febf1..c8b998d5 100644 --- a/node/storage-cleaner/README.md +++ b/node/storage-cleaner/README.md @@ -1,12 +1,12 @@ # ๐Ÿงน Node.js Storage Cleaner Function -Storage cleaner function to remove all files from the buckets older than X number of days. +Storage cleaner function to remove all files older than X number of days from the specified bucket. ## ๐Ÿงฐ Usage ### GET / -Remove the files older than X number of days from all buckets. +Remove files older than X days from the specified bucket **Response** From 1618675b378ccdfe2a0b30fc0e7096793ca23dc9 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Thu, 19 Oct 2023 21:14:15 +0530 Subject: [PATCH 16/18] limited the query count from 100 to 25 --- node/storage-cleaner/src/appwrite.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index 9c1d29eb..f9653929 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -22,7 +22,7 @@ class AppwriteService { let hasNextPage = true; const queries = [ Query.lessThan('$createdAt', getExpiryDate()), - Query.limit(100), + Query.limit(25), ]; do { const response = await this.storage.listFiles(bucketId, queries); From 0536ef1db8b1c3b24742ff15be03a3b10c29d6d1 Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Fri, 27 Oct 2023 15:49:21 +0530 Subject: [PATCH 17/18] added requested changes --- node/storage-cleaner/src/appwrite.js | 7 +++---- node/storage-cleaner/src/util.js | 17 +++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js index f9653929..1ec6967a 100644 --- a/node/storage-cleaner/src/appwrite.js +++ b/node/storage-cleaner/src/appwrite.js @@ -19,20 +19,19 @@ class AppwriteService { * @returns {Promise} A Promise that resolves when the bucket is cleaned. */ async cleanBucket(bucketId) { - let hasNextPage = true; + let response; const queries = [ Query.lessThan('$createdAt', getExpiryDate()), Query.limit(25), ]; do { - const response = await this.storage.listFiles(bucketId, queries); + response = await this.storage.listFiles(bucketId, queries); await Promise.all( response.files.map((file) => this.storage.deleteFile(bucketId, file.$id) ) ); - hasNextPage = response.files.length > 0; - } while (hasNextPage); + } while (response.files.length > 0); } } diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index 36edc715..4f209046 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -1,12 +1,14 @@ /** - * Calculate and return the expiry date based on the retention period. - * @returns {string} The calculated expiry date in ISO format + * Returns a date subtracted by the retention period from the current date. + * The retention period is fetched from the RETENTION_PERIOD_DAYS environment variable. + * Defaults to 30 days if the environment variable is not set or invalid. + * @returns {Date} The calculated expiry date. */ export function getExpiryDate() { - const retentionPeriod = +(process.env.RETENTION_PERIOD_DAYS ?? 30); - const expiryDate = new Date(); - expiryDate.setDate(expiryDate.getDate() - retentionPeriod); - return expiryDate.toISOString(); + const retentionPeriod = Number(process.env.RETENTION_PERIOD_DAYS ?? 30); + return new Date( + Date.now() - retentionPeriod * 24 * 60 * 60 * 1000 + ).toISOString(); } /** @@ -18,8 +20,7 @@ export function getExpiryDate() { export function throwIfMissing(obj, keys) { const missing = []; for (let key of keys) { - // using obj[key] != 0 for case when the value of key will be 0 - if (!(key in obj) || (!obj[key] && obj[key] != 0)) { + if (!(key in obj && obj[key] !== 0)) { missing.push(key); } } From 197dc01fd8ed89ef037ec6962bb7edcb0df6844a Mon Sep 17 00:00:00 2001 From: Gilbish Kosma Date: Fri, 27 Oct 2023 15:52:38 +0530 Subject: [PATCH 18/18] change return type of date in comment --- node/storage-cleaner/src/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js index 4f209046..58ccca44 100644 --- a/node/storage-cleaner/src/util.js +++ b/node/storage-cleaner/src/util.js @@ -2,7 +2,7 @@ * Returns a date subtracted by the retention period from the current date. * The retention period is fetched from the RETENTION_PERIOD_DAYS environment variable. * Defaults to 30 days if the environment variable is not set or invalid. - * @returns {Date} The calculated expiry date. + * @returns {string} The calculated expiry date in ISO 8601 format. */ export function getExpiryDate() { const retentionPeriod = Number(process.env.RETENTION_PERIOD_DAYS ?? 30);