diff --git a/node/storage-cleaner/.gitignore b/node/storage-cleaner/.gitignore new file mode 100644 index 00000000..6a7d6d8e --- /dev/null +++ b/node/storage-cleaner/.gitignore @@ -0,0 +1,130 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* \ No newline at end of file diff --git a/node/storage-cleaner/.prettierrc.json b/node/storage-cleaner/.prettierrc.json new file mode 100644 index 00000000..0a725205 --- /dev/null +++ b/node/storage-cleaner/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": true, + "singleQuote": true +} diff --git a/node/storage-cleaner/README.md b/node/storage-cleaner/README.md new file mode 100644 index 00000000..c8b998d5 --- /dev/null +++ b/node/storage-cleaner/README.md @@ -0,0 +1,63 @@ +# ๐Ÿงน Node.js Storage Cleaner Function + +Storage cleaner function to remove all files older than X number of days from the specified bucket. + +## ๐Ÿงฐ Usage + +### GET / + +Remove files older than X days from the specified bucket + +**Response** + +Sample `200` Response: Buckets cleaned + +## โš™๏ธ Configuration + +| Setting | Value | +| ----------------- | ------------- | +| Runtime | Node (18.0) | +| Entrypoint | `src/main.js` | +| Build Commands | `npm install` | +| Permissions | `any` | +| CRON | `0 1 * * *` | +| Timeout (Seconds) | 900 | + +## ๐Ÿ”’ Environment Variables + +### RETENTION_PERIOD_DAYS + +The number of days you want to retain a file. + +| Question | Answer | +| ------------ | ------ | +| Required | Yes | +| Sample Value | `1` | + +### APPWRITE_BUCKET_ID + +The ID of the bucket from which the files are to be deleted. + +| Question | Answer | +| ------------ | -------------- | +| Required | Yes | +| Sample Value | `652d...b4daf` | + +### APPWRITE_API_KEY + +API Key to talk to Appwrite backend APIs. + +| Question | Answer | +| ------------- | -------------------------------------------------------------------------------------------------- | +| Required | Yes | +| Sample Value | `d1efb...aec35` | +| Documentation | [Appwrite: Getting Started for Server](https://appwrite.io/docs/getting-started-for-server#apiKey) | + +### APPWRITE_ENDPOINT + +The URL endpoint of the Appwrite server. If not provided, it defaults to the Appwrite Cloud server: `https://cloud.appwrite.io/v1`. + +| Question | Answer | +| ------------ | ------------------------------ | +| Required | No | +| Sample Value | `https://cloud.appwrite.io/v1` | diff --git a/node/storage-cleaner/package-lock.json b/node/storage-cleaner/package-lock.json new file mode 100644 index 00000000..d4bc4a3b --- /dev/null +++ b/node/storage-cleaner/package-lock.json @@ -0,0 +1,210 @@ +{ + "name": "starter-template", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "starter-template", + "version": "1.0.0", + "dependencies": { + "node-appwrite": "^11.0.0" + }, + "devDependencies": { + "prettier": "^3.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-appwrite": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/node-appwrite/-/node-appwrite-11.0.0.tgz", + "integrity": "sha512-l+O0d1kCvY56NdlmsiWw91IlUzJszNW7FZodGCyvNlJeR4+JIpy9Mica3T4M+zvZGsbRqLhpghCysQmPBwwhQA==", + "dependencies": { + "axios": "^1.4.0", + "form-data": "^4.0.0" + } + }, + "node_modules/prettier": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", + "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + } + }, + "dependencies": { + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "requires": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, + "follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==" + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, + "node-appwrite": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/node-appwrite/-/node-appwrite-11.0.0.tgz", + "integrity": "sha512-l+O0d1kCvY56NdlmsiWw91IlUzJszNW7FZodGCyvNlJeR4+JIpy9Mica3T4M+zvZGsbRqLhpghCysQmPBwwhQA==", + "requires": { + "axios": "^1.4.0", + "form-data": "^4.0.0" + } + }, + "prettier": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", + "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", + "dev": true + }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + } + } +} diff --git a/node/storage-cleaner/package.json b/node/storage-cleaner/package.json new file mode 100644 index 00000000..aba07a36 --- /dev/null +++ b/node/storage-cleaner/package.json @@ -0,0 +1,16 @@ +{ + "name": "storage-cleaner", + "version": "1.0.0", + "description": "", + "main": "src/main.js", + "type": "module", + "scripts": { + "format": "prettier --write ." + }, + "dependencies": { + "node-appwrite": "^11.0.0" + }, + "devDependencies": { + "prettier": "^3.0.0" + } +} diff --git a/node/storage-cleaner/src/appwrite.js b/node/storage-cleaner/src/appwrite.js new file mode 100644 index 00000000..1ec6967a --- /dev/null +++ b/node/storage-cleaner/src/appwrite.js @@ -0,0 +1,38 @@ +import { Client, Storage, Query } from 'node-appwrite'; +import { getExpiryDate } from './util.js'; + +class AppwriteService { + constructor() { + const client = new Client() + .setEndpoint( + process.env.APPWRITE_ENDPOINT ?? 'https://cloud.appwrite.io/v1' + ) + .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID) + .setKey(process.env.APPWRITE_API_KEY); + this.storage = new Storage(client); + } + + /** + * Clean up files from the storage bucket by removing files older than a specified retention period. + * + * @param {string} bucketId - The ID of the storage bucket to clean. + * @returns {Promise} A Promise that resolves when the bucket is cleaned. + */ + async cleanBucket(bucketId) { + let response; + const queries = [ + Query.lessThan('$createdAt', getExpiryDate()), + Query.limit(25), + ]; + do { + response = await this.storage.listFiles(bucketId, queries); + await Promise.all( + response.files.map((file) => + this.storage.deleteFile(bucketId, file.$id) + ) + ); + } while (response.files.length > 0); + } +} + +export default AppwriteService; diff --git a/node/storage-cleaner/src/main.js b/node/storage-cleaner/src/main.js new file mode 100644 index 00000000..2b940084 --- /dev/null +++ b/node/storage-cleaner/src/main.js @@ -0,0 +1,16 @@ +import AppwriteService from './appwrite.js'; +import { throwIfMissing } from './util.js'; + +export default async ({ req, res, log, error }) => { + throwIfMissing(process.env, [ + 'APPWRITE_API_KEY', + 'RETENTION_PERIOD_DAYS', + 'APPWRITE_BUCKET_ID', + ]); + + const appwrite = new AppwriteService(); + + await appwrite.cleanBucket(process.env.APPWRITE_BUCKET_ID); + + return res.send('Buckets cleaned', 200); +}; diff --git a/node/storage-cleaner/src/util.js b/node/storage-cleaner/src/util.js new file mode 100644 index 00000000..58ccca44 --- /dev/null +++ b/node/storage-cleaner/src/util.js @@ -0,0 +1,30 @@ +/** + * Returns a date subtracted by the retention period from the current date. + * The retention period is fetched from the RETENTION_PERIOD_DAYS environment variable. + * Defaults to 30 days if the environment variable is not set or invalid. + * @returns {string} The calculated expiry date in ISO 8601 format. + */ +export function getExpiryDate() { + const retentionPeriod = Number(process.env.RETENTION_PERIOD_DAYS ?? 30); + return new Date( + Date.now() - retentionPeriod * 24 * 60 * 60 * 1000 + ).toISOString(); +} + +/** + * Throws an error if any of the keys are missing from the object + * @param {*} obj + * @param {string[]} keys + * @throws {Error} + */ +export function throwIfMissing(obj, keys) { + const missing = []; + for (let key of keys) { + if (!(key in obj && obj[key] !== 0)) { + missing.push(key); + } + } + if (missing.length > 0) { + throw new Error(`Missing required fields: ${missing.join(', ')}`); + } +}