From af09efbb27a5159feac5cfe2b47513e28b51b4be Mon Sep 17 00:00:00 2001 From: Daniel Hauschildt Date: Mon, 25 Mar 2024 17:18:14 +0000 Subject: [PATCH] Added modified isnet that works on webgpu --- bundle/models/isnet | 3 + bundle/models/isnet_fp16 | 3 + bundle/models/isnet_quint8 | 3 + bundle/models/large | 3 - bundle/models/medium | 3 - bundle/models/small | 3 - package-lock.json | 10 +-- packages/node-examples/src/example_001.cjs | 2 +- packages/node/.resources.mjs | 17 +---- packages/node/CHANGELOG.md | 10 ++- ...42403201448-Added_Modnet_models_Added.yaml | 2 +- ...08-Added_isnet_model_for_webgpu_Added.yaml | 9 +++ packages/node/src/schema.ts | 37 ++++++---- packages/web-data/.resources.mjs | 17 +---- .../web-examples/vite-project/src/App.vue | 18 ++--- packages/web/CHANGELOG.md | 10 +++ ...42403201442-Added_Modnet_models_Added.yaml | 2 +- ...o_execute_on_gpu_webgpu_and_cpu_Added.yaml | 2 +- ...08-Added_isnet_model_for_webgpu_Added.yaml | 9 +++ packages/web/src/index.ts | 9 ++- packages/web/src/inference.ts | 4 -- packages/web/src/schema.ts | 67 ++++++++++++++----- 22 files changed, 145 insertions(+), 98 deletions(-) create mode 100644 bundle/models/isnet create mode 100644 bundle/models/isnet_fp16 create mode 100644 bundle/models/isnet_quint8 delete mode 100644 bundle/models/large delete mode 100644 bundle/models/medium delete mode 100644 bundle/models/small create mode 100644 packages/node/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml create mode 100644 packages/web/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml diff --git a/bundle/models/isnet b/bundle/models/isnet new file mode 100644 index 0000000..5ea0962 --- /dev/null +++ b/bundle/models/isnet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb060f136a6060b9b9f9f8e4d2071aa3820ae8d5f46cc5fc31353d1cbc49b2d9 +size 176174548 diff --git a/bundle/models/isnet_fp16 b/bundle/models/isnet_fp16 new file mode 100644 index 0000000..9be0076 --- /dev/null +++ b/bundle/models/isnet_fp16 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a1bf44caac6d15de3c463f2d421672931f7dc843bd3fd445dbb3c91ab958460 +size 88189140 diff --git a/bundle/models/isnet_quint8 b/bundle/models/isnet_quint8 new file mode 100644 index 0000000..bf51433 --- /dev/null +++ b/bundle/models/isnet_quint8 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5366ec2895e1e703229e543d892dd610ef4064a527b95d0e29d51d913b28b2e0 +size 44374051 diff --git a/bundle/models/large b/bundle/models/large deleted file mode 100644 index 34c0ac9..0000000 --- a/bundle/models/large +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:17b7466d93bb60b0e88affa2b0e8b3eee309c7de183d394ce4b956339ebd95e6 -size 176173887 diff --git a/bundle/models/medium b/bundle/models/medium deleted file mode 100644 index 07fd170..0000000 --- a/bundle/models/medium +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b6e8497ba978a6f5fbb647e419d2696cd80df5a23cb6a8ea532021911bd76acb -size 88188479 diff --git a/bundle/models/small b/bundle/models/small deleted file mode 100644 index a618a71..0000000 --- a/bundle/models/small +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7001d60734fdc112dd9c062635fb59cd401fb82a9d4213134bce4dbd655c803a -size 44342436 diff --git a/package-lock.json b/package-lock.json index d50a132..e4a4404 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "workspace", - "version": "1.4.5", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "workspace", - "version": "1.4.5", + "version": "1.5.0", "workspaces": [ "packages/web", "packages/web-data", @@ -4597,7 +4597,7 @@ }, "packages/node": { "name": "@imgly/background-removal-node", - "version": "1.4.5", + "version": "1.5.0", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { "@types/lodash": "~4.14.195", @@ -4768,7 +4768,7 @@ }, "packages/web": { "name": "@imgly/background-removal", - "version": "1.4.5", + "version": "1.5.0", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { "@types/lodash": "~4.14.0", @@ -4795,7 +4795,7 @@ }, "packages/web-data": { "name": "@imgly/background-removal-data", - "version": "1.4.5", + "version": "1.5.0", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { "onnxruntime-web": "~1.17.0" diff --git a/packages/node-examples/src/example_001.cjs b/packages/node-examples/src/example_001.cjs index a2f446a..718f38d 100644 --- a/packages/node-examples/src/example_001.cjs +++ b/packages/node-examples/src/example_001.cjs @@ -27,7 +27,7 @@ async function run() { ); }, // model: 'small', - model: 'medium', + model: 'isnet', // model: 'large', // model: 'modnet', // model: 'modnet_fp16', diff --git a/packages/node/.resources.mjs b/packages/node/.resources.mjs index 1a83199..5553b8c 100644 --- a/packages/node/.resources.mjs +++ b/packages/node/.resources.mjs @@ -1,22 +1,7 @@ export default [ { path: '/models/', - source: '../../bundle/models/small', - mime: 'application/octet-steam' - }, - { - path: '/models/', - source: '../../bundle/models/medium', - mime: 'application/octet-steam' - }, - { - path: '/models/', - source: '../../bundle/models/large', - mime: 'application/octet-steam' - }, - { - path: '/models/', - source: '../../bundle/models/modnet*', + source: '../../bundle/models/*', mime: 'application/octet-steam' } ]; diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 5479da9..16f7e47 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -4,11 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.4.5] +## [Unreleased] ### Added -- Added ThirdPartyLicenses.json Added +- Added Modnet models Added + +- Added isnet model for webgpu Added + +## [1.4.5] + +Added ThirdPartyLicenses.json Added ## [1.4.0] diff --git a/packages/node/changelog/Unreleased/20242403201448-Added_Modnet_models_Added.yaml b/packages/node/changelog/Unreleased/20242403201448-Added_Modnet_models_Added.yaml index 8e70495..e56ac57 100644 --- a/packages/node/changelog/Unreleased/20242403201448-Added_Modnet_models_Added.yaml +++ b/packages/node/changelog/Unreleased/20242403201448-Added_Modnet_models_Added.yaml @@ -6,4 +6,4 @@ type: Added # type: Security # private: true description: | - Added Modnet models Added + Added Modnet models Added diff --git a/packages/node/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml b/packages/node/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml new file mode 100644 index 0000000..17017e1 --- /dev/null +++ b/packages/node/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml @@ -0,0 +1,9 @@ +--- +type: Added +# type: Added +# type: Changed +# type: Removed +# type: Security +# private: true +description: | + Added isnet model for webgpu Added diff --git a/packages/node/src/schema.ts b/packages/node/src/schema.ts index 8fd32d1..8120f4c 100644 --- a/packages/node/src/schema.ts +++ b/packages/node/src/schema.ts @@ -46,7 +46,18 @@ const ConfigSchema = z .describe('Progress callback.') .optional(), model: z - .enum(['small', 'medium', 'large', 'modnet', 'modnet_fp16']) + .preprocess((val) => { + switch (val) { + case 'large': + return 'isnet'; + case 'small': + return 'isnet_quint8'; + case 'medium': + return 'isnet_fp16'; + default: + return val; + } + }, z.enum(['isnet', 'isnet_fp16', 'isnet_quint8', 'modnet', 'modnet_fp16' /*, 'modnet_quint8'*/])) .default('medium'), output: z .object({ @@ -63,19 +74,21 @@ const ConfigSchema = z }) .default({}) }) - .default({}); + .default({}) + .transform((config) => { + if (config.debug) console.log('Config:', config); + if (config.debug && !config.progress) { + config.progress = + config.progress ?? + ((key, current, total) => { + console.debug(`Downloading ${key}: ${current} of ${total}`); + }); + } + return config; + }); type Config = z.infer; function validateConfig(configuration?: Config): Config { - const config = ConfigSchema.parse(configuration ?? {}); - if (config.debug) console.log('Config:', config); - if (config.debug && !config.progress) { - config.progress = - config.progress ?? - ((key, current, total) => { - console.debug(`Downloading ${key}: ${current} of ${total}`); - }); - } - return config; + return ConfigSchema.parse(configuration ?? {}); } diff --git a/packages/web-data/.resources.mjs b/packages/web-data/.resources.mjs index 8029c56..bb35381 100644 --- a/packages/web-data/.resources.mjs +++ b/packages/web-data/.resources.mjs @@ -6,22 +6,7 @@ export default [ }, { path: '/models/', - source: '../../bundle/models/small', - mime: 'application/octet-steam' - }, - { - path: '/models/', - source: '../../bundle/models/medium', - mime: 'application/octet-steam' - }, - // { - // path: '/models/', - // source: '../../bundle/models/large', - // mime: 'application/octet-steam' - // }, - { - path: '/models/', - source: '../../bundle/models/modnet*', + source: '../../bundle/models/*', mime: 'application/octet-steam' } ]; diff --git a/packages/web-examples/vite-project/src/App.vue b/packages/web-examples/vite-project/src/App.vue index 14d1f41..2b53fea 100644 --- a/packages/web-examples/vite-project/src/App.vue +++ b/packages/web-examples/vite-project/src/App.vue @@ -12,7 +12,7 @@ export default { name: 'App', setup() { const images = [ - 'https://images.unsplash.com/photo-1686002359940-6a51b0d64f68?ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D&auto=format&fit=crop&w=2048&q=80', + 'https://images.unsplash.com/photo-1686002359940-6a51b0d64f68?ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D&auto=format&fit=crop&w=1024&q=80', 'https://images.unsplash.com/photo-1590523278191-995cbcda646b?ixlib=rb-1.2.1&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9', 'https://images.unsplash.com/photo-1709248835088-03bb0946d6ab?q=80&w=3387&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D' ]; @@ -27,8 +27,8 @@ export default { const publicPath = new URL(import.meta.url); publicPath.pathname = '/js/'; const config = { - // debug: false, - debug: true, + debug: false, + // debug: true, publicPath: publicPath.href, progress: (key, current, total) => { const [type, subtype] = key.split(':'); @@ -36,18 +36,18 @@ export default { 0 )}%`; }, + device: 'gpu', // device: 'cpu', - // model: 'small', - // model: 'medium', - // model: 'large', - + // model: 'isnet', + // model: 'isnet_fp16', + // model: 'isnet_quint8', // model: 'modnet', // model: 'modnet_fp16', //# does not work on webgpu // model: 'modnet_quint8', output: { quality: 0.8, - // format: 'image/png' - format: 'image/jpeg' + format: 'image/png' + // format: 'image/jpeg' // format: 'image/webp' //format: 'image/x-rgba8' //format: 'image/x-alpha8' diff --git a/packages/web/CHANGELOG.md b/packages/web/CHANGELOG.md index caf5a5d..1ba6b66 100644 --- a/packages/web/CHANGELOG.md +++ b/packages/web/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added Modnet models Added + +- Added option to execute on gpu (webgpu) and cpu Added + +- Added isnet model for webgpu Added + ## [1.4.5] ### Added diff --git a/packages/web/changelog/Unreleased/20242403201442-Added_Modnet_models_Added.yaml b/packages/web/changelog/Unreleased/20242403201442-Added_Modnet_models_Added.yaml index 8e70495..e56ac57 100644 --- a/packages/web/changelog/Unreleased/20242403201442-Added_Modnet_models_Added.yaml +++ b/packages/web/changelog/Unreleased/20242403201442-Added_Modnet_models_Added.yaml @@ -6,4 +6,4 @@ type: Added # type: Security # private: true description: | - Added Modnet models Added + Added Modnet models Added diff --git a/packages/web/changelog/Unreleased/20242403201534-Added_option_to_execute_on_gpu_webgpu_and_cpu_Added.yaml b/packages/web/changelog/Unreleased/20242403201534-Added_option_to_execute_on_gpu_webgpu_and_cpu_Added.yaml index cdfc103..31a8117 100644 --- a/packages/web/changelog/Unreleased/20242403201534-Added_option_to_execute_on_gpu_webgpu_and_cpu_Added.yaml +++ b/packages/web/changelog/Unreleased/20242403201534-Added_option_to_execute_on_gpu_webgpu_and_cpu_Added.yaml @@ -6,4 +6,4 @@ type: Added # type: Security # private: true description: | - Added option to execute on gpu (webgpu) and cpu Added + Added option to execute on gpu (webgpu) and cpu Added diff --git a/packages/web/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml b/packages/web/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml new file mode 100644 index 0000000..17017e1 --- /dev/null +++ b/packages/web/changelog/Unreleased/20242503170808-Added_isnet_model_for_webgpu_Added.yaml @@ -0,0 +1,9 @@ +--- +type: Added +# type: Added +# type: Changed +# type: Removed +# type: Security +# private: true +description: | + Added isnet model for webgpu Added diff --git a/packages/web/src/index.ts b/packages/web/src/index.ts index 761ba5e..425df97 100644 --- a/packages/web/src/index.ts +++ b/packages/web/src/index.ts @@ -37,22 +37,25 @@ async function removeBackground( ): Promise { const { config, session } = await init(configuration); + if (config.progress) config.progress('compute:decode', 0, 4); + const imageTensor = await utils.imageSourceToImageData(image, config); const [width, height, channels] = imageTensor.shape; - + config.progress?.('compute:inference', 1, 4); const alphamask = await runInference(imageTensor, config, session); const stride = width * height; - + config.progress?.('compute:mask', 2, 4); const outImageTensor = imageTensor; for (let i = 0; i < stride; i += 1) { outImageTensor.data[4 * i + 3] = alphamask.data[i]; } - + config.progress?.('compute:encode', 3, 4); const outImage = await utils.imageEncode( outImageTensor, config.output.quality, config.output.format ); + config.progress?.('compute:encode', 4, 4); return outImage; } diff --git a/packages/web/src/inference.ts b/packages/web/src/inference.ts index 7bfd5e7..f1d2e4b 100644 --- a/packages/web/src/inference.ts +++ b/packages/web/src/inference.ts @@ -26,14 +26,11 @@ async function runInference( config: Config, session: any ): Promise> { - if (config.progress) config.progress('compute:inference', 0, 1); const resolution = 1024; const [srcHeight, srcWidth, srcChannels] = imageTensor.shape; - let tensorImage = tensorResizeBilinear(imageTensor, resolution, resolution); const inputTensor = tensorHWCtoBCHW(tensorImage); // this converts also from float to rgba - // run const predictionsDict = await runOnnxSession( session, [['input', inputTensor]], @@ -45,6 +42,5 @@ async function runInference( let alphamaskU8 = convertFloat32ToUint8(alphamask); alphamaskU8 = tensorResizeBilinear(alphamaskU8, srcWidth, srcHeight); - if (config.progress) config.progress('compute:inference', 1, 1); return alphamaskU8; } diff --git a/packages/web/src/schema.ts b/packages/web/src/schema.ts index 6f223ea..a5624f2 100644 --- a/packages/web/src/schema.ts +++ b/packages/web/src/schema.ts @@ -41,7 +41,18 @@ const ConfigSchema = z .describe('Progress callback.') .optional(), model: z - .enum(['small', 'medium', 'modnet', 'modnet_fp16']) + .preprocess((val) => { + switch (val) { + case 'large': + return 'isnet'; + case 'small': + return 'isnet_quint8'; + case 'medium': + return 'isnet_fp16'; + default: + return val; + } + }, z.enum(['isnet', 'isnet_fp16', 'isnet_quint8', 'modnet', 'modnet_fp16' /*, 'modnet_quint8'*/])) .default('medium'), output: z .object({ @@ -58,25 +69,45 @@ const ConfigSchema = z }) .default({}) }) - .default({}); + .default({}) + .transform((config) => { + if (config.debug) console.log('Config:', config); + if (config.debug && !config.progress) { + config.progress = + config.progress ?? + ((key, current, total) => { + console.debug(`Downloading ${key}: ${current} of ${total}`); + }); + + if (!crossOriginIsolated) { + if (config.debug) + console.debug( + 'Cross-Origin-Isolated is not enabled. Performance will be degraded. Please see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer.' + ); + } + } + if (config.device == 'gpu') { + switch (config.model) { + case 'isnet': + break; + case 'isnet_fp16': + case 'isnet_quint8': + if (config.debug) console.debug('Switching to f32 model for GPU'); + config.model = 'isnet'; + break; + case 'modnet': + break; + case 'modnet_fp16': + if (config.debug) console.debug('Switching to f32 model for GPU'); + config.model = 'modnet'; + break; + } + } + return config; + }); type Config = z.infer; function validateConfig(configuration?: Config): Config { - const config = ConfigSchema.parse(configuration ?? {}); - if (config.debug) console.log('Config:', config); - if (config.debug && !config.progress) { - config.progress = - config.progress ?? - ((key, current, total) => { - console.debug(`Downloading ${key}: ${current} of ${total}`); - }); - - if (!crossOriginIsolated) { - console.debug( - 'Cross-Origin-Isolated is not enabled. Performance will be degraded. Please see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer.' - ); - } - } - return config; + return ConfigSchema.parse(configuration ?? {}); }