From 4cdaf668d962ee8e3279ef709bd130695bccdcde Mon Sep 17 00:00:00 2001 From: Tomasz Pluskiewicz Date: Thu, 30 Nov 2023 12:39:47 +0100 Subject: [PATCH 1/2] feat!: extension must be explicit --- .changeset/shaggy-grapes-joke.md | 5 +++ .editorconfig | 3 ++ README.md | 21 ++++++++++++- index.ts | 30 +++++++++++++----- lib/env.ts | 3 +- lib/path.ts | 11 +++++-- test/__snapshots__/index.test.ts.snap | 42 +++++++++++++------------- test/resources/import-relative.ttl | 1 + test/resources/property/identifier.ttl | 1 + 9 files changed, 84 insertions(+), 33 deletions(-) create mode 100644 .changeset/shaggy-grapes-joke.md diff --git a/.changeset/shaggy-grapes-joke.md b/.changeset/shaggy-grapes-joke.md new file mode 100644 index 0000000..ae9a92d --- /dev/null +++ b/.changeset/shaggy-grapes-joke.md @@ -0,0 +1,5 @@ +--- +"rdf-transform-graph-imports": minor +--- + +When importing local files, add support for `code:extension` to keep `code:imports` same in both local and web documents diff --git a/.editorconfig b/.editorconfig index 88989c7..d5251ad 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,3 +4,6 @@ root = true indent_size = 2 indent_style = space insert_final_newline = true + +[*.snap] +indent_style = tab diff --git a/README.md b/README.md index ba553cc..0d6dd85 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ PREFIX sh: [ # relative URIs are relative to the document itself - code:imports <./property/identifier> ; + code:imports <./property/identifier.ttl> ; code:imports ; ] . @@ -77,3 +77,22 @@ const dataset = await rdf.dataset().import(stream.pipe(imports(rdf, { basePath: '/path/to/shape.ttl' }))) ``` + +### Reusing imports for local and remote documents + +You may face the situation that you want to import the same file from a local file and a remote resource +but do not publish the extension in the remote resource's URL. In this case, you must add a `code:extension` +property to the import. + +```turtle +PREFIX code: + +[ + code:imports <./property/identifier> ; + code:extension "ttl" ; +] . +``` + +If the above is a local file, e.g. `/path/to/shape.ttl`, the import will be resolved as `/path/to/property/identifier.ttl`. + +If the above is a remote resource, e.g. `https://example.com/shape`, the import will be resolved as `https://example.com/property/identifier.ttl`. diff --git a/index.ts b/index.ts index 0b791f2..f297071 100644 --- a/index.ts +++ b/index.ts @@ -1,4 +1,5 @@ import through2 from 'through2' +import type { AnyPointer } from 'clownface' import type { Quad } from '@rdfjs/types' import { resolveImport } from './lib/path.js' import Environment from './lib/env.js' @@ -11,24 +12,37 @@ interface Options { function transform(env: Environment, { basePath }: Options = {}) { const code = env.namespace('https://code.described.at/') + const importStatements: AnyPointer = env.clownface() + return through2.obj(async function (quad: Quad, _, done) { - if (quad.predicate.equals(code.imports)) { - try { - const importTarget = resolveImport(quad.object, basePath) + if (quad.predicate.equals(code.imports) || quad.predicate.equals(code.extension)) { + importStatements.dataset.add(quad) + return done() + } + + done(null, quad) + }, async function (done) { + try { + const imports = importStatements.has(code.imports) + .map(Import => { + const importPath = Import.out(code.imports).term! + const extension = Import.out(code.extension).value + return resolveImport(importPath, { basePath, extension }) + }) + + for (const importTarget of imports) { const importStream = fetchImport(env, importTarget) .pipe(transform(env, { basePath: importTarget })) for await (const importedQuad of importStream) { this.push(importedQuad) } - - return done() - } catch (e) { - return done(e) } + } catch (e: unknown) { + this.destroy(new Error(`Failed to import: ${e}`)) } - done(null, quad) + done() }) } diff --git a/lib/env.ts b/lib/env.ts index b12a47d..b93e29c 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -2,6 +2,7 @@ import type { Environment as RdfjsEnvironment } from '@rdfjs/environment/Environ import type { NamespaceFactory } from '@rdfjs/namespace/Factory.js' import type FsUtilsFactory from '@zazuko/rdf-utils-fs/Factory.js' import type { FetchFactory } from '@rdfjs/fetch-lite/Factory.js' +import type ClownfaceFactory from 'clownface/Factory.js' -type Environment = RdfjsEnvironment +type Environment = RdfjsEnvironment export default Environment diff --git a/lib/path.ts b/lib/path.ts index ed87010..b6e5e3b 100644 --- a/lib/path.ts +++ b/lib/path.ts @@ -2,7 +2,12 @@ import * as url from 'url' import type { Term } from '@rdfjs/types' import isURI from 'is-uri' -export function resolveImport(importNode: Term, basePath: string | URL | undefined) { +interface Options { + basePath?: string | URL + extension?: string +} + +export function resolveImport(importNode: Term, { basePath, extension }: Options = {}) { if (importNode.termType !== 'NamedNode') { throw new Error(`Import target must be a NamedNode, got ${importNode.termType}`) } @@ -12,5 +17,7 @@ export function resolveImport(importNode: Term, basePath: string | URL | undefin } const base = typeof basePath === 'string' ? url.pathToFileURL(basePath) : basePath - return url.fileURLToPath(new URL(importNode.value + '.ttl', base)) + + const filePath = extension ? `${importNode.value}.${extension}` : importNode.value + return url.fileURLToPath(new URL(filePath, base)) } diff --git a/test/__snapshots__/index.test.ts.snap b/test/__snapshots__/index.test.ts.snap index 3b8eb55..ac768b3 100644 --- a/test/__snapshots__/index.test.ts.snap +++ b/test/__snapshots__/index.test.ts.snap @@ -4,17 +4,17 @@ exports[`rdf-merge-stream imports remote resources from files 1`] = ` "@prefix sh: . @prefix schema: . - sh:minCount 1 ; - sh:maxCount 1 . +<> a sh:NodeShape ; + sh:property . - sh:minLength 3 ; + sh:path schema:identifier ; + sh:minLength 3 ; sh:and ( - ) ; - sh:path schema:identifier . + ) . -<> a sh:NodeShape ; - sh:property . + sh:minCount 1 ; + sh:maxCount 1 . " `; @@ -23,17 +23,17 @@ exports[`rdf-merge-stream imports remote resources from http 1`] = ` "@prefix sh: . @prefix schema: . - sh:minCount 1 ; - sh:maxCount 1 . + a sh:NodeShape ; + sh:property . - sh:minLength 3 ; + sh:path schema:identifier ; + sh:minLength 3 ; sh:and ( - ) ; - sh:path schema:identifier . + ) . - a sh:NodeShape ; - sh:property . + sh:minCount 1 ; + sh:maxCount 1 . " `; @@ -42,17 +42,17 @@ exports[`rdf-merge-stream merges file stream by relative path 1`] = ` "@prefix sh: . @prefix schema: . - sh:minCount 1 ; - sh:maxCount 1 . + a sh:NodeShape ; + sh:property . - sh:minLength 3 ; + sh:path schema:identifier ; + sh:minLength 3 ; sh:and ( - ) ; - sh:path schema:identifier . + ) . - a sh:NodeShape ; - sh:property . + sh:minCount 1 ; + sh:maxCount 1 . " `; diff --git a/test/resources/import-relative.ttl b/test/resources/import-relative.ttl index 225a126..61ea9ea 100644 --- a/test/resources/import-relative.ttl +++ b/test/resources/import-relative.ttl @@ -5,6 +5,7 @@ PREFIX sh: [ code:imports <./property/identifier> ; + code:extension "ttl" ; ] . shape: diff --git a/test/resources/property/identifier.ttl b/test/resources/property/identifier.ttl index 234d08a..eb9f283 100644 --- a/test/resources/property/identifier.ttl +++ b/test/resources/property/identifier.ttl @@ -5,6 +5,7 @@ prefix code: [ code:imports <./shared> ; + code:extension "ttl" ; ] . property:identifier From 7ed501966c6d39ee3d62f0ca7910e3bf6f55fc9c Mon Sep 17 00:00:00 2001 From: Tomasz Pluskiewicz Date: Thu, 30 Nov 2023 13:05:46 +0100 Subject: [PATCH 2/2] fix: handle fetch fails --- .changeset/selfish-pens-mix.md | 5 +++++ index.ts | 4 ++-- lib/fetchImport.ts | 22 ++++++------------- package-lock.json | 27 ++++++++++++++++++++++-- package.json | 7 +++++- test/index.test.ts | 10 +++++++++ test/mocha-setup.cjs | 4 ++++ test/server/index.ts | 8 ++++++- test/server/resources/invalid-import.ttl | 7 ++++++ 9 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 .changeset/selfish-pens-mix.md create mode 100644 test/mocha-setup.cjs create mode 100644 test/server/resources/invalid-import.ttl diff --git a/.changeset/selfish-pens-mix.md b/.changeset/selfish-pens-mix.md new file mode 100644 index 0000000..45f97bd --- /dev/null +++ b/.changeset/selfish-pens-mix.md @@ -0,0 +1,5 @@ +--- +"rdf-transform-graph-imports": patch +--- + +Failures to fetch remote import would break the stream diff --git a/index.ts b/index.ts index f297071..93c1d15 100644 --- a/index.ts +++ b/index.ts @@ -31,8 +31,8 @@ function transform(env: Environment, { basePath }: Options = {}) { }) for (const importTarget of imports) { - const importStream = fetchImport(env, importTarget) - .pipe(transform(env, { basePath: importTarget })) + const fetchStream = await fetchImport(env, importTarget) + const importStream = fetchStream.pipe(transform(env, { basePath: importTarget })) for await (const importedQuad of importStream) { this.push(importedQuad) diff --git a/lib/fetchImport.ts b/lib/fetchImport.ts index 36d638b..1621abe 100644 --- a/lib/fetchImport.ts +++ b/lib/fetchImport.ts @@ -1,23 +1,15 @@ -import { Readable, PassThrough } from 'readable-stream' +import { Readable } from 'readable-stream' import Environment from './env.js' -export default function (env: Environment, importTarget: string | URL) { +export default async function (env: Environment, importTarget: string | URL) { if (typeof importTarget === 'string') { return env.fromFile(importTarget) } - const remoteStream = new PassThrough({ objectMode: true }) - - env.fetch(importTarget.toString()) - .then(response => { - return response.quadStream() as unknown as Promise - }) - .then(quadStream => { - quadStream.pipe(remoteStream) - }) - .catch(error => { - remoteStream.emit('error', error) - }) + const response = await env.fetch(importTarget.toString()) + if (!response.ok) { + throw new Error(`Failed to fetch: ${response.statusText}`) + } - return remoteStream + return response.quadStream() as unknown as Promise } diff --git a/package-lock.json b/package-lock.json index dc7cd0a..a4602a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rdf-transform-graph-imports", - "version": "0.0.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rdf-transform-graph-imports", - "version": "0.0.0", + "version": "0.1.0", "license": "MIT", "dependencies": { "is-uri": "^1.2.6", @@ -21,6 +21,7 @@ "@tpluscode/eslint-config": "^0.4.4", "@types/absolute-url": "^2.0.0", "@types/chai": "^4.3.9", + "@types/chai-as-promised": "^7.1.8", "@types/express": "^4.17.20", "@types/is-uri": "^1.0.2", "@types/mocha": "^10.0.3", @@ -33,6 +34,7 @@ "absolute-url": "^2.0.0", "c8": "^8.0.1", "chai": "^4.3.10", + "chai-as-promised": "^7.1.1", "eslint-import-resolver-typescript": "^3.6.1", "express": "^4.18.2", "husky": "^8.0.3", @@ -2448,6 +2450,15 @@ "integrity": "sha512-69TtiDzu0bcmKQv3yg1Zx409/Kd7r0b5F1PfpYJfSHzLGtB53547V4u+9iqKYsTu/O2ai6KTb0TInNpvuQ3qmg==", "dev": true }, + "node_modules/@types/chai-as-promised": { + "version": "7.1.8", + "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.8.tgz", + "integrity": "sha512-ThlRVIJhr69FLlh6IctTXFkmhtP3NpMZ2QGq69StYLyKZFp/HOp1VdKZj7RvfNWYYcJ1xlbLGLLWj1UvP5u/Gw==", + "dev": true, + "dependencies": { + "@types/chai": "*" + } + }, "node_modules/@types/clownface": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/clownface/-/clownface-2.0.3.tgz", @@ -3886,6 +3897,18 @@ "node": ">=4" } }, + "node_modules/chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "dependencies": { + "check-error": "^1.0.2" + }, + "peerDependencies": { + "chai": ">= 2.1.2 < 5" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", diff --git a/package.json b/package.json index df94742..294ddce 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "@tpluscode/eslint-config": "^0.4.4", "@types/absolute-url": "^2.0.0", "@types/chai": "^4.3.9", + "@types/chai-as-promised": "^7.1.8", "@types/express": "^4.17.20", "@types/is-uri": "^1.0.2", "@types/mocha": "^10.0.3", @@ -60,6 +61,7 @@ "absolute-url": "^2.0.0", "c8": "^8.0.1", "chai": "^4.3.10", + "chai-as-promised": "^7.1.1", "eslint-import-resolver-typescript": "^3.6.1", "express": "^4.18.2", "husky": "^8.0.3", @@ -77,6 +79,9 @@ }, "mocha": { "extension": "ts", - "loader": "ts-node/esm" + "loader": "ts-node/esm", + "require": [ + "test/mocha-setup.cjs" + ] } } diff --git a/test/index.test.ts b/test/index.test.ts index e47995c..9bda954 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -59,6 +59,16 @@ describe('rdf-merge-stream', () => { // then expect(await turtle(merged)).toMatchSnapshot() }) + + it('fails when remote resource is not found', async () => { + // given + const response = await rdf.fetch('http://localhost:6666/invalid-import') + const root = await response.quadStream() as unknown as Readable + + // then + await expect(rdf.dataset().import(root.pipe(transform(rdf)))) + .to.be.eventually.rejectedWith('Failed to fetch: Not Found') + }) }) async function turtle(merged: Dataset) { diff --git a/test/mocha-setup.cjs b/test/mocha-setup.cjs new file mode 100644 index 0000000..8187a33 --- /dev/null +++ b/test/mocha-setup.cjs @@ -0,0 +1,4 @@ +const chai = require('chai') +const chaiAsPromised = require('chai-as-promised') + +chai.use(chaiAsPromised); diff --git a/test/server/index.ts b/test/server/index.ts index a42b816..5691cc9 100644 --- a/test/server/index.ts +++ b/test/server/index.ts @@ -1,5 +1,6 @@ import * as url from 'url' import * as path from 'path' +import * as fs from 'fs' import express from 'express' import * as absoluteUrl from 'absolute-url' import rdfHandler from '@rdfjs/express-handler' @@ -14,7 +15,12 @@ export function start() { .get('/*', (req, res) => { res.setHeader('Content-Type', 'text/turtle') - const quads = env.fromFile(path.join(__dirname, `${req.path}.ttl`), { + const ttlPath = path.join(__dirname, `${req.path}.ttl`) + if (!fs.existsSync(ttlPath)) { + return res.sendStatus(404) + } + + const quads = env.fromFile(ttlPath, { baseUri: req.absoluteUrl(), }) diff --git a/test/server/resources/invalid-import.ttl b/test/server/resources/invalid-import.ttl new file mode 100644 index 0000000..92d4e1a --- /dev/null +++ b/test/server/resources/invalid-import.ttl @@ -0,0 +1,7 @@ +PREFIX code: +PREFIX schema: +PREFIX sh: + +[ + code:imports ; +] .