Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: rocktimsaikia/meta-fetcher
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 1.0.0
Choose a base ref
...
head repository: rocktimsaikia/meta-fetcher
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
  • 16 commits
  • 13 files changed
  • 5 contributors

Commits on Sep 12, 2020

  1. Update readme.md

    rocktimsaikia authored Sep 12, 2020
    Copy the full SHA
    d15a1e1 View commit details
  2. Update package.json

    rocktimsaikia authored Sep 12, 2020
    Copy the full SHA
    ce9dda3 View commit details

Commits on Sep 13, 2020

  1. Copy the full SHA
    4c605fc View commit details

Commits on Sep 25, 2020

  1. Update readme.md

    rocktimsaikia authored Sep 25, 2020
    Copy the full SHA
    f5948d8 View commit details

Commits on Oct 8, 2020

  1. Copy the full SHA
    96eb3c2 View commit details

Commits on Oct 12, 2020

  1. Copy the full SHA
    7b84952 View commit details

Commits on Oct 14, 2020

  1. Support for default export (#5)

    * switch to ava from jest
    
    * add commonjs support export
    
    * remove pnpm
    
    * remove redundant packages
    rocktimsaikia authored Oct 14, 2020
    Copy the full SHA
    6130342 View commit details
  2. Copy the full SHA
    83ffc34 View commit details
  3. Copy the full SHA
    891d61e View commit details
  4. update readme

    rocktimsaikia committed Oct 14, 2020
    Copy the full SHA
    1bc1d7b View commit details
  5. v2.0.0

    rocktimsaikia committed Oct 14, 2020
    Copy the full SHA
    1cf9cf6 View commit details
  6. update examples

    rocktimsaikia committed Oct 14, 2020
    Copy the full SHA
    b208fc7 View commit details

Commits on Mar 5, 2021

  1. implement bug fix for paths (#7)

    sometimes path may be a CDN url
    hwrdtm authored Mar 5, 2021
    Copy the full SHA
    0b0ede6 View commit details

Commits on Mar 8, 2021

  1. Copy the full SHA
    4c8ed6a View commit details
  2. Copy the full SHA
    fd6a21b View commit details

Commits on Apr 7, 2021

  1. Copy the full SHA
    73ba2b0 View commit details
Showing with 143 additions and 145 deletions.
  1. +1 −2 .gitignore
  2. +1 −13 .travis.yml
  3. +0 −12 __tests__/index.ts
  4. +0 −13 __tests__/util.ts
  5. +1 −1 example/index.js
  6. +1 −1 example/index.ts
  7. +40 −45 package.json
  8. +29 −29 readme.md
  9. +24 −16 src/index.ts
  10. +4 −0 src/util.ts
  11. +11 −0 test/test.ts
  12. +17 −0 test/util.ts
  13. +14 −13 tsconfig.json
3 changes: 1 addition & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
/node_modules
/package-lock.json
/yarn-lock.json
/pnpm-lock.yaml
/yarn.lock
/dist
14 changes: 1 addition & 13 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,5 @@
language: node_js

node_js:
- "node"
- "12"
- "10"

before_install:
- npm install -g pnpm

install:
- pnpm install

script:
- pnpm run test

notifications:
email: false
12 changes: 0 additions & 12 deletions __tests__/index.ts

This file was deleted.

13 changes: 0 additions & 13 deletions __tests__/util.ts

This file was deleted.

2 changes: 1 addition & 1 deletion example/index.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const {fetchMetaData} = require('../dist');
const fetchMetaData = require('../dist');

(async () => {
const result = await fetchMetaData('https://hoppscotch.io/', {
2 changes: 1 addition & 1 deletion example/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Example script using url-fetch
'use strict';
import {fetchMetaData} from '../src';
import fetchMetaData from '../src';

(async () => {
const result = await fetchMetaData('https://microtip.now.sh', {
85 changes: 40 additions & 45 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,76 +1,71 @@
{
"name": "meta-fetcher",
"version": "1.0.0",
"version": "2.0.1",
"description": "Tiny URL meta-data fetcher that scraps the meta-data of a given URL string",
"keywords": [
"meta-man",
"url-metadata",
"url-metadata-scrapper",
"metadata-scrapper",
"website-metadata",
"website-metadata-scrapper",
"metadata",
"scrapper"
],
"bugs": {
"url": "https://github.com/rocktimsaikia/meta-fetch/issues"
},
"license": "MIT",
"author": {
"name": "Rocktim Saikia",
"email": "rocktimthedev@gmail.com",
"url": "https://rocktim.xyz"
"url": "https://rocktimsaikia.now.sh"
},
"main": "dist/index.js",
"module": "dist/index.esm.js",
"types": "dist/index.d.ts",
"files": [
"dist/**/*"
],
"repository": "rocktimsaikia/meta-fetcher",
"homepage": "https://github.com/rocktimsaikia/meta-fetcher",
"keywords": [
"meta-fetcher",
"url-metadata",
"url-metadata-scrapper",
"metadata-scrapper",
"website-metadata",
"website-metadata-scrapper",
"metadata",
"scrapper"
],
"scripts": {
"prepublishOnly": "pnpm run prebuild && pnpm run test:dev && pnpm run build",
"prebuild": "rimraf dist",
"build": "pnpm run build:esm && pnpm run build:cjs",
"build:cjs": "tsc --module commonjs",
"build:esm": "tsc --module esnext && cpy dist/index.js dist --rename index.esm.js",
"dev": "ts-node example/index.ts",
"dev:js": "node --trace-warnings example/index.js",
"test:dev": "xo && pnpm run test",
"test": "jest --verbose"
},
"xo": {
"extensions": [
"ts"
],
"ignores": [
"__tests__/**/*.ts"
],
"rules": {
"import/no-anonymous-default-export": 0,
"quote-props": 0
}
},
"jest": {
"preset": "ts-jest"
"build": "del dist && tsc",
"test": "xo && ava",
"test:ex": "node example/index.js",
"prepublishOnly": "npm run build"
},
"dependencies": {
"cheerio": "^1.0.0-rc.3",
"node-fetch": "^2.6.1"
},
"devDependencies": {
"@sindresorhus/tsconfig": "^0.7.0",
"@types/cheerio": "^0.22.21",
"@types/jsdom": "^16.2.4",
"@types/node": "^14.6.4",
"@types/node-fetch": "^2.5.7",
"cpy-cli": "^3.1.1",
"jest": "^26.4.2",
"rimraf": "^3.0.2",
"ts-jest": "^26.3.0",
"@typescript-eslint/eslint-plugin": "^4.4.1",
"@typescript-eslint/parser": "^4.4.1",
"ava": "^3.13.0",
"del-cli": "^3.0.1",
"eslint-config-xo-typescript": "^0.33.0",
"ts-node": "^9.0.0",
"typescript": "^4.0.2",
"typescript": "^4.0.3",
"xo": "^0.33.1"
},
"xo": {
"extends": "xo-typescript",
"ignores": [
"example/**/*"
]
},
"ava": {
"extensions": [
"ts"
],
"require": [
"ts-node/register"
]
},
"engines": {
"node": ">=12"
}
}
}
58 changes: 29 additions & 29 deletions readme.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# meta-fetcher :mag_right:

> Tiny URL meta-data fetcher that scraps the meta-data of a given `URL` string.
> Tiny URL meta-data fetcher that scraps the meta-data of a given `URL` string.
![Travis (.com) branch](https://img.shields.io/travis/com/rocktimsaikia/meta-fetcher/master)
[![XO code style](https://img.shields.io/badge/code_style-XO-5ed9c7.svg)](https://github.com/xojs/xo)
@@ -14,14 +14,15 @@ npm install meta-fetcher
```

## Basic Usage

```js
const {fetchMetaData} = require('meta-fetcher');
const fetchMetaData = require("meta-fetcher");

(async () => {
const result = await fetchMetaData('https://hoppscotch.io/');
console.log(result);
/*
const result = await fetchMetaData("https://hoppscotch.io/");
console.log(result);

/*
{
basic_metadata: {
website: 'https://hoppscotch.io/',
@@ -47,59 +48,58 @@ const {fetchMetaData} = require('meta-fetcher');
'https://hoppscotch.io/_nuxt/icons/icon_64x64.9834b3.png'
]
}
*/
*/
})();

```

## Advanced Usage (with options)

You can optionally set the `userAgent` and `fromEmail` options in request `Header` while fetching the meta-data.

```js
const {fetchMetaData} = require('meta-fetcher');
const fetchMetaData = require("meta-fetcher");

(async () => {
const result = await fetchMetaData('https://hoppscotch.io/', {
userAgent: 'Rocktim',
fromEmail: 'srocktim61@gmail.com'
const result = await fetchMetaData("https://hoppscotch.io/", {
userAgent: "Rocktim",
fromEmail: "srocktim61@gmail.com",
});
console.log(result);
})();
```

<br>

It can also fetch meta-data from `shortened-url` .For example:

```js
const {fetchMetaData} = require('meta-fetcher');
const fetchMetaData = require("meta-fetcher");

(async () => {
const result = await fetchMetaData('https://bit.ly/2Fj9sNF');
const result = await fetchMetaData("https://bit.ly/2Fj9sNF");
console.log(result);
})();
```

## Options

You can set these options in Header while fetching the data if needed.

| Option | Required | Default Value |
| :------------- | :----------: | -----------: |
| `userAgent` | No | `meta-fetcher` |
| `fromEMail` | No | `metafetch@email.com` |
| Option | Required | Default Value |
| :---------- | :------: | --------------------: |
| `userAgent` | No | `meta-fetcher` |
| `fromEMail` | No | `metafetch@email.com` |

## API
## Support

**metaDataFetch(url, options)**
You can support me and my work right here :

#### url
Type: `string`
url string that you want to fetch the meta-data from.
<a href="https://www.buymeacoffee.com/7BdaxfI"><img src="https://www.buymeacoffee.com/assets/img/guidelines/download-assets-sm-1.svg" height="40px"/></a>

#### options
Type: `object`
Optional `Header` paramerter you can set if needed.
## Related

## Contribute
For any new feature request or bug report, please open an issue or pull request in GitHub.
- [page-scrapper](https://github.com/rocktimsaikia/page-scrapper) - A simple node.js scrapper that pulls out all the links and images of a given site.

## License
MIT &copy; [Rocktim Saikia](https://rocktim.xyz)

MIT &copy; [Rocktim Saikia](https://rocktimsaikia.now.sh)
40 changes: 24 additions & 16 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -3,11 +3,13 @@ import fetch from 'node-fetch';
import * as cheerio from 'cheerio';
import {createValidUri} from './util';

type Metadata = Record<string | number | symbol, unknown>;

/**
* Fetches meta data of a given website url
* @param url | the website url to fetch the metadata from
*/
export const fetchMetaData = async (url: string, _options?: Options): Promise<Record<string, unknown>> => {
const fetchMetaData = async (url: string, _options?: Options): Promise<Metadata | undefined> => {
try {
const urlString: string = url.trim();

@@ -18,14 +20,14 @@ export const fetchMetaData = async (url: string, _options?: Options): Promise<Re

const options = {
userAgent: _options.userAgent ?? 'meta-fetch',
fromEmail: _options.userAgent ?? 'metafetch@email.com'
fromEmail: _options.fromEmail ?? 'metafetch@email.com'
};

const requestOptions = {
method: 'GET',
headers: {
'User-Agent': options.userAgent,
'From': options.fromEmail
From: options.fromEmail
}
};

@@ -37,7 +39,7 @@ export const fetchMetaData = async (url: string, _options?: Options): Promise<Re
const head = $('head');

// Basic site meta-data
const basicMeta = (): Record<string, unknown> => {
const basicMeta = (): Metadata => {
const website = response.url;
const title = head.find('title').text();
const desc = head.find('meta[name=description]').attr('content');
@@ -50,32 +52,32 @@ export const fetchMetaData = async (url: string, _options?: Options): Promise<Re
};

// Open graph basic
const fetchMeta = (): Record<string, unknown> => {
const fetchMeta = (): Metadata => {
const openGraphsArray = head.find('meta[property]');
const openGraphs = {};
const openGraphs: Metadata = {};
openGraphsArray.each((_, element) => {
const property = $(element)
.attr('property');
const content = $(element).attr('content');
if (!property.includes('twitter')) {
openGraphs[property] = content;
if (!property!.includes('twitter')) {
openGraphs[property!] = content;
}
});

return openGraphs;
};

// Open graph social
const fetchMetaSocial = (): Record<string, unknown> => {
const fetchMetaSocial = (): Metadata => {
const openGraphsArray = head.find('meta[name]');
const socials = {};
const socials: Metadata = {};

openGraphsArray.each((_, element) => {
const property = $(element).attr('name');
const content = $(element).attr('content');

if (property.includes('twitter')) {
socials[property] = content;
if (property!.includes('twitter')) {
socials[property!] = content;
}
});
return socials;
@@ -89,8 +91,8 @@ export const fetchMetaData = async (url: string, _options?: Options): Promise<Re
faviconArray.each((_, element) => {
const href = $(element).attr('href');

if (href.includes('shortcut icon') || href.includes('icon') || href.includes('apple-touch-startup-image') || href.includes('apple-touch-icon')) {
const validUri = createValidUri(response.url, href);
if (href!.includes('shortcut icon') || href!.includes('icon') || href!.includes('apple-touch-startup-image') || href!.includes('apple-touch-icon')) {
const validUri = createValidUri(response.url, href!);
favicons.push(validUri);
}
});
@@ -103,15 +105,15 @@ export const fetchMetaData = async (url: string, _options?: Options): Promise<Re
const openGraph_social = fetchMetaSocial();
const favicons = fetchFavicons();

const metaData: Record<string, unknown> = {
const metaData: Metadata = {
basic_metadata: basicMetaData,
opengraph: openGraphs,
opengraph_social: openGraph_social,
favicons
};

return metaData;
} catch (error) {
} catch (error: unknown) {
console.error(error);
}
};
@@ -121,3 +123,9 @@ interface Options{
userAgent?: string;
fromEmail?: string;
}

export default fetchMetaData;

// For CommonJS default export support
module.exports = fetchMetaData;
module.exports.default = fetchMetaData;
4 changes: 4 additions & 0 deletions src/util.ts
Original file line number Diff line number Diff line change
@@ -9,6 +9,10 @@ export const createValidUri = (host: string, path: string): string => {
return path;
}

if (path.startsWith('http') || path.startsWith('https')) {
return path;
}

const updatedPath = path.replace('/', '');
return `${host}${updatedPath}`;
};
11 changes: 11 additions & 0 deletions test/test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import test from 'ava';
import fetchMetaData from '../src';

test('check meta-data', async t => {
const response = await fetchMetaData('https://google.com');

t.true(Object.prototype.hasOwnProperty.call(response, 'basic_metadata'));
t.true(Object.prototype.hasOwnProperty.call(response, 'opengraph'));
t.true(Object.prototype.hasOwnProperty.call(response, 'opengraph_social'));
t.true(Object.prototype.hasOwnProperty.call(response, 'favicons'));
});
17 changes: 17 additions & 0 deletions test/util.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import test from 'ava';
import {createValidUri} from '../src/util';

test('creates valid URI', t => {
const result: string = createValidUri('https://github.com/', '/rocktimsaikia');
t.is(result, 'https://github.com/rocktimsaikia');
});

test('returns the path', t => {
const result: string = createValidUri('https://github.com/', 'https://github.com/rocktimsaikia');
t.is(result, 'https://github.com/rocktimsaikia');
});

test('returns CDN paths', t => {
const result: string = createValidUri('https://medium.com/', 'https://cdn-static-1.medium.com/_/fp/icons/Medium-Avatar-500x500.svg');
t.is(result, 'https://cdn-static-1.medium.com/_/fp/icons/Medium-Avatar-500x500.svg');
});
27 changes: 14 additions & 13 deletions tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@

{
"compilerOptions": {
"outDir": "dist",
"target": "es2018",
"sourceMap": false,
"incremental": false,
"skipLibCheck": true,
"declaration": true,
"esModuleInterop": true,
"lib": [
"es2018"
]
"extends": "@sindresorhus/tsconfig",
"compilerOptions": {
"outDir": "dist",
"target": "es2018",
"lib": [
"es2018"
],
"noImplicitReturns": false
},
"include": [
"src/**/*"
"include": [
"src"
],
"exclude": [
"example/**/*"
]
}