Skip to content

Commit

Permalink
building multi platform docker images (#101)
Browse files Browse the repository at this point in the history
* building multi platform docker images

* upgrading dependencies | using scraping ant for neubaukompass
  • Loading branch information
orangecoding authored Jul 24, 2024
1 parent 8ba3a53 commit 92db821
Show file tree
Hide file tree
Showing 9 changed files with 1,673 additions and 435 deletions.
1 change: 1 addition & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,4 @@ jobs:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64, linux/i386, linux/arm64
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ yarn run test
# Architecture
![Architecture](/doc/architecture.jpg "Architecture")

### Immoscout / Immonet
I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
### Immoscout / Immonet / NeubauKompass
I have added **experimental** support for Immoscout, Immonet and NeubauKompass. They all are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.

To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
Expand Down
2 changes: 1 addition & 1 deletion lib/provider/immoscout.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ function nullOrEmpty(val) {
function normalize(o) {
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
return Object.assign(o, { title, address, link });
}
function applyBlacklist(o) {
Expand Down
6 changes: 5 additions & 1 deletion lib/provider/neubauKompass.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import utils from '../utils.js';
let appliedBlackList = [];
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function normalize(o) {
return o;
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
return {...o, link};
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
Expand Down
3 changes: 2 additions & 1 deletion lib/services/scrapingAnt.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
import { metaInformation as immoNetInfo } from '../provider/immonet.js';
import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
import { config } from '../utils.js';

const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
'window.scrollTo(0,document.body.scrollHeight);'
).toString('base64')}`;

const needScrapingAnt = (id) => {
return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
};
export const transformUrlForScrapingAnt = (url, id) => {
let urlParams = '';
Expand Down
22 changes: 11 additions & 11 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "8.0.7",
"version": "8.1.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"start": "node index.js",
Expand Down Expand Up @@ -55,7 +55,7 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-ui": "2.60.0",
"@douyinfe/semi-ui": "2.62.1",
"@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2",
"@sendgrid/mail": "8.1.3",
Expand All @@ -64,7 +64,7 @@
"body-parser": "1.20.2",
"cookie-session": "2.1.0",
"handlebars": "4.7.8",
"highcharts": "11.4.3",
"highcharts": "11.4.6",
"highcharts-react-official": "3.2.1",
"lodash": "4.17.21",
"lowdb": "6.0.1",
Expand All @@ -84,25 +84,25 @@
"serve-static": "1.15.0",
"slack": "11.0.2",
"string-similarity": "^4.0.4",
"vite": "5.2.13",
"vite": "5.3.4",
"x-ray": "2.3.4"
},
"devDependencies": {
"@babel/core": "7.24.7",
"@babel/eslint-parser": "7.24.7",
"@babel/preset-env": "7.24.7",
"@babel/core": "7.24.9",
"@babel/eslint-parser": "7.24.8",
"@babel/preset-env": "7.24.8",
"@babel/preset-react": "7.24.7",
"chai": "5.1.1",
"eslint": "8.56.0",
"eslint-config-prettier": "8.8.0",
"eslint-plugin-react": "7.34.2",
"esmock": "2.6.5",
"eslint-plugin-react": "7.35.0",
"esmock": "2.6.7",
"history": "5.3.0",
"husky": "4.3.8",
"less": "4.2.0",
"lint-staged": "13.2.2",
"mocha": "10.4.0",
"prettier": "3.3.2",
"mocha": "10.7.0",
"prettier": "3.3.3",
"redux-logger": "3.0.6"
}
}
66 changes: 37 additions & 29 deletions test/provider/neubauKompass.test.js
Original file line number Diff line number Diff line change
@@ -1,36 +1,44 @@
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import { expect } from 'chai';
import {get} from '../mocks/mockNotification.js';
import {mockFredy, providerConfig} from '../utils.js';
import {expect} from 'chai';
import * as provider from '../../lib/provider/neubauKompass.js';
import * as scrapingAnt from '../../lib/services/scrapingAnt.js';

describe('#neubauKompass testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.neubauKompass, [], []);
it('should test neubauKompass provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj.serviceName).to.equal('neubauKompass');
notificationObj.payload.forEach((notify) => {
expect(notify).to.be.a('object');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.neubaukompass.de');
expect(notify.address).to.be.not.empty;
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.neubauKompass, [], []);
it.only('should test neubauKompass provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
if (!scrapingAnt.isScrapingAntApiKeySet()) {
/* eslint-disable no-console */
console.info('Skipping Neubaukompass test as ScrapingAnt Api Key is not set.');
/* eslint-enable no-console */
resolve();
return;
}
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj.serviceName).to.equal('neubauKompass');
notificationObj.payload.forEach((notify) => {
expect(notify).to.be.a('object');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.neubaukompass.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});
resolve();
});
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
description={
<div>
<p>
If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
If you chose Immoscout, Immonet or NeubauKompass as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
(See readme)
</p>
<p>
Expand Down
Loading

0 comments on commit 92db821

Please sign in to comment.