-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[no-issue] fix semrush redirects i18n json files (#1468)
* wip: rgx with # * fix: 1282 processedCount update links 30x * wip: adjust to scan src/i18n with simple quote * fix: adjust 30x link url * chore: disable continue console.log * fix: async correction to process file i18n * fix: replace 30x links to the 200
- Loading branch information
1 parent
8a3fe28
commit cb4b377
Showing
6 changed files
with
155 additions
and
144 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,110 +1,121 @@ | ||
| import { parse } from 'csv-parse' | ||
| import fs from 'fs' | ||
| import { createReadStream } from 'fs' | ||
| import path from 'path' | ||
| import { parse } from 'csv-parse'; | ||
| import { promises as fs } from 'fs'; | ||
| import { createReadStream } from 'fs'; | ||
| import path from 'path'; | ||
|
|
||
| let counterFoundLinks = 0 | ||
| let PATH = { | ||
| csv: './www.azion.com_permanent_redirects_20241225.csv', | ||
| i18n: `${process.env.OLDPWD}/src/i18n` | ||
| } | ||
| const wwwazioncom = 'https://www.azion.com' | ||
| let counterFoundLinks = 0; | ||
| const PATH = { | ||
| csv: './www.azion.com_permanent_redirects_20241225.csv', | ||
| i18n: `${process.env.OLDPWD}/src/i18n` | ||
| }; | ||
| const wwwazioncom = 'https://www.azion.com'; | ||
| const removeLangFromUrl = (url) => url.replace('/pt-br', '').replace('/en', ''); | ||
| const removeHostFromUrl = (url) => url.replace(wwwazioncom, ''); | ||
| const removeHostAndLangFromUrl = (url) => removeLangFromUrl(removeHostFromUrl(url)); | ||
| const isFromRoot = (url) => url === wwwazioncom; | ||
|
|
||
| async function loadRedirects() { | ||
| const redirects = [] | ||
| const parser = createReadStream(PATH.csv).pipe( | ||
| parse({ | ||
| columns: true, | ||
| skip_empty_lines: true, | ||
| }) | ||
| ) | ||
|
|
||
| for await (const record of parser) { | ||
| redirects.push({ | ||
| page: record.page, | ||
| initialUrl: record.initial_url, | ||
| destinationUrl: record.destination_url, | ||
| statusCode: record.status, | ||
| discovered: record.discovered | ||
| }) | ||
| } | ||
| const redirects = []; | ||
| const parser = createReadStream(PATH.csv).pipe( | ||
| parse({ | ||
| columns: true, | ||
| skip_empty_lines: true, | ||
| }) | ||
| ); | ||
|
|
||
| return redirects | ||
| try { | ||
| for await (const record of parser) { | ||
| redirects.push({ | ||
| page: record.page, | ||
| initialUrl: record.initial_url, | ||
| destinationUrl: record.destination_url, | ||
| statusCode: record.status, | ||
| discovered: record.discovered | ||
| }); | ||
| } | ||
| return redirects; | ||
| } catch (error) { | ||
| throw new Error(`Error loading redirects: ${error.message}`); | ||
| } | ||
| } | ||
|
|
||
| async function processFile(filePath, redirects) { | ||
| fs.readFile(filePath, async (err, content) => { | ||
| if(err) { | ||
| console.error(err) | ||
| return | ||
| } | ||
| try { | ||
| const content = await fs.readFile(filePath, 'utf-8'); | ||
| let newContent = content; | ||
| let fileModified = false; | ||
|
|
||
| const utf8Content = Buffer.from(content).toString('utf-8') | ||
| for (const item of redirects) { | ||
| const url30x = isFromRoot(item.initialUrl) ? wwwazioncom : removeHostAndLangFromUrl(item.initialUrl); | ||
| const url200 = removeHostAndLangFromUrl(item.destinationUrl); | ||
| const isRoot = isFromRoot(url30x); | ||
| const rgx = new RegExp(`'${url30x}'`, 'g'); | ||
| const contentMatch = newContent.match(rgx); | ||
|
|
||
| for (const item of redirects) { | ||
| const url30x = item.initialUrl === wwwazioncom ? wwwazioncom : item.initialUrl.replace(wwwazioncom, '') | ||
| const url200 = item.destinationUrl.replace(wwwazioncom, '') | ||
| const isRoot = url30x === wwwazioncom | ||
| const rgx = new RegExp(`${url30x}`, 'g') | ||
| const contentMatch = utf8Content.match(rgx) | ||
| if (!contentMatch) continue; | ||
|
|
||
| if(!contentMatch) { | ||
| // console.log(`NOT MATCH `, `${rgx} : ${url30x}`) | ||
| continue | ||
| } else { | ||
| console.log(`MATCH`, `${rgx} : ${url30x}`) | ||
| } | ||
| counterFoundLinks++ | ||
| counterFoundLinks++; | ||
| fileModified = true; | ||
|
|
||
| console.log(`{ | ||
| isRoot: ${isRoot}, | ||
| file: ${filePath}, | ||
| rgx: ${rgx}, | ||
| url30x: ${url30x}, | ||
| url200: ${url200}, | ||
| contentMatch: ${contentMatch}, | ||
| contentMatchCount: ${contentMatch.length}, | ||
| processedCount: ${counterFoundLinks} | ||
| }`) | ||
| console.log({ | ||
| isRoot, | ||
| file: filePath, | ||
| rgx: rgx.toString(), | ||
| url30x, | ||
| url200, | ||
| contentMatchCount: contentMatch.length, | ||
| processedCount: counterFoundLinks | ||
| }); | ||
|
|
||
| const newContent = utf8Content.replace(isRoot ? /'https\:\/\/www\.azion\.com\/'/ : rgx, `'${url200}'`) | ||
| await fs.writeFile(filePath, newContent, async (err) => { | ||
| if(err) throw err | ||
| console.log(`[OK] ${filePath} updated`) | ||
| }) | ||
| } | ||
| }) | ||
| } | ||
| newContent = newContent.replace( | ||
| isRoot ? /'https\:\/\/www\.azion\.com\/'/ : rgx, | ||
| `'${url200}'` | ||
| ); | ||
| } | ||
|
|
||
| function processDirectory(directory, redirects) { | ||
| fs.readdir(directory, { withFileTypes: true }, (err, entries) => { | ||
| if (err) { | ||
| console.error('[ERROR] directory can not be readed:', err) | ||
| return | ||
| } | ||
| if (fileModified) { | ||
| await fs.writeFile(filePath, newContent); | ||
| console.log(`[OK] ${filePath} updated`); | ||
| } | ||
| } catch (error) { | ||
| console.error(`[ERROR] Processing file ${filePath}:`, error); | ||
| } | ||
| } | ||
|
|
||
| for (const entry of entries) { | ||
| const fullPath = path.join(directory, entry.name) | ||
| async function processDirectory(directory, redirects) { | ||
| try { | ||
| const entries = await fs.readdir(directory, { withFileTypes: true }); | ||
| const processingPromises = entries.map(async (entry) => { | ||
| const fullPath = path.join(directory, entry.name); | ||
|
|
||
| if (entry.isDirectory()) { | ||
| return processDirectory(fullPath, redirects); | ||
| } else if (entry.isFile()) { | ||
| return processFile(fullPath, redirects); | ||
| } else { | ||
| console.warn(`[WARN] ${fullPath} is not a file or directory`); | ||
| } | ||
| }); | ||
|
|
||
| if(entry.isDirectory()) { | ||
| processDirectory(fullPath, redirects) | ||
| } else if (entry.isFile()) { | ||
| processFile(fullPath, redirects) | ||
| } else { | ||
| console.error(`[ERROR] ${fullPath} is not a file or directory`) | ||
| } | ||
| } | ||
| }) | ||
| await Promise.all(processingPromises); | ||
| } catch (error) { | ||
| console.error(`[ERROR] Processing directory ${directory}:`, error); | ||
| } | ||
| } | ||
|
|
||
| async function main() { | ||
| try { | ||
| const redirects = await loadRedirects() | ||
| processDirectory(PATH.i18n, redirects) | ||
| } catch (error) { | ||
| console.error('[ERROR] ', error) | ||
| process.exit(1) | ||
| } | ||
| try { | ||
| console.log('[INFO] Loading redirects...'); | ||
| const redirects = await loadRedirects(); | ||
| console.log(`[INFO] Loaded ${redirects.length} redirects`); | ||
|
|
||
| console.log('[INFO] Starting directory processing...'); | ||
| await processDirectory(PATH.i18n, redirects); | ||
| console.log(`[INFO] Processing complete. Found and replaced ${counterFoundLinks} links`); | ||
| } catch (error) { | ||
| console.error('[ERROR] Fatal error:', error); | ||
| process.exit(1); | ||
| } | ||
| } | ||
|
|
||
| main() | ||
| main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.