Skip to content

Commit

Permalink
remove container fixing code
Browse files Browse the repository at this point in the history
  • Loading branch information
@ilmartyrk
ilmartyrk committed Nov 4, 2024
1 parent 5f49e69 commit a09e64e
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 208 deletions.
74 changes: 1 addition & 73 deletions npm-shrinkwrap.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 1 addition & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
"campaign-mustache": "git+https://github.com/citizenos/campaign-mustache#0e6021098a3104011ced756e6409501cd0651f90",
"campaign-nodemailer": "1.1.0",
"campaign-terminal": "git+https://github.com/citizenos/campaign-terminal#83bca04d61f8730b23c5439601d513ceae7c4012",
"concat-stream": "^2.0.0",
"config": "3.3.12",
"connect-redis": "^7.1.1",
"cookie-parser": "1.4.6",
Expand Down Expand Up @@ -84,9 +83,7 @@
"twit": "2.2.11",
"undersign": "git+https://github.com/moll/js-undersign",
"uuid": "10.0.0",
"validator": "13.12.0",
"yauzl": "^3.1.3",
"yazl": "^3.1.0"
"validator": "13.12.0"
},
"devDependencies": {
"@eslint/js": "^9.7.0",
Expand Down
131 changes: 0 additions & 131 deletions routes/api/topic.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7026,137 +7026,6 @@ module.exports = function (app) {
}
});

const authApiKey = app.get('middleware.authApiKey');
app.get(['/api/downloads/bdocs/user/fix'], authApiKey, async function (req, res, next) {
const start = new Date(req.query.start || '01.01.2024').toISOString();
const end = new Date(req.query.end || '10.27.2024').toISOString();
console.log('START:', start, 'END:', end);
const yazl = require('yazl');
const yauzl = require('yauzl');
//TODO: Make use of streaming once Sequelize supports it - https://github.com/sequelize/sequelize/issues/2454
try {
const voteUserContainers = await VoteUserContainer.findAll(
{
where: {
createdAt: { [Op.lt]: end, [Op.gt]: start }
}
}
)
const count = voteUserContainers.length;
let updated = 0;
console.log('ROWS', count);

if (!count) {
return res.notFound();
}
voteUserContainers.forEach(async (row) => {
const container = row.container;
const newzip = new yazl.ZipFile();
await new Promise(function () {
// read the zip from buffer (entire zip, this cannot be streamed)
yauzl.fromBuffer(container, { lazyEntries: true }, (err, zip) => {
if (err) {
console.log("Error accessing artifact: ", err);
return;
}

// read each item in the zip
zip.readEntry();
zip.on("entry", function (entry) {
// we only want files in the src dir, skip others
// extract file
zip.openReadStream(entry, { decompress: entry.isCompressed() ? true : null }, function (err, readStream) {
if (err) {
zip.close();
console.log("Failed to read file in artifact: ", err);
return;
}

// collect data into buffer
let buffer = null;
readStream.on('data', function (d) {
if (!buffer) {
buffer = d;
} else {
buffer = Buffer.concat([buffer, d]);
}
});

// file data collection completed
readStream.on('end', function () {
// add it to the new zip (without the src dir in the path)
if (entry.fileName.indexOf('signature') > -1) {
const replaceLast = (str, pattern, replacement) => {
const match = typeof pattern === "string"
? pattern
: (str.match(new RegExp(pattern.source, "g")) || []).slice(-1)[0];
if (!match) return str;
const last = str.lastIndexOf(match);
return last !== -1
? `${str.slice(0, last)}${replacement}${str.slice(last + match.length)}`
: str;
};

let content = buffer.toString();
const match = content.match(/<ds:CanonicalizationMethod Algorithm="http:\/\/www\.w3\.org\/2001\/10\/xml-exc-c14n#" \/>/g);
if (match.length === 3) {
updated++;
content = replaceLast(content, '<ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#" />', '')
}
newzip.addBuffer(Buffer.from(content), entry.fileName);
} else {
newzip.addBuffer(buffer, entry.fileName);
}

// continue to next entry
zip.readEntry();
});

// fail on error
readStream.on('error', function (err) {
zip.close();
console.log("Failed to extract file from artifact: ", err);
return;
});
});

});

// all items processed
zip.on("end", async function () {
function slurp(stream, encoding) {

var concatStream = require("concat-stream")
return new Promise(function (resolve, reject) {
if (encoding) stream.setEncoding(encoding)
stream.pipe(concatStream(resolve))
stream.on("error", reject)
})
}
// console.log("Completed extracting all files", updated, count);
// all files added
newzip.end();
const container = await slurp(newzip.outputStream);
console.log('Updated', updated);
await VoteUserContainer.update({
container,
},{
where: {
voteId: row.voteId,
PID: row.PID
}
})
});
})
});
});

return res.ok();
} catch (err) {
return next(err);
}
});

const topicDownloadBdocFinal = async function (req, res, next) {
const topicId = req.params.topicId;
const voteId = req.params.voteId;
Expand Down

0 comments on commit a09e64e

Please sign in to comment.