Merge branch 'refs/heads/ipv6'

.env.vault

@@ -4,8 +4,8 @@
 #/--------------------------------------------------/
 
 # development
-DOTENV_VAULT_DEVELOPMENT="33CsfzPvDOoqxa9lfdpwB1DU1dRcJnQn96KmexLjzofA95Ad02sAjMJb8aSZ7EGAW1L0hv2KaG+zqOAcq2kOszdS8QVJEAGDgYdhRK6cIa4nJ6T1G1JaYdqEw/Mw095gMo2Sd6U4VmVa84Z1ZCM2QRaJHoDmUnXVPStG+7mW8m2h+cIEIr7fn5vkthyTIosn1BeuOsLshOEvnsf0oh3oJ9NIMHUWFyoU3l8/Yh5QSa1Y+8yZyJTiVVIsevCeBVfiMJ3Y8BlBBsGM00QZaaOS4Al+o27GMisWbtrm4tcpy+3AGwzY0f2osTT2+Qm4FCHH1MPtHiq4btqnCCOwmYP3cecQgYDGW0sBL+q2MY18LTVt5jcLRiF4n8dKIXzUz1wJ1dEDaQoVvYV5tzG6rT2y2SDrF8brqNuXXpDIyUIlktitGNS+wZn6Os3WI3m5nrnX92uOxgCMGwTwMus7mHVC0yCnGWhxgu9hxE/TwPCrCbAGI/g1PA04FyVL1xv400N91um9ChentpFkZ0+Ji6hry2qdsa9QotEkzXplHEiAehGrdSx++dpz5XppuToQwkubUMiOtyizmTuWcHamy2WpGN2CRlCJlQqp1MVfJbQjTi2rWxGauiQQ5lX53o4Fyq0p2chDtyHgB06+JnmdyaHMBm1PgIl79B9LZ4PT08Pb++Wgcq6VR1v+Ai1Gm0lSKHcyiL/Hs0wB3/IuK6z+hk/Ei1gc9zB1VNZM2rIfGfUrBArCsAYkbG3P8PjziOFE+k3WHIRZy7xHJFWS9mmwqFQk7LGokUwaZRJvZXfVhkamcOfXsywVMiuffhMfAS013XQuMGnOQlStPWaMRyQow/2rUUg3u9pcTu5gWpNqhxPlfDok4NcLi38Z8c+tr3zA76qJ1dPyN5sApjDBiO3jazhm9k3+OxJX9S0Jk6NSMOsR3Psk6bRejmOPfqdj9nVMgB31k64b7o/dXAwzkq0XxkdVXZCESQ=="
-DOTENV_VAULT_DEVELOPMENT_VERSION=5
+DOTENV_VAULT_DEVELOPMENT="ORWRGqNhVqFxxZUlkC2BkcIXYdopiQeazoo0M9na4tTpsb+kFr/AnuS4s1ubrqIEDjonAOwSPWKiDpg8QNmbi+nZhJAjhNYkGydHsoOKDdY4VYmy2M5Uq9J73qSgnwgQivhrgyNEWKA3ILIGnVSp8HT/0RCWnIYLAfSwOHLc19k54VYXa5pgUIkWY6Q1BrR7lSTQ/A7Poy9/UdHYzQQNNOnrb97xNL1F4VdOugMLAwmUrpgNWrqE8JW/KzmkNOg5L3X9bG77CrrndbYD6LEzqG8KRC2KJHJMXvnadqAim802swbta3l9SPtYEIC7IOaKLH6uzcHWwk46cD+INCVnvj2krK3DbP1m6JJ4FE4ubFLbv62YMuHv2Epk6u5PrDL944Yen+u2UG+znBpWkmyzyW3zpMftJih1xe4wAI4unVV7Skvc8S5wWYeg/7dud+I9vE1YInO9lgW8PGW2QowcCe7iqDdUpYE6mSfRdlA5lo85wBiR9oDrfC5klHi/bQy6bzTARce+okwymooqpchS7uGyqIGDfKCXU5JGM6cNQ77YXCbNSPm/Ah97WLY6REqPhqwtC7QMlhdXpVK0ElVkY6N3esEYofWgPLBFcPuBiZyxridzVDz58N+9Uc/d5Trj5zHtjTbiY1gFsTSFWfyBd/yo1foi8MOPyKN9n/0Oss7hGa3HZiEKecWwO6ywkB9eHONWN71s3U/1S6CC0Hf1Sar9pF57WvoBvN/frIJQHIEEiS2TO1B8V4e70xEX70VJyd6sRxwWaD7fuiniM2FyK5VC5w+BM/rkFLaRJ5yvEVbyIDy/YOR4CJmhKYFfzSNOtDPlVm5vBxMxBIMTS2++G76/kogkWNY6tqgBryJbqqGqAOBbjU6i8zeokkgOSrRLT0at7f87nrw5kJy7vtvYJGXjIMUP0YNGkaSHL7whlD2XWaS4cUgX7iLAEB4hr21ANIs5UBxFjO7rdFqfoTPogYx/1ohTZ+Gp6DVmDT+7/A=="
+DOTENV_VAULT_DEVELOPMENT_VERSION=6
 
 # ci
 DOTENV_VAULT_CI="cobvJmwalFLHh2/aOvGqqzZuyzLCC45ndhHcGiD3gqr/vJbUTMAjHa4J4GqI7pYFEwO6MrU3uXjqOp2JALptbcC7mzrvBmZfzMCtoUY7ntRGRv1cYk1TGdSk"
@@ -16,8 +16,8 @@ DOTENV_VAULT_STAGING="hKaFYHTv0LXURnAH912AB77DPNBw1H9SknzIB6MLSQPZ5ZerLA8D0dcC11
 DOTENV_VAULT_STAGING_VERSION=4
 
 # production
-DOTENV_VAULT_PRODUCTION="4i0+PxnoANEe8Yyu2lM07t9BdM4Mfnpp6FBwm9N9GcqFX1QJBTV6jQBVv9ixc7zlWeSMPilS0NDBe1jheJq+hvuKl9EffGdkq4WXLUIDHTQSfbTYqFome16utRyN3lekLr+dACSnnIHxUtvGXAVpEsIcYcpn8dZZKMq0FT+AF14u+IcHhgRy1J8nPbNitumcZMFPNUbmVgpCz8KbkZNGbNSokWBacBbbZLvLjcFKDXL5J4r/7gqEtzCfJQVYqnFRw4q37y8WZ6vaJbrC0rCEVmrHOAFlzB20oLiCxEhldbm3ZA/Cut05gWlutIRTgtT5wULAR8KWiqCq65ggjQE/2W+ZMawrivYhXn5ymOvNdd+Ad9a2yqKrdHPsZKKkYJVGEIMFDXDOjK/FZF9y02jXXhuc6BleRLEfF+2lEzTSirG998bdo/D3d4AWdUliWmYyy/SAqfSw0WfYmYkgRHKtBo9h+6SuFww="
-DOTENV_VAULT_PRODUCTION_VERSION=6
+DOTENV_VAULT_PRODUCTION="0u0NmWWHxu2LYf4qRdS0zXjh0dIvqWjqn6q6Yu6iaYWB0ug/87wgfW4MAyVKCbuk2Q/TCdTTdWaOkd5XNXv1SFk0hki0Y0VHYAE6BlF3Iq0BOFhuUISRVTBZGi2KiLbMv5mvgP4MVTn+MHulqQzTsEMnjGqO5pO6g15BhTymq0VEMqWrlx61YFd88zxNJWiIVLugIxcUgIb1wMFb1hXQrRInRD8D/azpVajNUrcwQqhxzevZ7aMN4cN9Yxn2JkW3Gy4LLAEGfEKQf68RtJU+MkZ7YLtDwwMU5B9rf69L5qTPTfiUoop3Cw3HY935T1++YAP7fNjCv2OfwQ/8W8BaN685YxtSFkUJBsVxY17PMACQzVmCX7zMHvQHTus9ac7B1ttv6QMt19pHcy0KzWLx4M827MiJ2OonBiOv8EIBQD/0nKkgHvlDKekeATPo+R9r9aHN1vygdJnvmEXz3W2zhGZKVE1iaNwdX2Ks2zA7i7PyXL0="
+DOTENV_VAULT_PRODUCTION_VERSION=7
 
 #/----------------settings/metadata-----------------/
 DOTENV_VAULT="vlt_086aa47a0b612c416faba636bc84b11a07bb4611b951b0713fa386f2aefd4294"

docker-compose.yml

@@ -18,7 +18,7 @@ services:
     secrets:
       - mineskin.jwt.private.key
     deploy:
-      replicas: 4
+      replicas: 5
       placement:
         constraints: [node.labels.mineskin-api == true]
       update_config:

src/generator/Generator.ts

@@ -786,12 +786,14 @@ export class Generator {
                 originalUrl = this.rewriteUrl(originalUrl, options);
                 // Try to find the source image
                 const followResponse = await this.followUrl(originalUrl, options.breadcrumb);
-                if (!followResponse) {
+                if (!followResponse || typeof followResponse === 'string') {
                     span?.setStatus({
                         code: 2,
                         message: "invalid_argument"
                     });
-                    throw new GeneratorError(GenError.INVALID_IMAGE_URL, "Failed to find image from url", 400, undefined, originalUrl);
+                    throw new GeneratorError(GenError.INVALID_IMAGE_URL,
+                        "Failed to find image from url" + (typeof followResponse === 'string' ? ": " + followResponse : ""),
+                        400, undefined, originalUrl);
                 }
                 // Validate response headers
                 const url = this.getUrlFromResponse(followResponse, originalUrl);
@@ -918,8 +920,8 @@ export class Generator {
         return urlStr;
     }
 
-    protected static async followUrl(urlStr: string, breadcrumb?: string): Promise<Maybe<AxiosResponse>> {
-        if (!urlStr) return undefined;
+    protected static async followUrl(urlStr: string, breadcrumb?: string): Promise<string | Maybe<AxiosResponse>> {
+        if (!urlStr) return "no url";
 
         return await Sentry.startSpan({
             op: "generate_followUrl",
@@ -928,10 +930,10 @@ export class Generator {
             try {
                 const url = new URL(urlStr);
                 if (!url.host || !url.pathname) {
-                    return undefined;
+                    return "invalid host or path";
                 }
                 if (!url.protocol || (url.protocol !== "http:" && url.protocol !== "https:")) {
-                    return undefined;
+                    return "invalid protocol";
                 }
                 const follow = URL_FOLLOW_WHITELIST.includes(url.host!);
                 return await Requests.genericRequest({
@@ -947,8 +949,11 @@ export class Generator {
                 });
             } catch (e) {
                 Sentry.captureException(e);
+                if (e?.message?.includes("timeout")) {
+                    return "timeout";
+                }
             }
-            return undefined;
+            return "request failed";
         })
     }
 
@@ -1374,7 +1379,7 @@ export class Generator {
                     code: 2,
                     message: "invalid_argument"
                 });
-                throw new GeneratorError(GenError.INVALID_IMAGE, "Invalid file size", 400);
+                throw new GeneratorError(GenError.INVALID_IMAGE, `Invalid file size (${size})`, 400);
             }
 
             let fType;

src/generator/Requests.ts

@@ -14,6 +14,7 @@ import { Maybe, timeout } from "../util";
 import { IAccountDocument } from "../typings";
 import * as https from "https";
 import { Span } from "@sentry/types";
+import { networkInterfaces } from "os";
 
 export const GENERIC = "generic";
 export const MOJANG_AUTH = "mojangAuth";
@@ -187,7 +188,7 @@ export class Requests {
             }
 
             this.genericRequest({
-                url: 'https://api4.ipify.org?format=json',
+                url: 'https://api.ipify.org?format=json',
                 method: 'GET'
             }).then(response => {
                 console.log(debug("Public IP 4: " + response.data.ip));
@@ -245,7 +246,24 @@ export class Requests {
             let proxyType = proxy["type"];
 
             if (proxyType === "ip" && "ip" in proxy) {
-                this.setupIPProxyAxiosInstance(key, proxyKey, proxy["ip"]!!, requestConfig, constr);
+                let ip = proxy["ip"]!!;
+                if ("auto6" === ip) {
+                    console.log(debug("Looking for local IPv6 address for " + proxyKey));
+                    const interfaces = networkInterfaces();
+                    for (let id in interfaces) {
+                        for (let i of interfaces[id]!!) {
+                            if (i.family === "IPv6" && !i.internal) {
+                                ip = i.address;
+                                console.log(debug("Found IPv6 address " + ip + " for " + proxyKey));
+                                break;
+                            }
+                        }
+                        if (ip !== proxy["ip"]) {
+                            break;
+                        }
+                    }
+                }
+                this.setupIPProxyAxiosInstance(key, proxyKey, ip, requestConfig, constr);
             } else {
                 this.setupProxiedAxiosInstance(key, proxyKey, proxy, requestConfig, constr);
             }
@@ -458,7 +476,7 @@ export class Requests {
 
     public static async genericRequest(request: AxiosRequestConfig, bread?: string): Promise<AxiosResponse> {
         this.addBreadcrumb(request, bread);
-        return await this.trackSentryQueued(request,async span=>{
+        return await this.trackSentryQueued(request, async span => {
             return await this.runAxiosRequest(request, this.axiosInstance);
         });
     }

src/index.ts

@@ -1,4 +1,5 @@
 import "dotenv/config"
+import "./instrument"
 import * as sourceMapSupport from "source-map-support";
 import * as Sentry from "@sentry/node";
 import * as path from "path";
@@ -33,10 +34,9 @@ import { GitConfig } from "@inventivetalent/gitconfig";
 import { GithubWebhook } from "@inventivetalent/express-github-webhook/dist/src";
 import { Stats } from "./generator/Stats";
 import { Requests } from "./generator/Requests";
-import { info, warn } from "./util/colors";
+import { debug, info, warn } from "./util/colors";
 import { Discord } from "./util/Discord";
 import { Balancer } from "./generator/Balancer";
-import { nodeProfilingIntegration } from '@sentry/profiling-node';
 
 
 sourceMapSupport.install();
@@ -55,29 +55,29 @@ console.log("\n" +
     "" + hostname + "\n" +
     "\n");
 
-{
-    console.log("Initializing Sentry")
-    Sentry.init({
-        dsn: process.env.SENTRY_DSN,
-        release: process.env.SOURCE_COMMIT || "unknown",
-        integrations: [
-            nodeProfilingIntegration()
-        ],
-        serverName: hostname,
-        tracesSampleRate: 0.1,
-        sampleRate: 0.8,
-        ignoreErrors: [
-            "No duplicate found",
-            "Invalid image file size",
-            "Invalid image dimensions",
-            "Failed to find image from url",
-            "Invalid file size"
-        ]
-    });
-
-    // app.use(Sentry.Handlers.requestHandler());
-    // app.use(Sentry.Handlers.tracingHandler());
-}
+// {
+//     console.log("Initializing Sentry")
+//     Sentry.init({
+//         dsn: process.env.SENTRY_DSN,
+//         release: process.env.SOURCE_COMMIT || "unknown",
+//         integrations: [
+//             nodeProfilingIntegration()
+//         ],
+//         serverName: hostname,
+//         tracesSampleRate: 0.1,
+//         sampleRate: 0.8,
+//         ignoreErrors: [
+//             "No duplicate found",
+//             "Invalid image file size",
+//             "Invalid image dimensions",
+//             "Failed to find image from url",
+//             "Invalid file size"
+//         ]
+//     });
+//
+//     // app.use(Sentry.Handlers.requestHandler());
+//     // app.use(Sentry.Handlers.tracingHandler());
+// }
 
 
 const app = express();
@@ -281,6 +281,8 @@ async function init() {
 
     const preErrorHandler: ErrorRequestHandler = (err, req: Request, res: Response, next: NextFunction) => {
         console.warn(warn((isBreadRequest(req) ? req.breadcrumb + " " : "") + "Error in a route " + err.message));
+        Sentry.setExtra("route", req.path);
+        console.debug(debug(req.path));
         if (err instanceof MineSkinError) {
             Sentry.setTags({
                 "error_type": err.name,
@@ -325,6 +327,7 @@ async function init() {
                             errorType: err.name,
                             errorCode: err.code,
                             error: err.msg,
+                            breadcrumb: isBreadRequest(req) ? req.breadcrumb : null,
                             nextRequest: Math.round((Date.now() / 1000) + delayInfo.seconds), // deprecated
 
                             delayInfo: {
@@ -335,10 +338,11 @@ async function init() {
                     }).catch(e => Sentry.captureException(e));
                 });
         } else {
-            console.warn(err);
+            console.error("Unexpected Error", err);
             res.status(500).json({
                 success: false,
-                error: "An unexpected error occurred"
+                error: "An unexpected error occurred",
+                breadcrumb: isBreadRequest(req) ? req.breadcrumb : null
             })
         }
     }

src/instrument.ts

@@ -0,0 +1,24 @@
+import * as Sentry from "@sentry/node";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";
+import { resolveHostname } from "./util";
+
+const hostname = resolveHostname();
+
+console.log("Initializing Sentry")
+Sentry.init({
+    dsn: process.env.SENTRY_DSN,
+    release: process.env.SOURCE_COMMIT || "unknown",
+    integrations: [
+        nodeProfilingIntegration()
+    ],
+    serverName: hostname,
+    tracesSampleRate: 0.1,
+    sampleRate: 0.8,
+    ignoreErrors: [
+        "No duplicate found",
+        "Invalid image file size",
+        "Invalid image dimensions",
+        "Failed to find image from url",
+        "Invalid file size"
+    ]
+});

src/routes/account.ts

@@ -429,6 +429,7 @@ export async function getUserFromRequest(req: Request, res: Response, reject: bo
     } catch (e) {
         console.warn("Failed to verify JWT", e);
         console.log(getIp(req))
+        console.log(cookie);
         if (e instanceof JsonWebTokenError) {
             if (reject) res.status(401).json({error: 'invalid auth (2)'})
         }

src/routes/accountManager.ts

@@ -318,6 +318,9 @@ export const register = (app: Application, config: MineSkinConfig) => {
         try {
             let server = await Generator.getPreferredAccountServer(req.query["type"] as string);
             server = await Generator.getServerFromProxy(server);
+            if (server.endsWith('6')) {
+                server = server.substring(0, server.length - 1);
+            }
             res.json({
                 server: server,
                 host: `${ server }.api.mineskin.org`

src/routes/generate.ts

@@ -30,6 +30,10 @@ import { getUserFromRequest } from "./account";
 export const register = (app: Application) => {
 
     app.use("/generate", corsWithAuthMiddleware);
+    app.use("/generate", (req: GenerateRequest, res: Response, next) => {
+        addBreadcrumb(req, res);
+        next();
+    });
     app.use("/generate", generateLimiter);
     app.use("/generate", async (req, res, next) => {
         try {
@@ -245,6 +249,15 @@ export const register = (app: Application) => {
         };
     }
 
+    function addBreadcrumb(req: GenerateRequest, res: Response) {
+        const breadcrumbId = md5(`${ getIp(req) }${ Date.now() }${ Math.random() }`).substr(0, 8);
+        const breadcrumb = nextBreadColor()(breadcrumbId);
+        req.breadcrumb = breadcrumb;
+        res.header("X-MineSkin-Breadcrumb", breadcrumbId);
+        res.header("X-MineSkin-Timestamp", `${ Date.now() }`);
+        Sentry.setExtra("generate_breadcrumb", breadcrumbId);
+    }
+
     function getAndValidateOptions(type: GenerateType, req: GenerateRequest, res: Response): GenerateOptions {
         return Sentry.startSpan({
             op: "generate_getAndValidateOptions",
@@ -264,11 +277,7 @@ export const register = (app: Application) => {
 
             const checkOnly = !!(req.body["checkOnly"] || req.query["checkOnly"])
 
-            const breadcrumbId = md5(`${ getIp(req) }${ Date.now() }${ variant }${ visibility }${ Math.random() }${ name }`).substr(0, 8);
-            const breadcrumb = nextBreadColor()(breadcrumbId);
-            req.breadcrumb = breadcrumb;
-            res.header("X-MineSkin-Breadcrumb", breadcrumbId);
-            res.header("X-MineSkin-Timestamp", `${ Date.now() }`);
+            const breadcrumb = req.breadcrumb;
 
             console.log(debug(`${ breadcrumb } Type:        ${ type }`))
             console.log(debug(`${ breadcrumb } Variant:     ${ variant }`));
@@ -284,7 +293,6 @@ export const register = (app: Application) => {
                 "generate_variant": variant,
                 "generate_visibility": visibility
             });
-            Sentry.setExtra("generate_breadcrumb", breadcrumbId);
 
             return {
                 model,