import * as http from 'http'; import * as url from 'url'; import * as path from 'path'; import { chromium } from 'playwright'; import { locks } from 'web-locks'; import crypto from 'crypto'; import fs from 'fs'; import { setTimeout } from 'timers/promises'; import { WebSocketServer } from 'ws'; import UserAgent from 'user-agents'; import { v4 as uuidv4 } from 'uuid'; // npm i playwright // npm i web-locks // npm i ws // npm i user-agents // npm i uuid // nohup cloudflared tunnel --url http://localhost:8080 --no-autoupdate & (or setsid) // setsid node playwright.mjs (nohup doesnt work) // curl 'https://gowah44030-playwright.hf.space/.. // https://gowah44030-playwright.hf.space/screenshot?js=1&url=https://www.investing.com globalThis.state = Object.assign(globalThis.state||{}, { browser: null, context: null, }); async function text_from_url(url, cookie) { let { browser, context } = globalThis.state; if (!browser) { await locks.request('playwright_browser', async lock => { browser = globalThis.state.browser = await chromium.launch(); context = globalThis.state.context = await browser.newContext({ javaScriptEnabled: false }/*devices['iPhone 11']*/); }); } const page = await context.newPage(); if (cookie) { if (cookie.endsWith(';')) cookie = cookie.slice(0, -1); let cookies = cookie.split(';'); cookies = cookies.map(it=>{ let [name, value] = it.split('='); return {name: name.trim(), value: value.trim(), url}; }); context.addCookies(cookies); } await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort()); await page.goto(url); let text; let i = 0; while (true) { let new_text = await page.evaluate(() => document.body.innerText); if (i > 5 || new_text?.length > 200) { text = new_text; break; } i++; await new Promise(resolve=>setTimeout(resolve, 1000)); } await page.close(); //await context.close(); // await browser.close(); return text; } async function screenshot(url, cookie, js_enabled) { let { browser, context } = globalThis.state; if (!browser) { console.log('launch browser'); await locks.request('playwright_browser', async lock => { browser = globalThis.state.browser = await chromium.launch(); context = globalThis.state.context = await browser.newContext({ javaScriptEnabled: js_enabled }/*devices['iPhone 11']*/); }); } const page = await context.newPage(); if (cookie) { if (cookie.endsWith(';')) cookie = cookie.slice(0, -1); let cookies = cookie.split(';'); cookies = cookies.map(it=>{ let [name, value] = it.split('='); return {name: name.trim(), value: value.trim(), url}; }); context.addCookies(cookies); } //await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort()); await page.goto(url); await setTimeout(2000); // let id = crypto.randomUUID(); // let path = `/code/${id}.png`; // await page.screenshot({ path, fullPage: true }); const buffer = await page.screenshot({fullPage: true}); await page.close(); //await context.close(); // await browser.close(); return buffer; } async function evaluate(url, code) { let { browser, context } = globalThis.state; if (!browser) { console.log('launch browser'); await locks.request('playwright_browser', async lock => { browser = globalThis.state.browser = await chromium.launch({ args: ['--disable-web-security'] //https://github.com/microsoft/playwright/issues/17631 }); context = globalThis.state.context = await browser.newContext({ javaScriptEnabled: true, }/*devices['iPhone 11']*/); }); } const page = await context.newPage(); page.on('console', async (msg) => {console.log(msg);}); //await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort()); await page.goto(url); let result = await page.evaluate(code); await page.close(); return result; } async function test_bot(url) { // https://gist.github.com/nicoandmee/1ec1b6a07c94f82df41d2496194ef3a6 // https://gowah44030-playwright.hf.space/bot?url=https://nowsecue.nl const UINT32_MAX = (2 ** 32) - 1; const WEBGL_RENDERERS = ['ANGLE (NVIDIA Quadro 2000M Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (NVIDIA Quadro K420 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro 2000M Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro K2000M Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 3800 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 4000 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (Intel(R) HD Graphics 4000 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (AMD Radeon R9 200 Series Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 4000 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 3000 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 4 Series Express Chipset Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) Graphics Media Accelerator 3150 Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) G41 Express Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 6150SE nForce 430 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 4000)', 'ANGLE (Mobile Intel(R) 965 Express Chipset Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family)', 'ANGLE (NVIDIA GeForce GTX 760 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (NVIDIA GeForce GTX 760 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (NVIDIA GeForce GTX 760 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (AMD Radeon HD 6310 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Graphics Media Accelerator 3600 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family Direct3D9 vs_0_0 ps_2_0)', 'ANGLE (AMD Radeon HD 6320 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family (Microsoft Corporation - WDDM 1.0) Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) G41 Express Chipset)', 'ANGLE (ATI Mobility Radeon HD 5470 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q45/Q43 Express Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 310M Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G41 Express Chipset Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 45 Express Chipset Family (Microsoft Corporation - WDDM 1.1) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 440 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 4300/4500 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7310 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics)', 'ANGLE (Intel(R) 4 Series Internal Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon(TM) HD 6480G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 3200 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7800 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G41 Express Chipset (Microsoft Corporation - WDDM 1.1) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 210 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 630 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7340 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) 82945G Express Chipset Family Direct3D9 vs_0_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GT 430 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 7025 / NVIDIA nForce 630a Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q35 Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) HD Graphics 4600 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7520G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD 760G (Microsoft Corporation WDDM 1.1) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 220 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9500 GT Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Graphics Media Accelerator HD Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9800 GT Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q965/Q963 Express Chipset Family (Microsoft Corporation - WDDM 1.0) Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GTX 550 Ti Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q965/Q963 Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (AMD M880G with ATI Mobility Radeon HD 4250 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GTX 650 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Mobility Radeon HD 5650 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 4200 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7700 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family)', 'ANGLE (Intel(R) 82945G Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (SiS Mirage 3 Graphics Direct3D9Ex vs_2_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GT 430)', 'ANGLE (AMD RADEON HD 6450 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon 3000 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) 4 Series Internal Chipset Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q35 Express Chipset Family (Microsoft Corporation - WDDM 1.0) Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GT 220 Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7640G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD 760G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 6450 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 640 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9200 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 610 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 6290 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Mobility Radeon HD 4250 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 8600 GT Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 5570 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 6800 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G45/G43 Express Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 4600 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro NVS 160M Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 3000)', 'ANGLE (NVIDIA GeForce G100)', 'ANGLE (AMD Radeon HD 8610G + 8500M Dual Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 4 Series Express Chipset Family Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 7025 / NVIDIA nForce 630a (Microsoft Corporation - WDDM) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q965/Q963 Express Chipset Family Direct3D9 vs_0_0 ps_2_0)', 'ANGLE (AMD RADEON HD 6350 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 5450 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9500 GT)', 'ANGLE (AMD Radeon HD 6500M/5600/5700 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 965 Express Chipset Family)', 'ANGLE (NVIDIA GeForce 8400 GS Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GTX 560 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 620 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GTX 660 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon(TM) HD 6520G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 240 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 8240 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro NVS 140M)', 'ANGLE (Intel(R) Q35 Express Chipset Family Direct3D9 vs_0_0 ps_2_0)']; function getBrowserfingerprint(buid, emulateFlag = 'desktop') { const generateUserAgent = new UserAgent({ deviceCategory: emulateFlag, }); const fingerprints = Array(1000).fill().map(() => generateUserAgent()); const WEBGL_PARAMETER = { WEBGL_VENDOR: 'Google Inc.', WEBGL_RENDERER: WEBGL_RENDERERS[Math.floor(Math.random() * WEBGL_RENDERERS.length)], }; const fingerprint = Object.assign(fingerprints[Math.floor(Math.random() * fingerprints.length)].data, WEBGL_PARAMETER); const buidHash = crypto.createHash('sha512').update(buid).digest(); fingerprint.BUID = buidHash.toString('base64'); fingerprint.random = (index) => { const idx = index % 124; if (idx < 62) return buidHash.readUInt32BE(idx) / UINT32_MAX; return buidHash.readUInt32LE(idx - 62) / UINT32_MAX; }; return fingerprint; } async function cloak(page, fingerprint, { minWidth = 1280, minHeight = 1024, } = {}) { console.debug(`fingerprint-webgl-vendor-${fingerprint.WEBGL_VENDOR}`); console.debug(`fingerprint-webgl-renderer-${fingerprint.WEBGL_RENDERER}`); console.debug(`fingerprint-ua-ua-${fingerprint.userAgent}`); console.debug(`fingerprint-ua-platform-${fingerprint.platform}`); console.debug(`fingerprint-deviceCategory-${fingerprint.deviceCategory}`); console.debug(`fingerprint-viewportHeight-${fingerprint.viewportHeight}`); console.debug(`fingerprint-viewportWidth-${fingerprint.viewportWidth}`); const LOG_OVERRIDE = true; if (LOG_OVERRIDE) { await page.on('console', (msg) => { if (msg && msg.text) { if (typeof msg.text === 'function') { debugConsole('PAGE LOG:', msg.text()); } else { debugConsole('PAGE LOG:', msg.text); } } else { debugConsole('PAGE LOG:', msg); } }); await page.on('pageerror', (err) => debug('PAGE ERR:', err)); } const DIMENSION = { isLandscape: true, width: minWidth > fingerprint.viewportWidth ? minWidth : (parseInt(minWidth + (fingerprint.random(0) * (fingerprint.screenWidth - minWidth)), 10)), height: minHeight > fingerprint.viewportHeight ? minHeight : (parseInt(minHeight + (fingerprint.random(1) * (fingerprint.screenHeight - minHeight)), 10)), }; await page.addInitScript(async (fingerprint, LO, D) => { const logOverride = (key, value) => { if (!LO) return value; console.warn(`Overriden: ${key}=${value}`); return value; }; function buildPlugin(spec) { const plugin = spec; plugin.length = spec.mimeTypes.length; spec.mimeTypes.forEach((m, i) => { plugin[i] = m; Object.assign(m, { enabledPlugin: plugin, }); }); delete spec.mimeTypes; return plugin; } const plugins = { length: 4, 0: buildPlugin({ mimeTypes: [{ type: 'application/x-google-chrome-pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: true, }], name: 'Chrome PDF Plugin', description: 'Portable Document Format', filename: 'internal-pdf-viewer', }), 1: buildPlugin({ mimeTypes: [{ type: 'application/pdf', suffixes: 'pdf', description: '', extensions: 'pdf', enabledPlugin: true, }], name: 'Chrome PDF Viewer', description: '', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', }), 2: buildPlugin({ mimeTypes: [{ type: 'application/x-nacl', suffixes: '', description: 'Native Client Executable', enabledPlugin: true, }, { type: 'application/x-pnacl', suffixes: '', description: 'Portable Native Client Executable', enabledPlugin: true, }, { type: 'text/html', suffixes: '', description: '', enabledPlugin: true, }, { type: 'application/x-ppapi-vysor', suffixes: '', description: '', enabledPlugin: true, }, { type: 'application/x-ppapi-vysor-audio', suffixes: '', description: '', enabledPlugin: true, }, ], name: 'Native Client', description: '', filename: fingerprint.platform === 'Win32' ? 'pepflashplayer.dll' : 'internal-nacl-plugin', }), 3: buildPlugin({ mimeTypes: [{ type: 'application/x-ppapi-widevine-cdm', suffixes: '', description: 'Widevine Content Decryption Module', enabledPlugin: true, }], name: 'Widevine Content Decryption Module', description: 'Enables Widevine licenses for playback of HTML audio/video content. (version: 1.4.9.1070)', filename: fingerprint.platform === 'Win32' ? 'widevinecdmadapter.dll' : 'widevinecdmadapter.plugin', }), }; window.screen.__defineGetter__('width', () => logOverride('width', fingerprint.screenWidth)); window.screen.__defineGetter__('availWidth', () => logOverride('availWidth', fingerprint.screenWidth)); window.__defineGetter__('innerWidth', () => logOverride('innerWidth', D.width)); window.__defineGetter__('outerWidth', () => logOverride('outerWidth', D.width)); window.screen.__defineGetter__('height', () => logOverride('height', fingerprint.screenHeight)); window.screen.__defineGetter__('availHeight', () => logOverride('availHeight', fingerprint.screenHeight)); window.__defineGetter__('innerHeight', () => logOverride('innerHeight', D.height - 74)); window.__defineGetter__('outerHeight', () => logOverride('outerHeight', D.height)); window.navigator.__defineGetter__('userAgent', () => logOverride('userAgent', fingerprint.userAgent)); window.navigator.__defineGetter__('platform', () => logOverride('platform', fingerprint.platform)); window.navigator.__defineGetter__('appName', () => logOverride('appName', fingerprint.appName)); window.navigator.__defineGetter__('appVersion', () => logOverride('appVersion', fingerprint.userAgent.substring(fingerprint.userAgent.indexOf('/') + 1, fingerprint.userAgent.length))); const newProto = navigator.__proto__; delete newProto.webdriver; navigator.__proto__ = newProto window.navigator.__defineGetter__('languages', () => logOverride('languages', ['en-US,en'])); window.navigator.__defineGetter__('getUserMedia', () => logOverride('getUserMedia', undefined)); window.navigator.__defineGetter__('webkitGetUserMedia', () => logOverride('webkitGetUserMedia', undefined)); // reject webRTC fingerprinting window.__defineGetter__('MediaStreamTrack', () => logOverride('MediaStreamTrack', undefined)); window.__defineGetter__('RTCPeerConnection', () => logOverride('RTCPeerConnection', undefined)); window.__defineGetter__('RTCSessionDescription', () => logOverride('RTCSessionDescription', undefined)); window.__defineGetter__('webkitMediaStreamTrack', () => logOverride('webkitMediaStreamTrack', undefined)); window.__defineGetter__('webkitRTCPeerConnection', () => logOverride('webkitRTCPeerConnection', undefined)); window.__defineGetter__('webkitRTCSessionDescription', () => logOverride('webkitRTCSessionDescription', undefined)); window.navigator.__defineGetter__('getUserMedia', () => logOverride('getUserMedia', undefined)); window.navigator.__defineGetter__('webkitGetUserMedia', () => logOverride('webkitGetUserMedia', undefined)); window.navigator.__defineGetter__('plugins', () => logOverride('plugins', plugins)); // handle canvas class WebGLRenderingContext { constructor(cvs) { this.extension = { UNMASKED_VENDOR_WEBGL: 37445, UNMASKED_RENDERER_WEBGL: 37446, }; this.canvas = cvs; this.parameter = ''; this.viewportWidth = cvs.width; this.viewportHeight = cvs.height; this.supportedExtensions = ['ANGLE_instanced_arrays', 'EXT_blend_minmax', 'EXT_color_buffer_half_float', 'EXT_frag_depth', 'EXT_shader_texture_lod', 'EXT_texture_filter_anisotropic', 'WEBKIT_EXT_texture_filter_anisotropic', 'EXT_sRGB', 'OES_element_index_uint', 'OES_standard_derivatives', 'OES_texture_float', 'OES_texture_float_linear', 'OES_texture_half_float', 'OES_texture_half_float_linear', 'OES_vertex_array_object', 'WEBGL_color_buffer_float', 'WEBGL_compressed_texture_s3tc', 'WEBKIT_WEBGL_compressed_texture_s3tc', 'WEBGL_compressed_texture_s3tc_srgb', 'WEBGL_debug_renderer_info', 'WEBGL_debug_shaders', 'WEBGL_depth_texture', 'WEBKIT_WEBGL_depth_texture', 'WEBGL_draw_buffers', 'WEBGL_lose_context', 'WEBKIT_WEBGL_lose_context']; } getExtension() { return this.extension; } getParameter() { return this.extension; } getSupportedExtensions() { return this.supportedExtensions; } } const canvas = document.createElement('canvas'); const canvasProto = Object.getPrototypeOf(canvas); const origGetContext = canvasProto.getContext; canvasProto.getContext = function getContext(...args) { const context = origGetContext && (origGetContext.call(this, ...args) || origGetContext.call(this, args[0])); if (!context) { logOverride('canvas.getContext', 'new WebGLRenderingContext()'); return new WebGLRenderingContext(this); } return context; }; canvasProto.getContext.toString = canvasProto.getContext.toString(); function hookPrototypeMethods(prefix, object) { if (!object) return; const originals = {}; const prototype = Object.getPrototypeOf(object); Object .getOwnPropertyNames(prototype) .filter((n) => { try { return typeof prototype[n] === 'function'; } catch (error) { return false; } }) .forEach((n) => { originals[n] = prototype[n]; prototype[n] = function fn(...args) { if (prefix === '2d' && (n === 'strokeText' || n === 'fillText')) { const temp = Array.from(args); temp[0] = fingerprint.BUID; temp[1] = Math.max(0, temp[1] - 2); temp[2] = Math.max(0, temp[2] - 2); originals[n].call(this, ...temp); } const result = originals[n].call(this, ...args); if (LO) { let jsonResult; try { jsonResult = JSON.stringify(result); } catch (e) {} console.warn('function called', prefix, n, JSON.stringify(args), 'result:', result, jsonResult, `${result}`); } return result; }; }); } const gls = []; try { gls.push(document.createElement('canvas').getContext('webgl')); gls.push(document.createElement('canvas').getContext('experimental-webgl')); } catch (e) {} gls.forEach((gl) => { const glProto = Object.getPrototypeOf(gl); const origGetParameter = glProto.getParameter; const debugInfo = gl.getExtension('WEBGL_debug_renderer_info'); if (gl) { glProto.getParameter = function getParameter(...args) { if (args[0] === debugInfo.UNMASKED_VENDOR_WEBGL) return logOverride('gl.getParameter.UNMASKED_VENDOR_WEBGL', fingerprint.WEBGL_VENDOR); if (args[0] === debugInfo.UNMASKED_RENDERER_WEBGL) return logOverride('gl.getParameter.UNMASKED_RENDERER_WEBGL', fingerprint.WEBGL_RENDERER); if (args[0] === 33901) return new Float32Array([1, 8191]); if (args[0] === 3386) return new Int32Array([16384, 16384]); if (args[0] === 35661) return 80; if (args[0] === 34076) return 16384; if (args[0] === 36349) return 1024; if (args[0] === 34024) return 16384; if (args[0] === 3379) return 16384; if (args[0] === 34921) return 16; if (args[0] === 36347) return 1024; return origGetParameter.call(this, ...args); }; } }); hookPrototypeMethods('webgl', document.createElement('canvas').getContext('webgl')); hookPrototypeMethods('experimental-webgl', document.createElement('canvas').getContext('experimental-webgl')); hookPrototypeMethods('2d', document.createElement('canvas').getContext('2d')); hookPrototypeMethods('canvas', canvas); hookPrototypeMethods('screen', window.screen); hookPrototypeMethods('navigator', window.navigator); hookPrototypeMethods('history', window.history); }, fingerprint, LOG_OVERRIDE, DIMENSION); // emulate permissions await page.addInitScript(() => { const originalQuery = window.navigator.permissions.query; window.navigator.permissions.__proto__.query = (parameters) => (parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters)); const oldCall = Function.prototype.call; function call() { return oldCall.apply(this, arguments); } Function.prototype.call = call; const nativeToStringFunctionString = Error.toString().replace( /Error/g, 'toString', ); const oldToString = Function.prototype.toString; function functionToString() { if (this === window.navigator.permissions.query) { return 'function query() { [native code] }'; } if (this === functionToString) { return nativeToStringFunctionString; } return oldCall.call(oldToString, this); } Function.prototype.toString = functionToString; }); // must to hook getters await page.goto('about:blank'); await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', }); await page.setViewport(DIMENSION); } const args = [ '--no-sandbox', '--remote-debugging-port=9222', '--disable-setuid-sandbox', '--ignore-certificate-errors', '--disk-cache-size=1', '--disable-infobars', '--disable-blink-features=AutomationControlled' ]; console.log('launchPersistentContext', url); const browser = await chromium.launchPersistentContext('', { headless: false, bypassCSP: true, ignoreHTTPSErrors: true, args: args, timezoneId: 'America/Los_Angeles', }); // generate a fingerprint const fingerprint = await getBrowserfingerprint(uuidv4()); let page = await browser.newPage(); // "cloak" our page with evasions await cloak(page, fingerprint); console.log('goto', url); //await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort()); await page.goto(url); console.log('goto end'); await setTimeout(2000); // let id = crypto.randomUUID(); // let path = `/code/${id}.png`; // await page.screenshot({ path, fullPage: true }); const buffer = await page.screenshot({fullPage: true}); await page.close(); await browser.close(); return buffer; } const server = http.createServer(async function(req, res) { const {query, pathname} = url.parse(req.url, true); console.log(req.url, new Date()); res.setHeader('Access-Control-Allow-Origin', '*') let _url = query.url; let cookie = query.cookie; let js_enabled = query.js; try { if (pathname == '/text') { let text = await text_from_url(_url, cookie); res.end(text); } else if (pathname == '/screenshot') { let buffer = await screenshot(_url, cookie, js_enabled); res.writeHead(200,{'Content-type':'image/png'}); res.end(buffer); // const readStream = fs.createReadStream(path); // res.writeHead(200,{'Content-type':'image/png'}); // readStream.pipe(res); // res.end(); } else if (pathname == '/bot') { let buffer = await test_bot(_url); res.writeHead(200,{'Content-type':'image/png'}); res.end(buffer); } else if (pathname == '/evaluate') { const buffers = []; for await (const chunk of req) { buffers.push(chunk); } const data = Buffer.concat(buffers).toString(); const {url, code} = JSON.parse(data); try { let result = await evaluate(url, code); res.end(''+result); } catch (e) { res.end(e.toString()); } } else { res.end('hello'); } } catch (e) { res.end(e.toString()); } }); const wss = new WebSocketServer({ server }); wss.on('connection', function connection(ws, req) { console.log('wss.connection', req.url, req.headers); ws.isAlive = true; ws.on('pong', function heartbeat(){ //'Pong messages are automatically sent in response to ping messages as required by the spec.' //console.log('pong'); this.isAlive = true; }); ws.on('message', function message(data) { console.log('received: %s', data); }); ws.addEventListener('close', function close() { console.log('ws.close'); }); //ws.send('something'); }); // https://github.com/websockets/ws const interval = setInterval(function ping() { wss.clients.forEach(function each(ws) { if (ws.isAlive === false) return ws.terminate(); ws.isAlive = false; ws.ping(); ws.send('ping');//why: used by client to detect dead connection (because browser cant access ping/pong frames...(?)) }); }, 30000); server.listen(7860);