You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
377 lines
9.5 KiB
377 lines
9.5 KiB
2 years ago
|
import puppeteer from 'puppeteer';
|
||
|
|
import handler from 'serve-handler';
|
||
|
|
import http from 'http';
|
||
|
|
import pixelmatch from 'pixelmatch';
|
||
|
|
import jimp from 'jimp';
|
||
|
|
import fs from 'fs';
|
||
|
|
|
||
|
|
const port = 1234;
|
||
|
|
const pixelThreshold = 0.1; // threshold error in one pixel
|
||
|
|
const maxFailedPixels = 0.05; // total failed pixels
|
||
|
|
|
||
|
|
const networkTimeout = 600;
|
||
|
|
const networkTax = 2000; // additional timeout for resources size
|
||
|
|
const pageSizeMinTax = 1.0; // in mb, when networkTax = 0
|
||
|
|
const pageSizeMaxTax = 5.0; // in mb, when networkTax = networkTax
|
||
|
|
const renderTimeout = 1200;
|
||
|
|
const maxAttemptId = 3; // progresseve attempts
|
||
|
|
const progressFunc = n => 1 + n;
|
||
|
|
|
||
|
|
const width = 400;
|
||
|
|
const height = 250;
|
||
|
|
const viewScale = 2;
|
||
|
|
const jpgQuality = 95;
|
||
|
|
|
||
|
|
const exceptionList = [
|
||
|
|
|
||
|
|
'index',
|
||
|
|
'css3d_youtube', // video tag not deterministic enough
|
||
|
|
'webaudio_visualizer', // audio can't be analyzed without proper audio hook
|
||
|
|
'webgl_effects_ascii', // blink renders text differently in every platform
|
||
|
|
'webgl_loader_imagebitmap', // takes too long to load?
|
||
|
|
'webgl_loader_texture_lottie', // not sure why this fails
|
||
|
|
'webgl_loader_texture_pvrtc', // not supported in CI, useless
|
||
|
|
'webgl_morphtargets_face', // To investigate...
|
||
|
|
'webgl_nodes_materials_standard', // puppeteer does not support import maps yet
|
||
|
|
'webgl_postprocessing_crossfade', // fails for some misterious reason
|
||
|
|
'webgl_raymarching_reflect', // exception for Github Actions
|
||
|
|
'webgl_renderer_pathtracer', // slow to render
|
||
|
|
'webgl_test_memory2', // gives fatal error in puppeteer
|
||
|
|
'webgl_tiled_forward', // exception for Github Actions
|
||
|
|
'webgl_video_kinect', // video tag not deterministic enough
|
||
|
|
'webgl_video_panorama_equirectangular', // video tag not deterministic enough?
|
||
|
|
'webgl_worker_offscreencanvas', // in a worker, not robust
|
||
|
|
// webxr
|
||
|
|
'webxr_ar_lighting',
|
||
|
|
// webgpu
|
||
|
|
'webgpu_audio_processing',
|
||
|
|
'webgpu_compute',
|
||
|
|
'webgpu_cubemap_adjustments',
|
||
|
|
'webgpu_cubemap_mix',
|
||
|
|
'webgpu_depth_texture',
|
||
|
|
'webgpu_equirectangular',
|
||
|
|
'webgpu_instance_mesh',
|
||
|
|
'webgpu_instance_uniform',
|
||
|
|
'webgpu_lights_custom',
|
||
|
|
'webgpu_lights_selective',
|
||
|
|
'webgpu_loader_gltf',
|
||
|
|
'webgpu_materials',
|
||
|
|
'webgpu_nodes_playground',
|
||
|
|
'webgpu_particles',
|
||
|
|
'webgpu_rtt',
|
||
|
|
'webgpu_sandbox',
|
||
|
|
'webgpu_skinning_instancing',
|
||
|
|
'webgpu_skinning_points',
|
||
|
|
'webgpu_skinning',
|
||
|
|
'webgpu_sprites'
|
||
|
|
].concat( ( process.platform === 'win32' ) ? [
|
||
|
|
|
||
|
|
'webgl_effects_ascii' // windows fonts not supported
|
||
|
|
|
||
|
|
] : [] );
|
||
|
|
|
||
|
|
console.green = ( msg ) => console.log( `\x1b[32m${ msg }\x1b[37m` );
|
||
|
|
console.red = ( msg ) => console.log( `\x1b[31m${ msg }\x1b[37m` );
|
||
|
|
console.null = () => {};
|
||
|
|
|
||
|
|
|
||
|
|
/* Launch server */
|
||
|
|
|
||
|
|
const server = http.createServer( ( req, resp ) => handler( req, resp ) );
|
||
|
|
server.listen( port, async () => await pup );
|
||
|
|
server.on( 'SIGINT', () => process.exit( 1 ) );
|
||
|
|
|
||
|
|
|
||
|
|
/* Launch browser */
|
||
|
|
|
||
|
|
const pup = puppeteer.launch( {
|
||
|
|
headless: ! process.env.VISIBLE,
|
||
|
|
args: [
|
||
|
|
'--use-gl=swiftshader',
|
||
|
|
'--no-sandbox',
|
||
|
|
'--enable-surface-synchronization'
|
||
|
|
]
|
||
|
|
} ).then( async browser => {
|
||
|
|
|
||
|
|
|
||
|
|
/* Prepare page */
|
||
|
|
|
||
|
|
const page = ( await browser.pages() )[ 0 ];
|
||
|
|
await page.setViewport( { width: width * viewScale, height: height * viewScale } );
|
||
|
|
|
||
|
|
const cleanPage = fs.readFileSync( 'test/e2e/clean-page.js', 'utf8' );
|
||
|
|
const injection = fs.readFileSync( 'test/e2e/deterministic-injection.js', 'utf8' );
|
||
|
|
await page.evaluateOnNewDocument( injection );
|
||
|
|
|
||
|
|
const threeJsBuild = fs.readFileSync( 'build/three.module.js', 'utf8' )
|
||
|
|
.replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' );
|
||
|
|
await page.setRequestInterception( true );
|
||
|
|
|
||
|
|
page.on( 'console', msg => ( msg.text().slice( 0, 8 ) === 'Warning.' ) ? console.null( msg.text() ) : {} );
|
||
|
|
page.on( 'request', async ( request ) => {
|
||
|
|
|
||
|
|
if ( request.url() === 'http://localhost:1234/build/three.module.js' ) {
|
||
|
|
|
||
|
|
await request.respond( {
|
||
|
|
status: 200,
|
||
|
|
contentType: 'application/javascript; charset=utf-8',
|
||
|
|
body: threeJsBuild
|
||
|
|
} );
|
||
|
|
|
||
|
|
} else {
|
||
|
|
|
||
|
|
await request.continue();
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
} );
|
||
|
|
page.on( 'response', async ( response ) => {
|
||
|
|
|
||
|
|
try {
|
||
|
|
|
||
|
|
await response.buffer().then( buffer => pageSize += buffer.length );
|
||
|
|
|
||
|
|
} catch ( e ) {
|
||
|
|
|
||
|
|
console.null( `Warning. Wrong request. \n${ e }` );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
} );
|
||
|
|
|
||
|
|
|
||
|
|
/* Find files */
|
||
|
|
|
||
|
|
const isMakeScreenshot = process.argv[ 2 ] == '--make';
|
||
|
|
const isExactList = process.argv.length > ( 2 + isMakeScreenshot );
|
||
|
|
|
||
|
|
const exactList = process.argv.slice( isMakeScreenshot ? 3 : 2 )
|
||
|
|
.map( f => f.replace( '.html', '' ) );
|
||
|
|
|
||
|
|
const files = fs.readdirSync( './examples' )
|
||
|
|
.filter( s => s.slice( - 5 ) === '.html' )
|
||
|
|
.map( s => s.slice( 0, s.length - 5 ) )
|
||
|
|
.filter( f => isExactList ? exactList.includes( f ) : ! exceptionList.includes( f ) );
|
||
|
|
|
||
|
|
|
||
|
|
/* Loop for each file, with CI parallelism */
|
||
|
|
|
||
|
|
let pageSize, file, attemptProgress;
|
||
|
|
const failedScreenshots = [];
|
||
|
|
|
||
|
|
let beginId = 0;
|
||
|
|
let endId = files.length;
|
||
|
|
|
||
|
|
if ( 'CI' in process.env ) {
|
||
|
|
|
||
|
|
const jobs = 8;
|
||
|
|
|
||
|
|
beginId = Math.floor( parseInt( process.env.CI.slice( 0, 1 ) ) * files.length / jobs );
|
||
|
|
endId = Math.floor( ( parseInt( process.env.CI.slice( - 1 ) ) + 1 ) * files.length / jobs );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
for ( let id = beginId; id < endId; ++ id ) {
|
||
|
|
|
||
|
|
/* At least 3 attempts before fail */
|
||
|
|
|
||
|
|
let attemptId = isMakeScreenshot ? 1.5 : 0;
|
||
|
|
|
||
|
|
while ( attemptId < maxAttemptId ) {
|
||
|
|
|
||
|
|
/* Load target page */
|
||
|
|
|
||
|
|
file = files[ id ];
|
||
|
|
attemptProgress = progressFunc( attemptId );
|
||
|
|
pageSize = 0;
|
||
|
|
|
||
|
|
try {
|
||
|
|
|
||
|
|
await page.goto( `http://localhost:${ port }/examples/${ file }.html`, {
|
||
|
|
waitUntil: 'networkidle2',
|
||
|
|
timeout: networkTimeout * attemptProgress
|
||
|
|
} );
|
||
|
|
|
||
|
|
} catch {
|
||
|
|
|
||
|
|
console.null( 'Warning. Network timeout exceeded...' );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
|
||
|
|
/* Render page */
|
||
|
|
|
||
|
|
await page.evaluate( cleanPage );
|
||
|
|
|
||
|
|
await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ) => {
|
||
|
|
|
||
|
|
|
||
|
|
/* Resource timeout */
|
||
|
|
|
||
|
|
const resourcesSize = Math.min( 1, ( pageSize / 1024 / 1024 - pageSizeMinTax ) / pageSizeMaxTax );
|
||
|
|
await new Promise( resolve => setTimeout( resolve, networkTax * resourcesSize * attemptProgress ) );
|
||
|
|
|
||
|
|
|
||
|
|
/* Resolve render promise */
|
||
|
|
|
||
|
|
window._renderStarted = true;
|
||
|
|
|
||
|
|
await new Promise( function ( resolve ) {
|
||
|
|
|
||
|
|
performance._now = performance._now || performance.now;
|
||
|
|
|
||
|
|
const renderStart = performance._now();
|
||
|
|
|
||
|
|
const waitingLoop = setInterval( function () {
|
||
|
|
|
||
|
|
const renderEcceded = ( performance._now() - renderStart > renderTimeout * attemptProgress );
|
||
|
|
if ( window._renderFinished || renderEcceded ) {
|
||
|
|
|
||
|
|
if ( renderEcceded ) {
|
||
|
|
|
||
|
|
console.log( 'Warning. Render timeout exceeded...' );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
clearInterval( waitingLoop );
|
||
|
|
resolve();
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}, 0 );
|
||
|
|
|
||
|
|
} );
|
||
|
|
|
||
|
|
}, pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress );
|
||
|
|
|
||
|
|
} catch ( e ) {
|
||
|
|
|
||
|
|
if ( ++ attemptId === maxAttemptId ) {
|
||
|
|
|
||
|
|
console.red( `Something completely wrong. 'Network timeout' is small for your machine. file: ${ file } \n${ e }` );
|
||
|
|
failedScreenshots.push( file );
|
||
|
|
continue;
|
||
|
|
|
||
|
|
} else {
|
||
|
|
|
||
|
|
console.log( 'Another attempt..' );
|
||
|
|
await new Promise( resolve => setTimeout( resolve, networkTimeout * attemptProgress ) );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
if ( isMakeScreenshot ) {
|
||
|
|
|
||
|
|
|
||
|
|
/* Make screenshots */
|
||
|
|
|
||
|
|
attemptId = maxAttemptId;
|
||
|
|
( await jimp.read( await page.screenshot() ) )
|
||
|
|
.scale( 1 / viewScale ).quality( jpgQuality )
|
||
|
|
.write( `./examples/screenshots/${ file }.jpg` );
|
||
|
|
|
||
|
|
console.green( `file: ${ file } generated` );
|
||
|
|
|
||
|
|
|
||
|
|
} else if ( fs.existsSync( `./examples/screenshots/${ file }.jpg` ) ) {
|
||
|
|
|
||
|
|
|
||
|
|
/* Diff screenshots */
|
||
|
|
|
||
|
|
const actual = ( await jimp.read( await page.screenshot() ) ).scale( 1 / viewScale ).quality( jpgQuality ).bitmap;
|
||
|
|
const expected = ( await jimp.read( fs.readFileSync( `./examples/screenshots/${ file }.jpg` ) ) ).bitmap;
|
||
|
|
const diff = actual;
|
||
|
|
|
||
|
|
let numFailedPixels;
|
||
|
|
|
||
|
|
try {
|
||
|
|
|
||
|
|
numFailedPixels = pixelmatch( expected.data, actual.data, diff.data, actual.width, actual.height, {
|
||
|
|
threshold: pixelThreshold,
|
||
|
|
alpha: 0.2,
|
||
|
|
diffMask: process.env.FORCE_COLOR === '0',
|
||
|
|
diffColor: process.env.FORCE_COLOR === '0' ? [ 255, 255, 255 ] : [ 255, 0, 0 ]
|
||
|
|
} );
|
||
|
|
|
||
|
|
} catch {
|
||
|
|
|
||
|
|
attemptId = maxAttemptId;
|
||
|
|
console.red( `Something completely wrong. Image sizes does not match in file: ${ file }` );
|
||
|
|
failedScreenshots.push( file );
|
||
|
|
continue;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
numFailedPixels /= actual.width * actual.height;
|
||
|
|
|
||
|
|
/* Print results */
|
||
|
|
|
||
|
|
if ( numFailedPixels < maxFailedPixels ) {
|
||
|
|
|
||
|
|
attemptId = maxAttemptId;
|
||
|
|
console.green( `diff: ${ numFailedPixels.toFixed( 3 ) }, file: ${ file }` );
|
||
|
|
|
||
|
|
} else {
|
||
|
|
|
||
|
|
if ( ++ attemptId === maxAttemptId ) {
|
||
|
|
|
||
|
|
console.red( `ERROR! Diff wrong in ${ numFailedPixels.toFixed( 3 ) } of pixels in file: ${ file }` );
|
||
|
|
failedScreenshots.push( file );
|
||
|
|
continue;
|
||
|
|
|
||
|
|
} else {
|
||
|
|
|
||
|
|
console.log( 'Another attempt...' );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
} else {
|
||
|
|
|
||
|
|
attemptId = maxAttemptId;
|
||
|
|
console.log( `Warning! Screenshot not exists: ${ file }` );
|
||
|
|
continue;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/* Finish */
|
||
|
|
|
||
|
|
if ( failedScreenshots.length ) {
|
||
|
|
|
||
|
|
if ( failedScreenshots.length > 1 ) {
|
||
|
|
|
||
|
|
console.red( 'List of failed screenshots: ' + failedScreenshots.join( ' ' ) );
|
||
|
|
|
||
|
|
} else {
|
||
|
|
|
||
|
|
console.red( `If you sure that all is right, try to run \`npm run make-screenshot ${ failedScreenshots[ 0 ] }\`` );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ endId - beginId } screenshots not pass.` );
|
||
|
|
|
||
|
|
} else if ( ! isMakeScreenshot ) {
|
||
|
|
|
||
|
|
console.green( `TEST PASSED! ${ endId - beginId } screenshots correctly rendered.` );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
setTimeout( () => {
|
||
|
|
|
||
|
|
server.close();
|
||
|
|
browser.close();
|
||
|
|
process.exit( failedScreenshots.length );
|
||
|
|
|
||
|
|
}, 300 );
|
||
|
|
|
||
|
|
} );
|