Mar 2023
Video to canvas
Here I explore drawing frames of video to canvas, which exposes more possibilities of manipulating the pixel data to apply cool effects on.
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
<meta charset="utf-8">
<style>
* { box-sizing: border-box; }
html, body { height: 100%; margin: 0; overflow: hidden; font-family: system-ui; }
#root { width: 100%; height: 100%; overflow: hidden; position: relative; display: flex; align-items: center; justify-content: center; }
canvas { position: absolute; left: 0; top: 0; background: #000; width: 100%; height: 100%; object-fit: contain; }
.start { cursor: pointer; position: absolute; top: 0; right: 0; bottom: 0; left: 0; background: #000; display: flex; align-items: center; justify-content: center; color: #fff; font-weight: bold; }
.action { position: absolute; left: 30px; bottom: 30px; z-index: 1; user-select: none; }
</style>
<script>
const createNode = (options) => {
const node = document.createElement(options.tag);
if(options.className) node.setAttribute('class',options.className);
if(options.innerHTML) node.innerHTML = options.innerHTML;
if(options.attributes) Object.keys(options.attributes).forEach(key => node.setAttribute(key,options.attributes[key]) );
if(options.style) Object.keys(options.style).forEach(key => node.style[key] = options.style[key]);
if(options.event_listeners) Object.keys(options.event_listeners).forEach(key => node.addEventListener(key,options.event_listeners[key]) )
if(options.root) options.root.appendChild(node);
return node;
}
class App {
constructor(){
document.addEventListener('DOMContentLoaded',this.handleReady.bind(this));
}
handleReady(){
if(this.interval) cancelAnimationFrame(this.interval);
this.els = {};
this.video_playing = false;
this.ready = false;
this.effect_on = false;
this.els.root = document.getElementById('root');
this.els.root.innerHTML = ``;
this.video = createNode({ tag: 'video', attributes: { src: `geisha.mp4`, loop: 'true', playsinline: 'true' } })
this.els.canvas = createNode({ root: this.els.root, tag: 'canvas' });
this.els.canvas.addEventListener('click', event => {
if(this.video_playing) this.video.pause();
else this.video.play();
});
this.context = this.els.canvas.getContext('2d');
this.els.start = createNode({ root: this.els.root, tag: 'div', className: 'start' })
this.els.start_button = createNode({ root: this.els.start, tag: 'button', innerHTML: 'Click to load video' })
this.els.start_button.addEventListener('click',this.handleClickStart.bind(this));
// this.els.start.addEventListener('touchstart',this.handleClickStart.bind(this));
this.video.addEventListener('loadedmetadata', async event => {
this.els.canvas.width = this.video.videoWidth;
this.els.canvas.height = this.video.videoHeight;
this.els.action = await createNode({ root: this.els.root, tag: 'button', className: 'action', innerHTML: 'Toggle effect', style: { display: `none` } })
this.els.action.addEventListener('click', event => {
this.effect_on = !this.effect_on;
});
this.interval = window.requestAnimationFrame(this.render.bind(this));
})
}
async handleClickStart(event){
await this.video.play();
this.els.start.style.display = `none`;
this.els.action.style.display = `block`;
}
rgbSplit(imageData, options) {
// https://hangindev.com/blog/rgb-splitting-effect-with-html5-canvas-and-javascript
const { rOffset = 0, gOffset = 0, bOffset = 0 } = options;
const originalArray = imageData.data;
const newPixels = new Uint8ClampedArray(originalArray);
for (let i = 0; i < originalArray.length; i += 4) {
newPixels[i + 0 + rOffset * 4] = originalArray[i + 0]; // 🔴
newPixels[i + 1 + gOffset * 4] = originalArray[i + 1]; // 🟢
newPixels[i + 2 + bOffset * 4] = originalArray[i + 2]; // 🔵
}
return new ImageData(newPixels, imageData.width, imageData.height);
}
drawVideo(){
this.context.drawImage(this.video, 0, 0, this.video.videoWidth, this.video.videoHeight);
if(this.effect_on){
const imageData = this.context.getImageData(0, 0, this.video.videoWidth, this.video.videoHeight);
const updatedImageData = this.rgbSplit(imageData, { rOffset: 20, gOffset: -10, bOffset: 10 });
this.context.putImageData(updatedImageData, 0, 0);
}
}
render(){
this.video_playing = !!(this.video && this.video.currentTime > 0 && !this.video.paused && !this.video.ended && this.video.readyState > 2);
if(this.video_playing){
this.drawVideo();
}
this.interval = window.requestAnimationFrame(this.render.bind(this));
}
}
const app = new App();
</script>
</head>
<body>
<div id="root"></div>
</body>
</html>
To demonstrate, I'll use some raw footage of the geisha from Bladerunner I converted into an mp4 file. Click the document to initiate the vide playback. Click the document again to pause/resume. Toggle the effect on/off with the button.
In this case, I took some time to find a function which splits the RGB values of image pixel data. There are a myriad of complex solutions for image manipulation, and I was quite surprised at how easy and fast this solution turned out to be.
Using requestAnimationFrame, I draw each frame of the video, get its pixel data, apply the filter, and redraw to the canvas.
Project Assets
| Asset | File | Description |
|---|---|---|
geisha.mp4 |
Download | Video file of the smoking geisha from Bladerunner |