Fancy audio visualization

.github/workflows/build.yml

@@ -0,0 +1,35 @@
+name: build gh-page
+run-name: ${{ github.actor }} is building gh page
+on:
+  push:
+    branches: [ "master" ]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        name: Setup node
+        with:
+          node-version: '20'
+          cache: npm
+      - name: Install
+        run: npm ci
+      - run: npm run build
+      - name: Upload Artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: './dist'
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      pages: write
+      id-token: write
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -10,6 +10,7 @@ lerna-debug.log*
 node_modules
 dist
 public
+build
 dist-ssr
 *.local
 

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Alexander Faitelson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,9 @@
+# An Introduction to *Web Worklets* and Audio Visualization
+
+Here you can see [live example](https://treeskar.github.io/audio-visualiazation/) of audio and paint worklets usage.
+
+## All process can be presented in to four steps
+1. We connect to user's media stream.
+2. Transform **digital sound** from media stream **to analytical data** by using Web Audio API. In our case we interested in Sound intensity (Db) over time.
+3. Then we transform **analytical data to visualization data** with D3. It means Db over time to points (x/y) coordinates.
+4. At the end we transform our **points** (visualization data) **to pixels** (render images) with Canvas API.

package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "scripts": {
     "start": "vite",
-    "build": "tsc && vite build",
+    "build": "tsc && vite build --minify false --outDir dist --base \"\" --assetsInlineLimit 0",
     "preview": "vite preview"
   },
   "devDependencies": {

src/main.ts

@@ -1,4 +1,5 @@
-import {SOUND_COLOR, SOUND_INTENSITY} from './utils.ts';
+import {SOUND_COLOR, SOUND_DATA} from './utils.ts';
+import paintWorkletUrl from './paint.ts?worker&url';
 
 async function getAudioAnalyser(mediaStream: MediaStream): Promise<AnalyserNode> {
 	// Audio Web API primary paradigm is of an audio graph, where a number of AudioNodes are connected together to define the overall audio rendering.
@@ -26,8 +27,8 @@ async function getAudioAnalyser(mediaStream: MediaStream): Promise<AnalyserNode>
  */
 function runWithRequestAnimationFrame(callback: () => void): void {
 	requestAnimationFrame(() => {
-		callback();
 		runWithRequestAnimationFrame(callback)
+		callback();
 	});
 }
 
@@ -48,16 +49,16 @@ function registerCSSProperty(name: string, syntax = '*', initialValue?: string):
 	const audioDbData = new Float32Array(audioAnalyser.fftSize);
 
 	// register css properties
-	registerCSSProperty(SOUND_INTENSITY, '*');
 	registerCSSProperty(SOUND_COLOR, '<color>', '#fff');
+	registerCSSProperty(SOUND_DATA, '*');
 
 	// add paint worklet
-	await CSS.paintWorklet.addModule(new URL('./paint.ts', import.meta.url));
+	await CSS.paintWorklet.addModule(paintWorkletUrl);
 
 	runWithRequestAnimationFrame(() => {
 		// audio waveform, is a time domain display of sound amplitude/intensity (decibels dB) in a range -1 to 1.
 		audioAnalyser.getFloatTimeDomainData(audioDbData);
 		// update sound intensity data
-		document.body.style.setProperty(SOUND_INTENSITY, audioDbData.join(','));
+		document.body.style.setProperty(SOUND_DATA, audioDbData.join(','));
 	});
 }().then());

src/paint.ts

@@ -1,17 +1,15 @@
 import * as d3 from 'd3';
-import {SOUND_COLOR, SOUND_INTENSITY} from './utils.ts';
+import {SOUND_COLOR, SOUND_DATA} from './utils.ts';
 
 class SoundPainter implements PaintProcessor {
-	static inputProperties = [SOUND_INTENSITY, SOUND_COLOR];
+	static inputProperties = [SOUND_DATA, SOUND_COLOR];
 
-	private getSoundIntensity(props: StylePropertyMapReadOnly): Float32Array {
-		const soundIntensityValue = props.get(SOUND_INTENSITY) ?? '';
-		const data = soundIntensityValue
-			.toString()
+	private parseSoundData(soundRawData: string): Float32Array {
+		const data = soundRawData
 			.split(',')
 			.map(parseFloat)
-			// Infinity number indicates absence of any sound, so we change it to zero in order to visualize the data.
-			.map((value: number) => Number.isFinite(value) ? value : 0);
+			// -Infinity number indicates absence of any sound, so we change it to zero in order to visualize the data.
+			.map((value: number) => Number.isFinite(value) ? value : -1);
 		return Float32Array.from(data);
 	}
 
@@ -22,11 +20,9 @@ class SoundPainter implements PaintProcessor {
 		// x scale maps item index to a "x" coordinate on canvas
 		const xScale = d3.scaleLinear([0, length - 1], [0, size.width]);
 		// in order to render area shape we need two y coordinates
-		const minY = size.height;
-		const maxY = 0;
 		// y scale maps sound intensity value to a "y" coordinate on canvas
-		const y0Scale = d3.scaleLinear([-1, 1], [minY, maxY]);
-		const y1Scale = d3.scaleLinear([-1, 1], [maxY, minY]);
+		const y0Scale = d3.scaleLinear([-1, 1], [size.height, 0]);
+		const y1Scale = d3.scaleLinear([-1, 1], [0, size.height]);
 
 		return d3
 			.area<number>()
@@ -38,12 +34,13 @@ class SoundPainter implements PaintProcessor {
 	}
 
 	paint(context: CanvasRenderingContext2D, size: PaintSize, props: StylePropertyMapReadOnly): void {
-		const soundIntensity = this.getSoundIntensity(props);
-		const soundWaveDrawFn = this.getDrawFn(context, size, soundIntensity.length);
+		const color = props.get(SOUND_COLOR)!.toString();
+		const soundRawData = props.get(SOUND_DATA)!.toString();
+		const analyticalData = this.parseSoundData(soundRawData);
+		const soundWaveDrawFn = this.getDrawFn(context, size, analyticalData.length);
 		// render sound wave
-		const color = '' + props.get(SOUND_COLOR);
 		context.beginPath();
-		soundWaveDrawFn(soundIntensity);
+		soundWaveDrawFn(analyticalData);
 		context.fillStyle = color;
 		context.shadowColor = color;
 		context.shadowBlur = 100;

src/style.css

@@ -46,7 +46,7 @@ html, body {
 
 .sound-wave {
 	background-image: paint(sound);
-	background-size: 300% 300%;
+	background-size: 115% 115%;
 }
 
 /* First wave */
@@ -77,7 +77,7 @@ html, body {
 .sound-wave:nth-child(3) {
 	/* animates highlighting color */
 	animation: 5s linear infinite soundColorAnimation;
-	background-size: 250% 250%;
+	background-size: 100% 90%;
 	background-position: center;
 	mix-blend-mode: color-dodge;
 }

src/utils.ts

@@ -1,2 +1,2 @@
-export const SOUND_INTENSITY = '--sound-intensity';
+export const SOUND_DATA = '--sound-data';
 export const SOUND_COLOR = '--sound-color';