From 897bc631f73ad1266c6b8e4b5487655e48c9fbd5 Mon Sep 17 00:00:00 2001 From: Vavency <vavency@gmail.com> Date: Wed, 21 Feb 2024 16:17:13 +0000 Subject: [PATCH] Optimized pattern drawing in MkModPlayer --- .../frontend/src/components/MkModPlayer.vue | 464 +++++++++++++----- packages/frontend/src/scripts/chiptune2.ts | 46 ++ 2 files changed, 386 insertions(+), 124 deletions(-) diff --git a/packages/frontend/src/components/MkModPlayer.vue b/packages/frontend/src/components/MkModPlayer.vue index f61144cbca..75053cbc37 100644 --- a/packages/frontend/src/components/MkModPlayer.vue +++ b/packages/frontend/src/components/MkModPlayer.vue @@ -7,14 +7,17 @@ </div> <div v-else class="mod-player-enabled"> - <div class="pattern-display" @click="togglePattern()"> + <div class="pattern-display" @click="togglePattern()" @scroll="scrollHandler" @scrollend="scrollEndHandle"> <div v-if="patternHide" class="pattern-hide"> <b><i class="ph-eye ph-bold ph-lg"></i> Pattern Hidden</b> <span>{{ i18n.ts.clickToShow }}</span> </div> + <span class="patternShadowTop"></span> + <span class="patternShadowBottom"></span> <canvas ref="displayCanvas" class="pattern-canvas"></canvas> </div> <div class="controls"> + <input v-if="patternScrollSliderShow" ref="patternScrollSlider" v-model="patternScrollSliderPos" class="pattern-slider" type="range" min="0" max="100" step="0.01" style=""/> <button class="play" @click="playPause()"> <i v-if="playing" class="ph-pause ph-bold ph-lg"></i> <i v-else class="ph-play ph-bold ph-lg"></i> @@ -33,43 +36,30 @@ </template> <script lang="ts" setup> -import { ref, nextTick, computed } from 'vue'; +import { ref, nextTick, computed, watch, onDeactivated, onMounted } from 'vue'; import * as Misskey from 'misskey-js'; import { i18n } from '@/i18n.js'; import { defaultStore } from '@/store.js'; import { ChiptuneJsPlayer, ChiptuneJsConfig } from '@/scripts/chiptune2.js'; +import { isTouchUsing } from '@/scripts/touch.js'; + +const colours = { + background: '#000000', + foreground: { + default: '#ffffff', + quarter: '#ffff00', + instr: '#80e0ff', + volume: '#80ff80', + fx: '#ff80e0', + operant: '#ffe080', + }, +}; const CHAR_WIDTH = 6; const CHAR_HEIGHT = 12; const ROW_OFFSET_Y = 10; - -const colours = { - background: '#000000', - default: { - active: '#ffffff', - inactive: '#808080', - }, - quarter: { - active: '#ffff00', - inactive: '#ffe135', - }, - instr: { - active: '#80e0ff', - inactive: '#0099cc', - }, - volume: { - active: '#80ff80', - inactive: '#008000', - }, - fx: { - active: '#ff80e0', - inactive: '#800060', - }, - operant: { - active: '#ffe080', - inactive: '#806000', - }, -}; +const MAX_TIME_SPENT = 50; +const MAX_TIME_PER_ROW = 15; const props = defineProps<{ module: Misskey.entities.DriveFile @@ -79,29 +69,57 @@ const isSensitive = computed(() => { return props.module.isSensitive; }); const url = computed(() => { return props.module.url; }); let hide = ref((defaultStore.state.nsfw === 'force') ? true : isSensitive.value && (defaultStore.state.nsfw !== 'ignore')); let patternHide = ref(false); -let firstFrame = ref(true); let playing = ref(false); let displayCanvas = ref<HTMLCanvasElement>(); let progress = ref<HTMLProgressElement>(); let position = ref(0); +let patternScrollSlider = ref<HTMLProgressElement>(); +let patternScrollSliderShow = ref(false); +let patternScrollSliderPos = ref(0); const player = ref(new ChiptuneJsPlayer(new ChiptuneJsConfig())); -const rowBuffer = 24; +const maxRowNumbers = 0xFF; +const rowBuffer = 26; let buffer = null; let isSeeking = false; +let firstFrame = true; +let lastPattern = -1; +let lastDrawnRow = -1; +let numberRowCanvas = new OffscreenCanvas(2 * CHAR_WIDTH + 1, maxRowNumbers * CHAR_HEIGHT + 1); +let alreadyHiddenOnce = false; +let alreadyDrawn = [false]; +let patternTime = { 'current': 0, 'max': 0, 'initial': 0 }; -player.value.load(url.value).then((result) => { - buffer = result; - try { - player.value.play(buffer); - progress.value!.max = player.value.duration(); - display(); - } catch (err) { - console.warn(err); +function bakeNumberRow() { + let ctx = numberRowCanvas.getContext('2d', { alpha: false }) as OffscreenCanvasRenderingContext2D; + ctx.font = '10px monospace'; + + for (let i = 0; i < maxRowNumbers; i++) { + let rowText = i.toString(16); + if (rowText.length === 1) rowText = '0' + rowText; + + ctx.fillStyle = colours.foreground.default; + if (i % 4 === 0) ctx.fillStyle = colours.foreground.quarter; + + ctx.fillText(rowText, 0, 10 + i * 12); } - player.value.stop(); -}).catch((error) => { - console.error(error); +} + +onMounted(() => { + player.value.load(url.value).then((result) => { + buffer = result; + try { + player.value.play(buffer); + progress.value!.max = player.value.duration(); + bakeNumberRow(); + display(); + } catch (err) { + console.warn(err); + } + player.value.stop(); + }).catch((error) => { + console.error(error); + }); }); function playPause() { @@ -133,7 +151,7 @@ function stop(noDisplayUpdate = false) { if (!noDisplayUpdate) { try { player.value.play(buffer); - display(); + display(true); } catch (err) { console.warn(err); } @@ -162,104 +180,256 @@ function performSeek() { function toggleVisible() { hide.value = !hide.value; - if (!hide.value && patternHide.value) { - firstFrame.value = true; - patternHide.value = false; + if (!hide.value) { + lastPattern = -1; + lastDrawnRow = -1; } nextTick(() => { stop(hide.value); }); } function togglePattern() { patternHide.value = !patternHide.value; - if (!patternHide.value) { - if (player.value.getRow() === 0) { - try { - player.value.play(buffer); - display(); - } catch (err) { - console.warn(err); - } - player.value.stop(); + handleScrollBarEnable(); + + if (player.value.getRow() === 0 && player.value.getPattern() === 0) { + try { + player.value.play(buffer); + display(true); + } catch (err) { + console.warn(err); } + player.value.stop(); + } else { + display(true); } } -function display() { - if (!displayCanvas.value) { - stop(); - return; - } - - if (patternHide.value) return; - - if (firstFrame.value) { - firstFrame.value = false; - patternHide.value = true; - } - +function drawPattern() { + if (!displayCanvas.value) return; const canvas = displayCanvas.value; + const startTime = performance.now(); const pattern = player.value.getPattern(); + const nbRows = player.value.getPatternNumRows(pattern); const row = player.value.getRow(); + const halfbuf = rowBuffer / 2; + const minRow = row - halfbuf; + const maxRow = row + halfbuf; + + let rowDif = 0; + let nbChannels = 0; if (player.value.currentPlayingNode) { nbChannels = player.value.currentPlayingNode.nbChannels; } - if (canvas.width !== 12 + 84 * nbChannels + 2) { - canvas.width = 12 + 84 * nbChannels + 2; - canvas.height = 12 * rowBuffer; - } - const nbRows = player.value.getPatternNumRows(pattern); - const ctx = canvas.getContext('2d') as CanvasRenderingContext2D; - ctx.font = '10px monospace'; - ctx.fillStyle = colours.background; - ctx.fillRect(0, 0, canvas.width, canvas.height); - ctx.fillStyle = colours.default.inactive; - for (let rowOffset = 0; rowOffset < rowBuffer; rowOffset++) { - const rowToDraw = row - rowBuffer / 2 + rowOffset; - if (rowToDraw >= 0 && rowToDraw < nbRows) { - const active = (rowToDraw === row) ? 'active' : 'inactive'; - let rowText = parseInt(rowToDraw).toString(16); - if (rowText.length === 1) { - rowText = '0' + rowText; - } - ctx.fillStyle = colours.default[active]; - if (rowToDraw % 4 === 0) { - ctx.fillStyle = colours.quarter[active]; - } - ctx.fillText(rowText, 0, 10 + rowOffset * 12); - for (let channel = 0; channel < nbChannels; channel++) { - const part = player.value.getPatternRowChannel(pattern, rowToDraw, channel); - const baseOffset = (2 + (part.length + 1) * channel) * CHAR_WIDTH; - const baseRowOffset = ROW_OFFSET_Y + rowOffset * CHAR_HEIGHT; + if (pattern === lastPattern) { + rowDif = row - lastDrawnRow; + } else { + if (patternTime.initial !== 0 && !alreadyHiddenOnce) { + const trackerTime = player.value.currentPlayingNode.getProcessTime(); - ctx.fillStyle = colours.default[active]; - ctx.fillText('|', baseOffset, baseRowOffset); - - const note = part.substring(0, 3); - ctx.fillStyle = colours.default[active]; - ctx.fillText(note, baseOffset + CHAR_WIDTH, baseRowOffset); - - const instr = part.substring(4, 6); - ctx.fillStyle = colours.instr[active]; - ctx.fillText(instr, baseOffset + CHAR_WIDTH * 5, baseRowOffset); - - const volume = part.substring(6, 9); - ctx.fillStyle = colours.volume[active]; - ctx.fillText(volume, baseOffset + CHAR_WIDTH * 7, baseRowOffset); - - const fx = part.substring(10, 11); - ctx.fillStyle = colours.fx[active]; - ctx.fillText(fx, baseOffset + CHAR_WIDTH * 11, baseRowOffset); - - const op = part.substring(11, 13); - ctx.fillStyle = colours.operant[active]; - ctx.fillText(op, baseOffset + CHAR_WIDTH * 12, baseRowOffset); + if (patternTime.initial + trackerTime.max > MAX_TIME_SPENT && trackerTime.max + patternTime.max > MAX_TIME_PER_ROW) { + alreadyHiddenOnce = true; + togglePattern(); + return; } } + + patternTime = { 'current': 0, 'max': 0, 'initial': 0 }; + alreadyDrawn = []; + if (canvas.width !== (12 + 84 * nbChannels + 2)) canvas.width = 12 + 84 * nbChannels + 2; + if (canvas.height !== (12 * nbRows)) canvas.height = 12 * nbRows; + } + + const ctx = canvas.getContext('2d', { alpha: false, desynchronized: true }) as CanvasRenderingContext2D; + if (ctx.font !== '10px monospace') ctx.font = '10px monospace'; + ctx.imageSmoothingEnabled = false; + if (pattern !== lastPattern) { + ctx.fillStyle = colours.background; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.drawImage( numberRowCanvas, 0, 0 ); + } + + ctx.fillStyle = colours.foreground.default; + for (let rowOffset = minRow + rowDif; rowOffset < maxRow + rowDif; rowOffset++) { + const rowToDraw = rowOffset - rowDif; + + if (alreadyDrawn[rowToDraw] === true) continue; + + if (rowToDraw >= 0 && rowToDraw < nbRows) { + const baseOffset = 2 * CHAR_WIDTH; + const baseRowOffset = ROW_OFFSET_Y + rowToDraw * CHAR_HEIGHT; + let done = drawRow(ctx, rowToDraw, nbChannels, pattern, baseOffset, baseRowOffset); + + alreadyDrawn[rowToDraw] = done; + } + } + + lastDrawnRow = row; + lastPattern = pattern; + + patternTime.current = performance.now() - startTime; + if (patternTime.initial !== 0 && patternTime.current > patternTime.max) patternTime.max = patternTime.current; + else if (patternTime.initial === 0) patternTime.initial = patternTime.current; +} + +function drawPetternPreview() { + if (!displayCanvas.value) return; + const canvas = displayCanvas.value; + + const pattern = player.value.getPattern(); + const nbRows = player.value.getPatternNumRows(pattern); + const row = player.value.getRow(); + const halfbuf = rowBuffer / 2; + alreadyDrawn = []; + + let nbChannels = 0; + if (player.value.currentPlayingNode) { + nbChannels = player.value.currentPlayingNode.nbChannels; + } + if (canvas.width !== (12 + 84 * nbChannels + 2)) canvas.width = 12 + 84 * nbChannels + 2; + if (canvas.height !== (12 * rowBuffer)) canvas.height = 12 * rowBuffer; + + const ctx = canvas.getContext('2d', { alpha: false }) as CanvasRenderingContext2D; + ctx.font = '10px monospace'; + ctx.imageSmoothingEnabled = false; + ctx.fillStyle = colours.background; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.drawImage( numberRowCanvas, 0, (halfbuf - row) * CHAR_HEIGHT ); + + for (let rowOffset = 0; rowOffset < rowBuffer; rowOffset++) { + const rowToDraw = rowOffset + row - halfbuf; + + if (rowToDraw >= 0 && rowToDraw < nbRows) { + const baseOffset = 2 * CHAR_WIDTH; + const baseRowOffset = ROW_OFFSET_Y + rowOffset * CHAR_HEIGHT; + drawRow(ctx, rowToDraw, nbChannels, pattern, baseOffset, baseRowOffset); + } else if (rowToDraw >= 0) { + const baseRowOffset = ROW_OFFSET_Y + rowOffset * CHAR_HEIGHT; + ctx.fillStyle = colours.background; + ctx.fillRect(0, baseRowOffset - CHAR_HEIGHT, CHAR_WIDTH * 2, baseRowOffset); + } + } + + lastPattern = -1; + lastDrawnRow = -1; +} + +function drawRow(ctx: CanvasRenderingContext2D, row: number, channels: number, pattern: number, drawX = (2 * CHAR_WIDTH), drawY = ROW_OFFSET_Y) { + if (!player.value.currentPlayingNode) return false; + if (alreadyDrawn[row]) return true; + const spacer = 11; + const space = ' '; + let seperators = ''; + let note = ''; + let instr = ''; + let volume = ''; + let fx = ''; + let op = ''; + for (let channel = 0; channel < channels; channel++) { + const part = player.value.getPatternRowChannel(pattern, row, channel); + + seperators += '|' + space.repeat( spacer + 2 ); + note += part.substring(0, 3) + space.repeat( spacer ); + instr += part.substring(4, 6) + space.repeat( spacer + 1 ); + volume += part.substring(6, 9) + space.repeat( spacer ); + fx += part.substring(10, 11) + space.repeat( spacer + 2 ); + op += part.substring(11, 13) + space.repeat( spacer + 1 ); + } + + ctx.fillStyle = colours.foreground.default; + ctx.fillText(seperators, drawX, drawY); + + ctx.fillStyle = colours.foreground.default; + ctx.fillText(note, drawX + CHAR_WIDTH, drawY); + + ctx.fillStyle = colours.foreground.instr; + ctx.fillText(instr, drawX + CHAR_WIDTH * 5, drawY); + + ctx.fillStyle = colours.foreground.volume; + ctx.fillText(volume, drawX + CHAR_WIDTH * 7, drawY); + + ctx.fillStyle = colours.foreground.fx; + ctx.fillText(fx, drawX + CHAR_WIDTH * 11, drawY); + + ctx.fillStyle = colours.foreground.operant; + ctx.fillText(op, drawX + CHAR_WIDTH * 12, drawY); + + return true; +} + +function display(skipOptimizationChecks = false) { + if (!displayCanvas.value || !displayCanvas.value.parentElement) { + stop(); + return; + } + + if (patternHide.value && !skipOptimizationChecks) return; + + if (firstFrame) { + // Changing it to false should enable pattern display by default. + patternHide.value = true; + handleScrollBarEnable(); + firstFrame = false; + } + + const row = player.value.getRow(); + const pattern = player.value.getPattern(); + + if ( row === lastDrawnRow && pattern === lastPattern && !skipOptimizationChecks) return; + + // Size vs speed + if (patternHide.value) drawPetternPreview(); + else drawPattern(); + + displayCanvas.value.style.top = !patternHide.value ? 'calc( 50% - ' + (row * CHAR_HEIGHT) + 'px )' : '0%'; +} + +let suppressScrollSliderWatcher = false; + +function scrollHandler() { + suppressScrollSliderWatcher = true; + + if (!patternScrollSlider.value) return; + if (!displayCanvas.value) return; + if (!displayCanvas.value.parentElement) return; + + patternScrollSliderPos.value = (displayCanvas.value.parentElement.scrollLeft) / (displayCanvas.value.width - displayCanvas.value.parentElement.offsetWidth) * 100; + patternScrollSlider.value.style.opacity = '1'; +} + +function scrollEndHandle() { + suppressScrollSliderWatcher = false; + + if (!patternScrollSlider.value) return; + patternScrollSlider.value.style.opacity = ''; +} + +function handleScrollBarEnable() { + patternScrollSliderShow.value = (!patternHide.value && !isTouchUsing); + if (patternScrollSliderShow.value !== true) return; + + if (!displayCanvas.value) return; + if (!displayCanvas.value.parentElement) return; + if (firstFrame) { + patternScrollSliderShow.value = (12 + 84 * player.value.getPatternNumRows(player.value.getPattern()) + 2 > displayCanvas.value.parentElement.offsetWidth); + } else { + patternScrollSliderShow.value = (displayCanvas.value.width > displayCanvas.value.parentElement.offsetWidth); } } +watch(patternScrollSliderPos, () => { + if (suppressScrollSliderWatcher) return; + if (!displayCanvas.value) return; + if (!displayCanvas.value.parentElement) return; + + displayCanvas.value.parentElement.scrollLeft = (displayCanvas.value.width - displayCanvas.value.parentElement.offsetWidth) * patternScrollSliderPos.value / 100; +}); + +onDeactivated(() => { + stop(); +}); + </script> <style lang="scss" scoped> @@ -290,6 +460,7 @@ function display() { cursor: pointer; top: 12px; right: 12px; + z-index: 4; } > .pattern-display { @@ -299,22 +470,55 @@ function display() { overflow-y: hidden; background-color: black; text-align: center; - .pattern-canvas { - background-color: black; - height: 100%; + max-height: 312px; /* magic_number = CHAR_HEIGHT * rowBuffer, needs to be in px */ + + scrollbar-width: none; + + &::-webkit-scrollbar { + display: none; } + + .pattern-canvas { + position: relative; + background-color: black; + image-rendering: pixelated; + pointer-events: none; + z-index: 0; + } + + .patternShadowTop { + background: #00000080; + width: 100%; + height: calc( 50% - 14px ); + translate: 0 -100%; + top: calc( 50% - 14px ); + position: absolute; + pointer-events: none; + z-index: 2; + } + + .patternShadowBottom { + background: #00000080; + width: 100%; + height: calc( 50% - 12px ); + top: calc( 50% - 1px ); + position: absolute; + pointer-events: none; + z-index: 2; + } + .pattern-hide { display: flex; flex-direction: column; justify-content: center; align-items: center; background: rgba(64, 64, 64, 0.3); - backdrop-filter: blur(2em); + backdrop-filter: var(--modalBgFilter); color: #fff; font-size: 12px; position: absolute; - z-index: 0; + z-index: 4; width: 100%; height: 100%; @@ -328,7 +532,7 @@ function display() { display: flex; width: 100%; background-color: var(--bg); - z-index: 1; + z-index: 5; > * { padding: 4px 8px; @@ -353,6 +557,18 @@ function display() { margin: 4px 8px; overflow-x: hidden; + &.pattern-slider { + position: absolute; + width: calc( 100% - 8px * 2 ); + top: calc( 100% - 21px * 3 ); + opacity: 0%; + transition: opacity 0.2s; + + &:hover { + opacity: 100%; + } + } + &:focus { outline: none; diff --git a/packages/frontend/src/scripts/chiptune2.ts b/packages/frontend/src/scripts/chiptune2.ts index e52adb07d1..5a5b1d6c24 100644 --- a/packages/frontend/src/scripts/chiptune2.ts +++ b/packages/frontend/src/scripts/chiptune2.ts @@ -150,6 +150,27 @@ ChiptuneJsPlayer.prototype.getRow = function () { return 0; }; +ChiptuneJsPlayer.prototype.getNumPatterns = function () { + if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) { + return libopenmpt._openmpt_module_get_num_patterns(this.currentPlayingNode.modulePtr); + } + return 0; +}; + +ChiptuneJsPlayer.prototype.getCurrentSpeed = function () { + if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) { + return libopenmpt._openmpt_module_get_current_speed(this.currentPlayingNode.modulePtr); + } + return 0; +}; + +ChiptuneJsPlayer.prototype.getCurrentTempo = function () { + if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) { + return libopenmpt._openmpt_module_get_current_tempo(this.currentPlayingNode.modulePtr); + } + return 0; +}; + ChiptuneJsPlayer.prototype.getPatternNumRows = function (pattern: number) { if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) { return libopenmpt._openmpt_module_get_pattern_num_rows(this.currentPlayingNode.modulePtr, pattern); @@ -164,6 +185,20 @@ ChiptuneJsPlayer.prototype.getPatternRowChannel = function (pattern: number, row return ''; }; +ChiptuneJsPlayer.prototype.getCtls = function () { + if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) { + return libopenmpt._openmpt_module_get_ctls(this.currentPlayingNode.modulePtr); + } + return 0; +}; + +ChiptuneJsPlayer.prototype.version = function () { + if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) { + return libopenmpt._openmpt_get_library_version(); + } + return 0; +}; + ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: object) { const maxFramesPerChunk = 4096; const processNode = this.audioContext.createScriptProcessor(2048, 0, 2); @@ -178,6 +213,7 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje processNode.paused = false; processNode.leftBufferPtr = libopenmpt._malloc(4 * maxFramesPerChunk); processNode.rightBufferPtr = libopenmpt._malloc(4 * maxFramesPerChunk); + processNode.perf = { 'current': 0, 'max': 0 }; processNode.cleanup = function () { if (this.modulePtr !== 0) { libopenmpt._openmpt_module_destroy(this.modulePtr); @@ -205,7 +241,13 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje processNode.togglePause = function () { this.paused = !this.paused; }; + processNode.getProcessTime = function() { + const max = this.perf.max; + this.perf.max = 0; + return { 'current': this.perf.current, 'max': max }; + }; processNode.onaudioprocess = function (e) { + let startTimeP1 = performance.now(); const outputL = e.outputBuffer.getChannelData(0); const outputR = e.outputBuffer.getChannelData(1); let framesToRender = outputL.length; @@ -231,11 +273,13 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje const currentPattern = libopenmpt._openmpt_module_get_current_pattern(this.modulePtr); const currentRow = libopenmpt._openmpt_module_get_current_row(this.modulePtr); + startTimeP1 = startTimeP1 - performance.now(); if (currentPattern !== this.patternIndex) { processNode.player.fireEvent('onPatternChange'); } processNode.player.fireEvent('onRowChange', { index: currentRow }); + const startTimeP2 = performance.now(); while (framesToRender > 0) { const framesPerChunk = Math.min(framesToRender, maxFramesPerChunk); const actualFramesPerChunk = libopenmpt._openmpt_module_read_float_stereo(this.modulePtr, this.context.sampleRate, framesPerChunk, this.leftBufferPtr, this.rightBufferPtr); @@ -262,6 +306,8 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje this.cleanup(); error ? processNode.player.fireEvent('onError', { type: 'openmpt' }) : processNode.player.fireEvent('onEnded'); } + this.perf.current = performance.now() - startTimeP2 + startTimeP1; + if (this.perf.current > this.perf.max) this.perf.max = this.perf.current; }; return processNode; };