From b8eb3f091bd895e2f438d308d57bd86a56278051 Mon Sep 17 00:00:00 2001 From: Janne Mareike Koschinski Date: Wed, 8 Jan 2025 17:51:07 +0100 Subject: [PATCH 1/2] initial implementation of program/preview --- .editorconfig | 3 + vocto/config.py | 5 + vocto/port.py | 2 + voctocore/lib/commands.py | 2 + voctocore/lib/pipeline.py | 15 +++ voctocore/lib/previewscene.py | 121 +++++++++++++++++ voctocore/lib/videopremix.py | 241 ++++++++++++++++++++++++++++++++++ voctogui/lib/ui.py | 21 ++- voctogui/ui/voctogui.ui | 26 +++- 9 files changed, 433 insertions(+), 3 deletions(-) create mode 100644 voctocore/lib/previewscene.py create mode 100644 voctocore/lib/videopremix.py diff --git a/.editorconfig b/.editorconfig index b3b5f855..5271ae28 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,6 +5,9 @@ end_of_line = lf insert_final_newline = true charset = utf-8 +[*.ui] +indent_size = 2 + [*.py] indent_style = space indent_size = 4 diff --git a/vocto/config.py b/vocto/config.py index e0ae9d9d..43531937 100644 --- a/vocto/config.py +++ b/vocto/config.py @@ -418,6 +418,9 @@ def getSRTServerEnabled(self): def getPreviewsEnabled(self): return self.getboolean('previews', 'enabled', fallback=False) + def getPreviewMixEnabled(self): + return self.getboolean('mix', 'preview', fallback=False) + def getAVRawOutputEnabled(self): return self.getboolean('avrawoutput', 'enabled', fallback=True) @@ -478,6 +481,8 @@ def getOverlayUserAutoOff(self): def _getInternalSources(self): sources = ["mix"] + if self.getPreviewMixEnabled(): + sources += ["premix"] if self.getBlinderEnabled(): sources += ["blinder", "mix-blinded"] for source in self.getLiveSources(): diff --git a/vocto/port.py b/vocto/port.py index c7115c90..f552be10 100644 --- a/vocto/port.py +++ b/vocto/port.py @@ -25,6 +25,8 @@ class Port(object): SOURCES_PREVIEW = SOURCES_OUT+OFFSET_PREVIEW LIVE_OUT = 15000 LIVE_PREVIEW = LIVE_OUT+OFFSET_PREVIEW + PREMIX_OUT = 21000 + PREMIX_PREVIEW = PREMIX_OUT+OFFSET_PREVIEW LOCALPLAYOUT_OUT = 19000 def __init__(self, name, source=None, audio=None, video=None): diff --git a/voctocore/lib/commands.py b/voctocore/lib/commands.py index 3bbbb60b..1fea1e8d 100644 --- a/voctocore/lib/commands.py +++ b/voctocore/lib/commands.py @@ -199,6 +199,8 @@ def transition(self, command): def best(self, command): """tests if transition to the composite described by command is possible. """ + if self.pipeline.vpremix is not None: + self.pipeline.vpremix.setComposite(command) transition = self.pipeline.vmix.testTransition(command) if transition: return OkResponse('best','transition', *transition) diff --git a/voctocore/lib/pipeline.py b/voctocore/lib/pipeline.py index d51e3738..1ac4e010 100644 --- a/voctocore/lib/pipeline.py +++ b/voctocore/lib/pipeline.py @@ -17,6 +17,7 @@ from lib.sources import spawn_source from lib.srtserver import SRTServerSink from lib.videomix import VideoMix +from lib.videopremix import VideoPreMix from vocto.debug import gst_generate_dot from vocto.port import Port @@ -72,6 +73,12 @@ def __init__(self): self.vmix = VideoMix() self.bins.append(self.vmix) + self.vpremix = None + if Config.getPreviewMixEnabled(): + self.log.info('Creating Videopremixer') + self.vpremix = VideoPreMix() + self.bins.append(self.vpremix) + for idx, background in enumerate(Config.getBackgroundSources()): # create background source source = spawn_source( @@ -84,6 +91,10 @@ def __init__(self): dest = AVRawOutput('mix', Port.MIX_OUT, use_audio_mix=True) self.bins.append(dest) self.ports.append(Port('mix', dest)) + if Config.getPreviewMixEnabled(): + dest = AVRawOutput('premix', Port.PREMIX_OUT, use_audio_mix=True) + self.bins.append(dest) + self.ports.append(Port('premix', dest)) # add localui if Config.getProgramOutputEnabled(): @@ -96,6 +107,10 @@ def __init__(self): dest = AVPreviewOutput('mix', Port.MIX_PREVIEW, use_audio_mix=True) self.bins.append(dest) self.ports.append(Port('preview-mix', dest)) + if Config.getPreviewMixEnabled(): + dest = AVPreviewOutput('premix', Port.PREMIX_PREVIEW, use_audio_mix=True) + self.bins.append(dest) + self.ports.append(Port('preview-premix', dest)) # create blinding sources and mixer if Config.getBlinderEnabled(): diff --git a/voctocore/lib/previewscene.py b/voctocore/lib/previewscene.py new file mode 100644 index 00000000..672f4c9a --- /dev/null +++ b/voctocore/lib/previewscene.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +import logging +import gi +gi.require_version('GstController', '1.0') +from gi.repository import Gst, GstController +from vocto.transitions import Frame, L, T, R, B + +class PreviewScene: + """ Scene is the adaptor between the gstreamer compositor + and voctomix frames. + With commit() you add frames at a specified play time + """ + log = logging.getLogger('Scene') + + def __init__(self, sources, pipeline, fps, start_sink, cropping=True): + """ initialize with a gstreamer pipeline and names + of the sources to manage + """ + # frames to apply from + self.frames = dict() + # binding pads to apply to + self.pads = dict() + self.cpads = dict() if cropping else None + # time per frame + self.frame_time = int(Gst.SECOND / fps) + + def bind(pad, prop): + """ adds a binding to a gstreamer property + pad's property + """ + # set up a new control source + cs = GstController.InterpolationControlSource() + # stop control source's internal interpolation + cs.set_property( + 'mode', GstController.InterpolationMode.NONE) + # create control binding + cb = GstController.DirectControlBinding.new_absolute( + pad, prop, cs) + # add binding to pad + pad.add_control_binding(cb) + # return binding + return cs + + # walk all sources + for idx, source in enumerate(sources): + # initially invisible + self.frames[source] = None + # get mixer pad from pipeline + mixerpad = (pipeline + .get_by_name('videopremixer') + .get_static_pad('sink_%s' % (idx + start_sink))) + # add dictionary of binds to all properties + # we vary for this source + self.pads[source] = { + 'xpos': bind(mixerpad, 'xpos'), + 'ypos': bind(mixerpad, 'ypos'), + 'width': bind(mixerpad, 'width'), + 'height': bind(mixerpad, 'height'), + 'alpha': bind(mixerpad, 'alpha'), + 'zorder': bind(mixerpad, 'zorder'), + } + # get mixer and cropper pad from pipeline + if self.cpads is not None: + cropperpad = (pipeline + .get_by_name("cropper-%s" % source)) + self.cpads[source] = { + 'croptop': bind(cropperpad, 'top'), + 'cropleft': bind(cropperpad, 'left'), + 'cropbottom': bind(cropperpad, 'bottom'), + 'cropright': bind(cropperpad, 'right') + } + # ready to initialize gstreamer + self.dirty = False + + def commit(self, source, frames): + ''' commit multiple frames to the current gstreamer scene ''' + self.log.debug("Commit %d frame(s) to source %s", len(frames), source) + self.frames[source] = frames + self.dirty = True + + def set(self, source, frame): + ''' commit single frame to the current gstreamer scene ''' + self.log.debug("Set frame to source %s", source) + self.frames[source] = [frame] + self.dirty = True + + def push(self, at_time=0): + ''' apply all committed frames to GStreamer pipeline ''' + # get pad for given source + for source, frames in self.frames.items(): + if not frames: + frames = [Frame(zorder=-1,alpha=0)] + self.log.info("Pushing %d frame(s) to source '%s' at time %dms", len( + frames), source, at_time / Gst.MSECOND) + # reset time + time = at_time + # get GStreamer property pad for this source + pad = self.pads[source] + cpad = self.cpads[source] if self.cpads else None + self.log.debug(" %s", Frame.str_title()) + # apply all frames of this source to GStreamer pipeline + for idx, frame in enumerate(frames): + self.log.debug("%2d: %s", idx, frame) + cropped = frame.cropped() + alpha = frame.float_alpha() + # transmit frame properties into mixing pipeline + pad['xpos'].set(time, cropped[L]) + pad['ypos'].set(time, cropped[T]) + pad['width'].set(time, cropped[R] - cropped[L]) + pad['height'].set(time, cropped[B] - cropped[T]) + pad['alpha'].set(time, alpha) + pad['zorder'].set(time, frame.zorder if alpha != 0 else -1) + if cpad: + cpad['croptop'].set(time, frame.crop[T]) + cpad['cropleft'].set(time, frame.crop[L]) + cpad['cropbottom'].set(time, frame.crop[B]) + cpad['cropright'].set(time, frame.crop[R]) + # next frame time + time += self.frame_time + self.frames[source] = None + self.dirty = False diff --git a/voctocore/lib/videopremix.py b/voctocore/lib/videopremix.py new file mode 100644 index 00000000..94694fdf --- /dev/null +++ b/voctocore/lib/videopremix.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +import logging + +from configparser import NoOptionError +from enum import Enum, unique +import gi +gi.require_version('GstController', '1.0') +from gi.repository import Gst +from lib.config import Config +from vocto.transitions import Composites, Transitions, Frame, fade_alpha +from lib.previewscene import PreviewScene +from lib.overlay import Overlay +from lib.args import Args + +from vocto.composite_commands import CompositeCommand + + +class VideoPreMix(object): + log = logging.getLogger('VideoPreMix') + + def __init__(self): + # read sources from confg file + self.bgSources = Config.getBackgroundSources() + self.sources = Config.getVideoSources() + self.log.info('Configuring mixer for %u source(s) and %u background source(s)', len(self.sources), len(self.bgSources)) + + # load composites from config + self.log.info("Reading transitions configuration...") + self.composites = Config.getComposites() + + # load transitions from configuration + self.transitions = Config.getTransitions(self.composites) + self.scene = None + self.bgScene = None + self.overlay = None + + Config.getAudioStreams() + + # build GStreamer mixing pipeline descriptor + self.bin = "" if Args.no_bins else """ + bin.( + name=VideoPreMix + """ + self.bin += """ + compositor + name=videopremixer + """ + if Config.hasOverlay(): + self.bin += """\ + ! queue + max-size-time=3000000000 + name=queue-overlay + ! gdkpixbufoverlay + name=overlay + overlay-width={width} + overlay-height={height} + """.format( + width=Config.getVideoResolution()[0], + height=Config.getVideoResolution()[1] + ) + if Config.getOverlayFile(): + self.bin += """\ + location={overlay} + alpha=1.0 + """.format(overlay=Config.getOverlayFilePath(Config.getOverlayFile())) + else: + self.log.info("No initial overlay source configured.") + + self.bin += """\ + ! identity + name=sigpre + ! {vcaps} + ! queue + max-size-time=3000000000 + ! tee + name=video-premix + """.format( + vcaps=Config.getVideoCaps() + ) + + for idx, background in enumerate(self.bgSources): + self.bin += """ + video-{name}. + ! queue + max-size-time=3000000000 + name=queue-video-{name} + ! videopremixer. + """.format(name=background) + + for idx, name in enumerate(self.sources): + self.bin += """ + video-{name}. + ! videobox + name=cropper-{name} + ! queue + max-size-time=3000000000 + name=queue-videopremixer-{name} + ! videopremixer. + """.format( + name=name, + idx=idx + ) + + self.bin += "" if Args.no_bins else """) + """ + + def attach(self, pipeline): + self.log.debug('Binding Handoff-Handler for ' + 'Synchronus mixer manipulation') + self.pipeline = pipeline + sigpre = pipeline.get_by_name('sigpre') + sigpre.connect('handoff', self.on_handoff) + + self.log.debug('Initializing Mixer-State') + # initialize pipeline bindings for all sources + self.bgScene = PreviewScene(self.bgSources, pipeline, self.transitions.fps, 0, cropping=False) + self.scene = PreviewScene(self.sources, pipeline, self.transitions.fps, len(self.bgSources)) + self.compositeMode = None + self.sourceA = None + self.sourceB = None + self.setCompositeEx(Composites.targets(self, self.composites)[ + 0].name, self.sources[0], self.sources[1]) + + if Config.hasOverlay(): + self.overlay = Overlay( + pipeline, Config.getOverlayFile(), Config.getOverlayBlendTime()) + + def __str__(self): + return 'VideoPreMix' + + def getPlayTime(self): + # get play time from mixing pipeline or assume zero + return self.pipeline.get_pipeline_clock().get_time() - \ + self.pipeline.get_base_time() + + def on_handoff(self, object, buffer): + playTime = self.getPlayTime() + if self.bgScene and self.bgScene.dirty: + # push background scene to gstreamer + self.log.debug('Applying new background at %d ms', + playTime / Gst.MSECOND) + self.bgScene.push(playTime) + if self.scene and self.scene.dirty: + # push scene to gstreamer + self.log.debug('Applying new mix at %d ms', + playTime / Gst.MSECOND) + self.scene.push(playTime) + + def setCompositeEx(self, newCompositeName=None, newA=None, newB=None): + # expect strings or None as parameters + assert not newCompositeName or type(newCompositeName) == str + assert not newA or type(newA) == str + assert not newB or type(newB) == str + + # get current composite + if not self.compositeMode: + curCompositeName = None + self.log.info("Request composite %s(%s,%s)", + newCompositeName, newA, newB) + else: + curCompositeName = self.compositeMode + curA = self.sourceA + curB = self.sourceB + self.log.info("Request composite change from %s(%s,%s) to %s(%s,%s)", + curCompositeName, curA, curB, newCompositeName, newA, newB) + + # check if there is any None parameter and fill it up with + # reasonable value from the current scene + if curCompositeName and not (newCompositeName and newA and newB): + # use current state if not defined by parameter + if not newCompositeName: + newCompositeName = curCompositeName + if not newA: + newA = curA if newB != curA else curB + if not newB: + newB = curA if newA == curB else curB + self.log.debug("Completing wildcarded composite to %s(%s,%s)", + newCompositeName, newA, newB) + # post condition: we should have all parameters now + assert newA != newB + assert newCompositeName and newA and newB + + # fetch composites + newComposite = self.composites[newCompositeName] + + # if new scene is complete + if newComposite and newA in self.sources and newB in self.sources: + self.log.debug("New composite shall be %s(%s,%s)",newComposite.name, newA, newB) + # try to find a matching transition from current to new scene + transition = None + targetA, targetB = newA, newB + # z-orders of A and B + below = 100 + above = 101 + # found transition? + # apply new scene (hard cut) + self.log.debug("setting composite '%s' to scene", newComposite.name) + self.scene.set(targetA, newComposite.Az(below)) + self.scene.set(targetB, newComposite.Bz(above)) + # make all other sources invisible + for source in self.sources: + if source not in [targetA, targetB]: + self.log.debug("making source %s invisible", source) + self.scene.set(source, Frame(True, alpha=0, zorder=-1)) + + # get current and new background source by the composites + curBgSource = Config.getBackgroundSource(curCompositeName) + newBgSource = Config.getBackgroundSource(newCompositeName) + if curBgSource != newBgSource: + # apply new scene (hard cut) + self.log.debug("setting new background to scene") + # just switch to new background + bgFrame = Frame(True, zorder=0, rect=[0,0,*Config.getVideoResolution()]) + self.bgScene.set(newBgSource, bgFrame) + # make all other background sources invisible + for source in self.bgSources: + if source not in [curBgSource,newBgSource]: + self.log.debug("making background source %s invisible", source) + self.bgScene.set(source, Frame(True, alpha=0, zorder=-1)) + else: + # report unknown elements of the target scene + if not newComposite: + self.log.error("Unknown composite '%s'", newCompositeName) + if not newA in self.sources: + self.log.error("Unknown source '%s'", newA) + if not newB in self.sources: + self.log.error("Unknown source '%s'", newB) + + # remember scene we've set + self.compositeMode = newComposite.name + self.sourceA = newA + self.sourceB = newB + + def setComposite(self, command): + ''' parse switch to the composite described by string command ''' + # expect string as parameter + assert type(command) == str + # parse command + command = CompositeCommand.from_str(command) + self.log.debug("Setting new composite by string '%s'", command) + self.setCompositeEx(command.composite, command.A, command.B) diff --git a/voctogui/lib/ui.py b/voctogui/lib/ui.py index 9ee33a4a..5482f745 100644 --- a/voctogui/lib/ui.py +++ b/voctogui/lib/ui.py @@ -51,9 +51,18 @@ def setup(self): # Connect Close-Handler self.win.connect('delete-event', Gtk.main_quit) - output_aspect_ratio = self.find_widget_recursive( - self.win, 'output_aspect_ratio') + output_aspect_ratio = self.find_widget_recursive(self.win, 'output_aspect_ratio') + output_aspect_ratio_preview = self.find_widget_recursive(self.win, 'output_aspect_ratio_preview') output_aspect_ratio.props.ratio = Config.getVideoRatio() + if Config.getPreviewMixEnabled(): + output_aspect_ratio_preview.props.hexpand = True + output_aspect_ratio_preview.props.vexpand = True + output_aspect_ratio_preview.set_visible(True) + output_aspect_ratio_preview.props.ratio = Config.getVideoRatio() + else: + output_aspect_ratio_preview.props.hexpand = False + output_aspect_ratio_preview.props.vexpand = False + output_aspect_ratio_preview.set_visible(False) audio_box = self.find_widget_recursive(self.win, 'audio_box') @@ -85,6 +94,14 @@ def setup(self): port=Port.MIX_PREVIEW if Config.getPreviewsEnabled() else Port.MIX_OUT, name="MIX" ) + self.premix_video_display = None + if Config.getPreviewMixEnabled(): + self.premix_video_display = VideoDisplay( + self.find_widget_recursive(self.win, 'video_main_preview'), + None, + port=Port.PREMIX_PREVIEW if Config.getPreviewsEnabled() else Port.PREMIX_OUT, + name="PREMIX" + ) for idx, livepreview in enumerate(Config.getLivePreviews()): if Config.getPreviewsEnabled(): diff --git a/voctogui/ui/voctogui.ui b/voctogui/ui/voctogui.ui index e2c8752a..08deab4a 100644 --- a/voctogui/ui/voctogui.ui +++ b/voctogui/ui/voctogui.ui @@ -105,6 +105,30 @@ True False + + + True + False + 0 + none + False + + + 100 + True + False + False + True + True + + + + + False + True + 0 + + True @@ -126,7 +150,7 @@ False True - 0 + 1 From 5a7784f49422070a50315ce31d73e684bf633235 Mon Sep 17 00:00:00 2001 From: Janne Mareike Koschinski Date: Fri, 10 Jan 2025 00:03:48 +0100 Subject: [PATCH 2/2] fix issue where preview and program would share a cropper or overlay mixer --- voctocore/lib/previewscene.py | 4 ++-- voctocore/lib/videopremix.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/voctocore/lib/previewscene.py b/voctocore/lib/previewscene.py index 672f4c9a..da96901e 100644 --- a/voctocore/lib/previewscene.py +++ b/voctocore/lib/previewscene.py @@ -10,7 +10,7 @@ class PreviewScene: and voctomix frames. With commit() you add frames at a specified play time """ - log = logging.getLogger('Scene') + log = logging.getLogger('PreviewScene') def __init__(self, sources, pipeline, fps, start_sink, cropping=True): """ initialize with a gstreamer pipeline and names @@ -62,7 +62,7 @@ def bind(pad, prop): # get mixer and cropper pad from pipeline if self.cpads is not None: cropperpad = (pipeline - .get_by_name("cropper-%s" % source)) + .get_by_name("precropper-%s" % source)) self.cpads[source] = { 'croptop': bind(cropperpad, 'top'), 'cropleft': bind(cropperpad, 'left'), diff --git a/voctocore/lib/videopremix.py b/voctocore/lib/videopremix.py index 94694fdf..20ced1e2 100644 --- a/voctocore/lib/videopremix.py +++ b/voctocore/lib/videopremix.py @@ -49,9 +49,9 @@ def __init__(self): self.bin += """\ ! queue max-size-time=3000000000 - name=queue-overlay + name=queue-preoverlay ! gdkpixbufoverlay - name=overlay + name=preoverlay overlay-width={width} overlay-height={height} """.format( @@ -83,7 +83,7 @@ def __init__(self): video-{name}. ! queue max-size-time=3000000000 - name=queue-video-{name} + name=queue-prevideo-{name} ! videopremixer. """.format(name=background) @@ -91,7 +91,7 @@ def __init__(self): self.bin += """ video-{name}. ! videobox - name=cropper-{name} + name=precropper-{name} ! queue max-size-time=3000000000 name=queue-videopremixer-{name}