From dfa992a4878971ec965ef9afed9d3e198fcbece6 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Mon, 30 Apr 2018 16:13:18 +0100 Subject: [PATCH 01/14] Remove requirement for pytest-capturelog, removed from No longer needed because it has been merged into the core. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 39e7c76f6..463a1b9f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ sphinx-rtd-theme pytest-bdd pytest-cov pytest-mock -pytest-capturelog pytest-twisted coverage pytest-runner From 0f9d9616405d15ecfa1937b5ab40a8dcd27b759d Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Mon, 30 Apr 2018 17:25:02 +0100 Subject: [PATCH 02/14] Refactor proxy handling for client Track a change to proxy handling in Autobahn / twisted for #477. There's probably a more elegant way, but this works for now... Lacking a test, but have verified it locally. --- ebu_tt_live/config/backend.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ebu_tt_live/config/backend.py b/ebu_tt_live/config/backend.py index c8c03b185..5cc0b3b0c 100644 --- a/ebu_tt_live/config/backend.py +++ b/ebu_tt_live/config/backend.py @@ -137,8 +137,6 @@ def _ws_create_server_factory(self, listen, producer=None, consumer=None): def _ws_create_client_factories(self, connect, producer=None, consumer=None, proxy=None): factory_args = {} - if proxy: - factory_args.update({'host': proxy.host, 'port': proxy.port}) for dst in connect: client_factory = self._websocket.BroadcastClientFactory( url=dst.geturl(), @@ -147,6 +145,10 @@ def _ws_create_client_factories(self, connect, producer=None, consumer=None, pro **factory_args ) client_factory.protocol = self._websocket.BroadcastClientProtocol + if proxy: + proxy_dict = {u'host': proxy.host, u'port': proxy.port} + client_factory.proxy = proxy_dict + client_factory.connect() def ws_backend_producer(self, custom_producer, listen=None, connect=None, proxy=None): From 58bb257e386612a91f31b903281be56826c6f0be Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Tue, 1 May 2018 17:30:53 +0100 Subject: [PATCH 03/14] More elegant fix for proxy We don't need the HTTPProxyConfig object at all anymore, just a `dict` will do. --- ebu_tt_live/config/backend.py | 4 +--- ebu_tt_live/config/carriage.py | 6 +----- ebu_tt_live/utils.py | 4 +--- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/ebu_tt_live/config/backend.py b/ebu_tt_live/config/backend.py index 5cc0b3b0c..465c459e8 100644 --- a/ebu_tt_live/config/backend.py +++ b/ebu_tt_live/config/backend.py @@ -145,9 +145,7 @@ def _ws_create_client_factories(self, connect, producer=None, consumer=None, pro **factory_args ) client_factory.protocol = self._websocket.BroadcastClientProtocol - if proxy: - proxy_dict = {u'host': proxy.host, u'port': proxy.port} - client_factory.proxy = proxy_dict + client_factory.proxy = proxy client_factory.connect() diff --git a/ebu_tt_live/config/carriage.py b/ebu_tt_live/config/carriage.py index b1f0f1624..a4c92af93 100644 --- a/ebu_tt_live/config/carriage.py +++ b/ebu_tt_live/config/carriage.py @@ -2,7 +2,6 @@ from ebu_tt_live.carriage.direct import DirectCarriageImpl from ebu_tt_live.carriage.websocket import WebsocketProducerCarriage, WebsocketConsumerCarriage from ebu_tt_live.carriage import filesystem -from ebu_tt_live.utils import HTTPProxyConfig from ebu_tt_live.strings import ERR_CONF_PROXY_CONF_VALUE, ERR_NO_SUCH_COMPONENT from ebu_tt_live.errors import ConfigurationError from ebu_tt_live.strings import CFG_FILENAME_PATTERN, CFG_MESSAGE_PATTERN @@ -134,10 +133,7 @@ def parse_proxy_address(value): match = proxy_regex.match(value) if match: # Ignoring the protocol part for now as it is only a http proxy - result = HTTPProxyConfig( - host=match.group('host'), - port=int(match.group('port')) - ) + result = {u'host': match.group('host'), u'port': int(match.group('port'))} elif value: # In this case something was provided that isn't a falsy value but the parsing failed. raise ConfigurationError( diff --git a/ebu_tt_live/utils.py b/ebu_tt_live/utils.py index 000598249..9ae3afca2 100644 --- a/ebu_tt_live/utils.py +++ b/ebu_tt_live/utils.py @@ -358,8 +358,6 @@ def __call__(cls, *args, **kwargs): instance = super(AutoRegisteringABCMeta, cls).__call__(*args, **kwargs) return instance -HTTPProxyConfig = collections.namedtuple('HTTPProxyConfig', ['host', 'port']) - # The following section is taken from https://github.com/django/django/blob/master/django/test/utils.py # This is a relatively simple XML comparator implementation based on Python's minidom library. @@ -467,4 +465,4 @@ def first_node(document): want_root = first_node(parseString(want)) got_root = first_node(parseString(got)) - return check_element(want_root, got_root) \ No newline at end of file + return check_element(want_root, got_root) From 6cb5f7445244f52dec723e7ad389e8107c4d7832 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Tue, 8 May 2018 14:17:25 +0100 Subject: [PATCH 04/14] Comment out debug log line that sometimes gives errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Have seen this error a couple of times during live operation: ```Shell Unhandled error in Deferred: [CRITICAL] (2018-05-08 13:43:42,705) in twisted[154] - Unhandled error in Deferred: [CRITICAL] (2018-05-08 13:43:42,705) in twisted[154] - Traceback (most recent call last): File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/venv/lib/python2.7/site-packages/twisted/internet/defer.py", line 150, in maybeDeferred result = f(*args, **kw) File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/node/consumer.py", line 134, in convert_next_segment end=self.last_segment_end + self._segment_length File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/node/consumer.py", line 122, in get_segment sequence_number=self._segment_counter File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/documents/ebutt3.py", line 1004, in extract_segment document_segments=document_segments File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/documents/ebutt3_splicer.py", line 29, in __init__ self._do_splice() File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/documents/ebutt3_splicer.py", line 66, in _do_splice merged_body = merged_body.merge(current_tt.body, self._dataset) File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/bindings/__init__.py", line 974, in merge self._merge_deconflict_ids(element=other_elem, dest=merged_body, ids=ids) File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/bindings/__init__.py", line 949, in _merge_deconflict_ids cls._merge_deconflict_ids(item.value, copied_elem, ids) File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/bindings/__init__.py", line 949, in _merge_deconflict_ids cls._merge_deconflict_ids(item.value, copied_elem, ids) File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/bindings/__init__.py", line 949, in _merge_deconflict_ids cls._merge_deconflict_ids(item.value, copied_elem, ids) File "/Users/megitn02/Code/ebu/ebu-tt-live-toolkit/ebu_tt_live/bindings/__init__.py", line 942, in _merge_deconflict_ids log.debug('processing child: {} of {}'.format(item.value, element)) UnicodeEncodeError: 'ascii' codec can't encode character u'\xa3' in position 1: ordinal not in range(128) ``` So commenting out that line. No idea how that character can appear in the relevant line, it's a £ sign. --- ebu_tt_live/bindings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebu_tt_live/bindings/__init__.py b/ebu_tt_live/bindings/__init__.py index 4b1eb4347..1381af0ee 100644 --- a/ebu_tt_live/bindings/__init__.py +++ b/ebu_tt_live/bindings/__init__.py @@ -939,7 +939,7 @@ def _merge_deconflict_ids(cls, element, dest, ids): output = [] for item in children: - log.debug('processing child: {} of {}'.format(item.value, element)) + #log.debug('processing child: {} of {}'.format(item.value, element)) if isinstance(item, NonElementContent): copied_stuff = copy.copy(item.value) output.append(copied_stuff) From 7a595d4a4035dac318139d664bc9c709de8b72a5 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Tue, 8 May 2018 14:17:45 +0100 Subject: [PATCH 05/14] Remove debug print statement from deduplicator --- ebu_tt_live/node/deduplicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebu_tt_live/node/deduplicator.py b/ebu_tt_live/node/deduplicator.py index e58364aeb..ced39e88d 100644 --- a/ebu_tt_live/node/deduplicator.py +++ b/ebu_tt_live/node/deduplicator.py @@ -55,7 +55,7 @@ def remove_duplication(self, document): if document.binding.head.styling is not None: styles = document.binding.head.styling.style - print styles + document.binding.head.styling.style = None self.CollateUniqueVals(styles, old_id_dict, new_id_dict, hash_dict) From 00a94c9ded93fad9287a235d3838a65ed0f1fce2 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Tue, 8 May 2018 15:57:08 +0100 Subject: [PATCH 06/14] Add a first message counter value Simplest fix for #481 --- ebu_tt_live/carriage/filesystem.py | 5 +++-- ebu_tt_live/config/carriage.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ebu_tt_live/carriage/filesystem.py b/ebu_tt_live/carriage/filesystem.py index ef4db4475..09ad04ed7 100644 --- a/ebu_tt_live/carriage/filesystem.py +++ b/ebu_tt_live/carriage/filesystem.py @@ -72,20 +72,21 @@ def __init__(self, file_name_pattern = CFG_FILENAME_PATTERN, message_file_name_pattern = CFG_MESSAGE_PATTERN, circular_buf_size = 0, - suppress_manifest = False): + suppress_manifest = False, + first_msg_counter = 0): self._dirpath = dirpath if not os.path.exists(self._dirpath): os.makedirs(self._dirpath) self._file_name_pattern = file_name_pattern self._message_file_name_pattern = message_file_name_pattern self._counter = 0 + self._msg_counter = first_msg_counter self._circular_buf_size = circular_buf_size if circular_buf_size > 0 : self._circular_buf = RotatingFileBuffer(maxlen=circular_buf_size) self._suppress_manifest = suppress_manifest # Get a set of default clocks self._default_clocks = {} - self._msg_counter = 0 def _get_default_clock(self, sequence_identifier, time_base, clock_mode=None): clock_obj = self._default_clocks.get(sequence_identifier, None) diff --git a/ebu_tt_live/config/carriage.py b/ebu_tt_live/config/carriage.py index a4c92af93..a8aa401d6 100644 --- a/ebu_tt_live/config/carriage.py +++ b/ebu_tt_live/config/carriage.py @@ -75,6 +75,10 @@ class FilesystemOutput(ConfigurableComponent): default=False, doc='Suppress output of a manifest file (default false)' ) + required_config.add_option( + 'begin_count', + default=0, + doc='Value to begin counting at for patterns including {counter}; the first output value will be this plus 1.') def __init__(self, config, local_config): super(FilesystemOutput, self).__init__(config, local_config) @@ -83,7 +87,8 @@ def __init__(self, config, local_config): file_name_pattern=self.config.filename_pattern, message_file_name_pattern=self.config.message_filename_pattern, circular_buf_size=self.config.rotating_buf, - suppress_manifest=self.config.suppress_manifest) + suppress_manifest=self.config.suppress_manifest, + first_msg_counter=self.config.begin_count) From 1890be21c220a00d707e52ee2dd37e84809248fd Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Tue, 8 May 2018 16:41:11 +0100 Subject: [PATCH 07/14] Use `codecs.open` and ignore errors This strips out unencodable characters and if not fixes, at least masks #483 by using `codecs.open` and telling it to ignore errors. --- ebu_tt_live/carriage/filesystem.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ebu_tt_live/carriage/filesystem.py b/ebu_tt_live/carriage/filesystem.py index 09ad04ed7..1b2434570 100644 --- a/ebu_tt_live/carriage/filesystem.py +++ b/ebu_tt_live/carriage/filesystem.py @@ -9,6 +9,7 @@ import six import os import time +import codecs log = logging.getLogger(__name__) @@ -154,7 +155,7 @@ def emit_data(self, data, sequence_identifier=None, sequence_number=None, # can be selected once at the beginning and dereferenced rather than repeating # if statements. filepath = os.path.join(self._dirpath, filename) - with open(filepath, 'w') as destfile: + with codecs.open(filepath, mode='w', errors='ignore') as destfile: destfile.write(data) destfile.flush() @@ -199,7 +200,7 @@ def emit_data(self, data, sequence_identifier=None, sequence_number=None, new_manifest_line = CFG_MANIFEST_LINE_PATTERN.format( availability_time=timedelta_to_str_manifest(availability_time), filename=filename) - with open(self._manifest_path, 'a') as f: + with codecs.open(self._manifest_path, mode='a', errors='ignore') as f: f.write(new_manifest_line) @@ -237,11 +238,11 @@ def __init__(self, manifest_path, custom_consumer, do_tail): self._manifest_path = manifest_path self._custom_consumer = custom_consumer self._do_tail = do_tail - with open(manifest_path, 'r') as manifest: + with codecs.open(manifest_path, 'r') as manifest: self._manifest_lines_iter = iter(manifest.readlines()) def resume_reading(self): - with open(self._manifest_path, 'r') as manifest_file: + with codecs.open(self._manifest_path, 'r') as manifest_file: while True: manifest_line = manifest_file.readline() if not manifest_line: @@ -257,7 +258,7 @@ def resume_reading(self): availability_time_str, xml_file_name = manifest_line.rstrip().split(',') xml_file_path = os.path.join(self._dirpath, xml_file_name) xml_content = None - with open(xml_file_path, 'r') as xml_file: + with codecs.open(xml_file_path, 'r') as xml_file: xml_content = xml_file.read() data = [availability_time_str, xml_content] self._custom_consumer.on_new_data(data) From 61ebf213a610db3918bfcf7e7792f2c3690f7fdb Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Wed, 9 May 2018 16:24:12 +0100 Subject: [PATCH 08/14] Allow the first filename counter to be calculated Part of the fix for #481 - allow the filesystem output carriage begin count to be overridden in the ebuttd-encoder using a first document datetime and a document duration. The zeroth document is calculated as the difference between the current datetime and the specified datetime divided by the document duration. --- ebu_tt_live/carriage/filesystem.py | 3 +++ ebu_tt_live/config/clocks.py | 25 +++++++++++++++++++++ ebu_tt_live/config/node.py | 36 ++++++++++++++++++++++++++---- ebu_tt_live/node/encoder.py | 17 +++++++++++++- 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/ebu_tt_live/carriage/filesystem.py b/ebu_tt_live/carriage/filesystem.py index 1b2434570..27303a7c7 100644 --- a/ebu_tt_live/carriage/filesystem.py +++ b/ebu_tt_live/carriage/filesystem.py @@ -98,6 +98,9 @@ def _get_default_clock(self, sequence_identifier, time_base, clock_mode=None): self._default_clocks[sequence_identifier] = clock_obj return clock_obj + def set_message_counter(self, message_counter): + self._msg_counter = message_counter + def check_availability_time( self, sequence_identifier, time_base=None, clock_mode=None, availability_time=None): """ diff --git a/ebu_tt_live/config/clocks.py b/ebu_tt_live/config/clocks.py index 9fed98e78..dcaead7c6 100644 --- a/ebu_tt_live/config/clocks.py +++ b/ebu_tt_live/config/clocks.py @@ -1,5 +1,7 @@ from .common import ConfigurableComponent, Namespace from ebu_tt_live import clocks +from datetime import datetime, timedelta +import re from ebu_tt_live.errors import ConfigurationError from ebu_tt_live.strings import ERR_NO_SUCH_COMPONENT @@ -42,3 +44,26 @@ def get_clock(clock_type): type_name=clock_type ) ) + +def _int_or_none(value): + try: + return int(value) + except TypeError: + return 0 + +_datetime_groups_regex = re.compile('([0-9][0-9][0-9][0-9])-([0-9][0-9])-([0-9][0-9])T([0-9][0-9]):([0-5][0-9]):([0-5][0-9]|60)(?:\.([0-9]+))?') + +def get_date(date): + years, months, days, hours, minutes, seconds, microseconds = map( + lambda x: _int_or_none(x), + _datetime_groups_regex.match(date).groups() + ) + + return datetime( + year = years, + month = months, + day = days, + hour = hours, + minute = minutes, + second = seconds, + microsecond = microseconds) diff --git a/ebu_tt_live/config/node.py b/ebu_tt_live/config/node.py index b484cc63b..c2c7d47c5 100644 --- a/ebu_tt_live/config/node.py +++ b/ebu_tt_live/config/node.py @@ -1,5 +1,5 @@ from .common import ConfigurableComponent, Namespace, converters, RequiredConfig -from .clocks import get_clock +from .clocks import get_clock, get_date from .carriage import get_producer_carriage, get_consumer_carriage from ebu_tt_live import documents from ebu_tt_live import bindings @@ -10,6 +10,8 @@ from ebu_tt_live.errors import ConfigurationError from ebu_tt_live.strings import ERR_CONF_NO_SUCH_NODE from .adapters import ProducerNodeCarriageAdapter, ConsumerNodeCarriageAdapter +from datetime import datetime, timedelta +from math import floor class NodeBase(ConfigurableComponent): @@ -250,10 +252,26 @@ class EBUTTDEncoder(ProducerMixin, ConsumerMixin, NodeBase): required_config = Namespace() required_config.add_option('id', default='ebuttd-encoder') - required_config.add_option('media_time_zero', default='current') - required_config.add_option('default_namespace', default=False) + required_config.add_option( + 'media_time_zero', + default='current', + doc='The clock equivalent time to use for media time zero, defaults to the current time.') + required_config.add_option( + 'default_namespace', + default=False, + doc='Whether to use a default namespace, default false.') required_config.clock = Namespace() required_config.clock.add_option('type', default='local', from_string_converter=get_clock) + required_config.override_begin_count = Namespace() + required_config.override_begin_count.add_option( + 'first_doc_datetime', + doc='The time when the document numbered 1 was available, format YYYY-mm-DDTHH:MM:SS', + default = datetime.utcnow(), + from_string_converter=get_date) + required_config.override_begin_count.add_option( + 'doc_duration', + default=5.0, + doc='The duration of each document in seconds, default 5') _clock = None @@ -263,10 +281,20 @@ def _create_component(self, config): mtz = self._clock.component.get_time() else: mtz = bindings.ebuttdt.LimitedClockTimingType(str(self.config.media_time_zero)).timedelta + + begin_count = None + + if self.config.override_begin_count: + # override the carriage mech's document count + fdt = self.config.override_begin_count.first_doc_datetime + tn = datetime.utcnow() + begin_count = int(floor((tn - fdt).total_seconds() / self.config.override_begin_count.doc_duration)) + self.component = processing_node.EBUTTDEncoder( node_id=self.config.id, media_time_zero=mtz, - default_ns=self.config.default_namespace + default_ns=self.config.default_namespace, + begin_count=begin_count ) def __init__(self, config, local_config): diff --git a/ebu_tt_live/node/encoder.py b/ebu_tt_live/node/encoder.py index 50917715b..ccd6587e4 100644 --- a/ebu_tt_live/node/encoder.py +++ b/ebu_tt_live/node/encoder.py @@ -4,6 +4,8 @@ from ebu_tt_live.clocks.media import MediaClock from ebu_tt_live.documents.converters import EBUTT3EBUTTDConverter from ebu_tt_live.documents import EBUTTDDocument, EBUTT3Document +#from ebu_tt_live.carriage.filesystem import FilesystemProducerImpl +#from ebu_tt_live.carriage import FilesystemProducerImpl class EBUTTDEncoder(AbstractCombinedNode): @@ -13,9 +15,14 @@ class EBUTTDEncoder(AbstractCombinedNode): _default_ebuttd_doc = None _expects = EBUTT3Document _provides = EBUTTDDocument + # _begin_count is used to override the first output document count number. when + # provided as a constructor value it is stored, and set on the output carriage + # impl once before the first time emit_document is called. Then it is reset + # to None, which is used as the test to see if it needs to be used. + _begin_count = None def __init__(self, node_id, media_time_zero, default_ns=False, producer_carriage=None, - consumer_carriage=None, **kwargs): + consumer_carriage=None, begin_count=None, **kwargs): super(EBUTTDEncoder, self).__init__( producer_carriage=producer_carriage, consumer_carriage=consumer_carriage, @@ -25,6 +32,7 @@ def __init__(self, node_id, media_time_zero, default_ns=False, producer_carriage self._default_ns = default_ns media_clock = MediaClock() media_clock.adjust_time(timedelta(), media_time_zero) + self._begin_count = begin_count self._ebuttd_converter = EBUTT3EBUTTDConverter( media_clock=media_clock ) @@ -41,6 +49,13 @@ def process_document(self, document, **kwargs): converted_doc = EBUTTDDocument.create_from_raw_binding( self._ebuttd_converter.convert_document(document.binding) ) + + # If this is the first time, and there's a begin count override, apply it + if self._begin_count is not None: + # Will fail unless the concrete producer carriage impl is a FilesystemProducerImpl + self.producer_carriage.producer_carriage.set_message_counter(self._begin_count) + self._begin_count = None + # Specify the time_base since the FilesystemProducerImpl can't derive it otherwise. # Hard coded to 'media' because that's all that's permitted in EBU-TT-D. Alternative # would be to extract it from the EBUTTDDocument but since it's the only permitted From 8023b7a32068c17b98525f40efa9885243ce6430 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Wed, 9 May 2018 16:24:42 +0100 Subject: [PATCH 09/14] Document new config options Add config options documentation for `override_begin_count` and `begin_count`. --- docs/source/configurator.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/source/configurator.rst b/docs/source/configurator.rst index 93f63c54d..53648283f 100644 --- a/docs/source/configurator.rst +++ b/docs/source/configurator.rst @@ -106,8 +106,11 @@ Node type dependent options for [nodeN] : :: type="ebuttd-encoder" ├─media_time_zero : ["current" (default) | clock time at media time zero TODO: check format] ├─default_namespace : ["false" (default) | "true"] - └─clock - └─type : ["local" (default) | "auto" | "utc"] + ├─clock + │ └─type : ["local" (default) | "auto" | "utc"] + └─override_begin_count : override the counter for the zeroth output document (for filesystem only, beats begin_count) + ├─first_doc_datetime : datetime when first document would have been e.g. 1970-01-01T00:00:00.0 + └─doc_duration : duration in seconds of each document e.g. 3.84 type="buffer-delay" └─delay : delay in seconds, default 0 @@ -135,6 +138,7 @@ Output carriage type dependent options for "carriage": :: ├─rotating_buf : Rotating buffer size. This will keep the last N number of files created in the folder or all if 0, default 0 ├─suppress_manifest : Whether to suppress writing of a manifest file (e.g. for EBU-TT-D output). Default False ├─message_filename_pattern : File name pattern for message documents or EBU-TT-D documents. It can contain {sequence_identifier} and {counter} format parameters, default "{sequence_identifier}_msg_{counter}.xml" + ├─begin_count : value of zeroth {counter} format value: first output file will use this plus 1 - note that ebuttd-encoder can override this. └─filename_pattern : File name pattern for EBU-TT-Live documents. It needs to contain {counter} format parameter, which will be populated with the sequence number. Default "{sequence_identifier}_{counter}.xml" type="websocket" From a8eb4b8926571ef1bff3339adb658355b21c5b02 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Wed, 9 May 2018 18:38:13 +0100 Subject: [PATCH 10/14] Fix for datetimes with timezone e.g. Z at the end timezone is ignored, with warning if present and not Z, since UTC is assumed. --- ebu_tt_live/config/clocks.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ebu_tt_live/config/clocks.py b/ebu_tt_live/config/clocks.py index dcaead7c6..6ed271752 100644 --- a/ebu_tt_live/config/clocks.py +++ b/ebu_tt_live/config/clocks.py @@ -1,3 +1,4 @@ +import logging from .common import ConfigurableComponent, Namespace from ebu_tt_live import clocks from datetime import datetime, timedelta @@ -6,6 +7,9 @@ from ebu_tt_live.strings import ERR_NO_SUCH_COMPONENT +log = logging.getLogger(__name__) + + class LocalMachineClock(ConfigurableComponent): def __init__(self, config, local_config): @@ -45,19 +49,24 @@ def get_clock(clock_type): ) ) -def _int_or_none(value): +def _int_or_zero(value): try: return int(value) except TypeError: return 0 -_datetime_groups_regex = re.compile('([0-9][0-9][0-9][0-9])-([0-9][0-9])-([0-9][0-9])T([0-9][0-9]):([0-5][0-9]):([0-5][0-9]|60)(?:\.([0-9]+))?') +_datetime_groups_regex = re.compile('([0-9][0-9][0-9][0-9])-([0-9][0-9])-([0-9][0-9])T([0-9][0-9]):([0-5][0-9]):([0-5][0-9]|60)(?:\.([0-9]+))?([A-Z]+)?') def get_date(date): + m = _datetime_groups_regex.match(date) years, months, days, hours, minutes, seconds, microseconds = map( - lambda x: _int_or_none(x), - _datetime_groups_regex.match(date).groups() + lambda x: _int_or_zero(x), + m.group(1, 2, 3, 4, 5, 6, 7) ) + + tz = m.group(8) + if tz is not None and tz != 'Z': + log.warning('Ignoring provided timezone {}'.format(tz)) return datetime( year = years, @@ -66,4 +75,5 @@ def get_date(date): hour = hours, minute = minutes, second = seconds, - microsecond = microseconds) + microsecond = microseconds + ) From 9dc18acdccef5816e6f18a3b94535531094ce407 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Wed, 9 May 2018 18:41:30 +0100 Subject: [PATCH 11/14] Allow a remote URL to be specified for fetching the time Pass a URL through if one is provided. Fix a problem where the wrong media time was being used when the zero time was being set - needs to be based on a date as well as a time. --- ebu_tt_live/config/node.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ebu_tt_live/config/node.py b/ebu_tt_live/config/node.py index c2c7d47c5..e54f6a1ad 100644 --- a/ebu_tt_live/config/node.py +++ b/ebu_tt_live/config/node.py @@ -262,12 +262,12 @@ class EBUTTDEncoder(ProducerMixin, ConsumerMixin, NodeBase): doc='Whether to use a default namespace, default false.') required_config.clock = Namespace() required_config.clock.add_option('type', default='local', from_string_converter=get_clock) + required_config.clock.add_option('url', doc='URL from which to fetch remote time.') required_config.override_begin_count = Namespace() required_config.override_begin_count.add_option( 'first_doc_datetime', doc='The time when the document numbered 1 was available, format YYYY-mm-DDTHH:MM:SS', - default = datetime.utcnow(), - from_string_converter=get_date) + default = datetime.utcnow()) required_config.override_begin_count.add_option( 'doc_duration', default=5.0, @@ -278,15 +278,17 @@ class EBUTTDEncoder(ProducerMixin, ConsumerMixin, NodeBase): def _create_component(self, config): self._clock = self.config.clock.type(config, self.config.clock) if self.config.media_time_zero == 'current': + # should use URL? mtz = self._clock.component.get_time() else: - mtz = bindings.ebuttdt.LimitedClockTimingType(str(self.config.media_time_zero)).timedelta + mtz = get_date(self.config.media_time_zero)-datetime.min begin_count = None if self.config.override_begin_count: # override the carriage mech's document count fdt = self.config.override_begin_count.first_doc_datetime + # should use mtz? tn = datetime.utcnow() begin_count = int(floor((tn - fdt).total_seconds() / self.config.override_begin_count.doc_duration)) @@ -294,7 +296,8 @@ def _create_component(self, config): node_id=self.config.id, media_time_zero=mtz, default_ns=self.config.default_namespace, - begin_count=begin_count + begin_count=begin_count, + clock_url=self.config.clock.url ) def __init__(self, config, local_config): From 6d09b3bff341ef3aac3189099331b555b5a9cf0c Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Wed, 9 May 2018 18:42:51 +0100 Subject: [PATCH 12/14] Fetch remote time from provided URL If a URL is provided for the clock, make a remote request to get it and process the string, and use it to adjust the media clock. --- ebu_tt_live/node/encoder.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/ebu_tt_live/node/encoder.py b/ebu_tt_live/node/encoder.py index ccd6587e4..cfd0260d0 100644 --- a/ebu_tt_live/node/encoder.py +++ b/ebu_tt_live/node/encoder.py @@ -1,11 +1,14 @@ - -from datetime import timedelta +import logging +from datetime import timedelta, datetime from .base import AbstractCombinedNode from ebu_tt_live.clocks.media import MediaClock from ebu_tt_live.documents.converters import EBUTT3EBUTTDConverter from ebu_tt_live.documents import EBUTTDDocument, EBUTT3Document -#from ebu_tt_live.carriage.filesystem import FilesystemProducerImpl -#from ebu_tt_live.carriage import FilesystemProducerImpl +from ebu_tt_live.config.clocks import get_date +import requests + + +log = logging.getLogger(__name__) class EBUTTDEncoder(AbstractCombinedNode): @@ -22,7 +25,7 @@ class EBUTTDEncoder(AbstractCombinedNode): _begin_count = None def __init__(self, node_id, media_time_zero, default_ns=False, producer_carriage=None, - consumer_carriage=None, begin_count=None, **kwargs): + consumer_carriage=None, begin_count=None, clock_url=None, **kwargs): super(EBUTTDEncoder, self).__init__( producer_carriage=producer_carriage, consumer_carriage=consumer_carriage, @@ -31,7 +34,15 @@ def __init__(self, node_id, media_time_zero, default_ns=False, producer_carriage ) self._default_ns = default_ns media_clock = MediaClock() - media_clock.adjust_time(timedelta(), media_time_zero) + if clock_url is None: + media_clock.adjust_time(timedelta(), media_time_zero) + else: + log.info('Getting time from {}'.format(clock_url)) + r = requests.get(clock_url).text + log.info('Got response {}'.format(r)) + d = get_date(r) + t = d - datetime.min + media_clock.adjust_time(t, media_time_zero) self._begin_count = begin_count self._ebuttd_converter = EBUTT3EBUTTDConverter( media_clock=media_clock From 657ef829ef20b511b5be30eca58b0f7f8ce0cc7c Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Wed, 9 May 2018 19:00:00 +0100 Subject: [PATCH 13/14] Config docs update Form media_time_zero format and for clock url. --- docs/source/configurator.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/configurator.rst b/docs/source/configurator.rst index 53648283f..ff908ddda 100644 --- a/docs/source/configurator.rst +++ b/docs/source/configurator.rst @@ -104,10 +104,11 @@ Node type dependent options for [nodeN] : :: └─type : ["local" (default) | "auto" | "clock"] type="ebuttd-encoder" - ├─media_time_zero : ["current" (default) | clock time at media time zero TODO: check format] + ├─media_time_zero : ["current" (default) | clock time at media time zero in ISO format] ├─default_namespace : ["false" (default) | "true"] ├─clock - │ └─type : ["local" (default) | "auto" | "utc"] + │ ├─type : ["local" (default) | "auto" | "utc"] + │ └─url : url of source of time - expected to provide a single ISO format time string └─override_begin_count : override the counter for the zeroth output document (for filesystem only, beats begin_count) ├─first_doc_datetime : datetime when first document would have been e.g. 1970-01-01T00:00:00.0 └─doc_duration : duration in seconds of each document e.g. 3.84 From 7e33412a217cc02941e9cac2705c1676405edfe5 Mon Sep 17 00:00:00 2001 From: nigelmegitt Date: Thu, 10 May 2018 12:25:15 +0100 Subject: [PATCH 14/14] Only take the days from the reference remote... The times are provided by the input `real_clock_timedelta`. Fixes a big offset issue. --- ebu_tt_live/clocks/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebu_tt_live/clocks/media.py b/ebu_tt_live/clocks/media.py index 8ee08c714..979b5a1db 100644 --- a/ebu_tt_live/clocks/media.py +++ b/ebu_tt_live/clocks/media.py @@ -25,7 +25,7 @@ def get_machine_time(self): return current_time def get_media_time(self, real_clock_timedelta): - return self._reference_mapping.remote + (real_clock_timedelta - self._reference_mapping.local) + return timedelta(days=self._reference_mapping.remote.days) + (real_clock_timedelta - self._reference_mapping.local) def get_real_clock_time(self): return self._reference_mapping.remote + (self.get_machine_time() - self._reference_mapping.local)