From eed738ad09c59de6a6d616e4a0e6150012a56503 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 17 Dec 2017 14:49:34 +0900 Subject: [PATCH 01/40] Improve merge_results() --- rplugin/python3/deoplete/deoplete.py | 134 ++++++++++++++------------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index ab583122..387dbbb7 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -62,7 +62,7 @@ def completion_begin(self, context): try: is_async, complete_position, candidates = self.merge_results( - self.gather_results(context), context['input']) + self.gather_results(context), context) except Exception: error_tb(self._vim, 'Error while gathering completions') @@ -192,73 +192,80 @@ def gather_async_results(self, result, source): else: error_tb(self._vim, 'Errors from: %s' % source.name) - def merge_results(self, results, context_input): - merged_results = [] - all_candidates = [] - for result in [x for x in results - if not self.is_skip(x['context'], x['source'])]: - source = result['source'] + def process_filter(self, f, context): + try: + self.profile_start(context, f.name) + if (isinstance(context['candidates'], dict) and + 'sorted_candidates' in context['candidates']): + context_candidates = [] + context['is_sorted'] = True + for candidates in context['candidates']['sorted_candidates']: + context['candidates'] = candidates + context_candidates += f.filter(context) + context['candidates'] = context_candidates + else: + context['candidates'] = f.filter(context) + self.profile_end(f.name) + except Exception: + self._filter_errors[f.name] += 1 + if self._source_errors[f.name] > 2: + error(self._vim, 'Too many errors from "%s". ' + 'This filter is disabled until Neovim ' + 'is restarted.' % f.name) + self._filters.pop(f.name) + return + error_tb(self._vim, 'Errors from: %s' % f) - # Gather async results - if result['is_async']: - self.gather_async_results(result, source) + def source_result(self, result, context_input): + source = result['source'] - if not result['context']['candidates']: - continue + # Gather async results + if result['is_async']: + self.gather_async_results(result, source) - context = copy.deepcopy(result['context']) + if not result['context']['candidates']: + return [] - context['input'] = context_input - context['complete_str'] = context['input'][ - context['char_position']:] - context['is_sorted'] = False + # Source context + ctx = copy.deepcopy(result['context']) - # Filtering - ignorecase = context['ignorecase'] - smartcase = context['smartcase'] - camelcase = context['camelcase'] + ctx['input'] = context_input + ctx['complete_str'] = context_input[ctx['char_position']:] + ctx['is_sorted'] = False - # Set ignorecase - if (smartcase or camelcase) and re.search( - r'[A-Z]', context['complete_str']): - context['ignorecase'] = 0 + # Filtering + ignorecase = ctx['ignorecase'] + smartcase = ctx['smartcase'] + camelcase = ctx['camelcase'] - for f in [self._filters[x] for x - in source.matchers + source.sorters + source.converters - if x in self._filters]: - try: - self.profile_start(context, f.name) - if (isinstance(context['candidates'], dict) and - 'sorted_candidates' in context['candidates']): - context_candidates = [] - sorted_candidates = context['candidates'][ - 'sorted_candidates'] - context['is_sorted'] = True - for candidates in sorted_candidates: - context['candidates'] = candidates - context_candidates += f.filter(context) - context['candidates'] = context_candidates - else: - context['candidates'] = f.filter(context) - self.profile_end(f.name) - except Exception: - self._filter_errors[f.name] += 1 - if self._source_errors[f.name] > 2: - error(self._vim, 'Too many errors from "%s". ' - 'This filter is disabled until Neovim ' - 'is restarted.' % f.name) - self._filters.pop(f.name) - continue - error_tb(self._vim, 'Errors from: %s' % f) - - context['ignorecase'] = ignorecase - - # On post filter - if hasattr(source, 'on_post_filter'): - context['candidates'] = source.on_post_filter(context) - - if context['candidates']: - merged_results.append([context['candidates'], result]) + # Set ignorecase + if (smartcase or camelcase) and re.search( + r'[A-Z]', ctx['complete_str']): + ctx['ignorecase'] = 0 + + for f in [self._filters[x] for x + in source.matchers + source.sorters + source.converters + if x in self._filters]: + self.process_filter(f, ctx) + + ctx['ignorecase'] = ignorecase + + # On post filter + if hasattr(source, 'on_post_filter'): + ctx['candidates'] = source.on_post_filter(ctx) + + if ctx['candidates']: + return [ctx['candidates'], result] + return [] + + def merge_results(self, results, context): + merged_results = [] + all_candidates = [] + for result in [x for x in results + if not self.is_skip(x['context'], x['source'])]: + source_result = self.source_result(result, context['input']) + if source_result: + merged_results.append(source_result) is_async = len([x for x in results if x['context']['is_async']]) > 0 @@ -269,10 +276,9 @@ def merge_results(self, results, context_input): for x in merged_results]) for [candidates, result] in merged_results: - context = result['context'] + ctx = result['context'] source = result['source'] - prefix = context['input'][ - complete_position:context['complete_position']] + prefix = ctx['input'][complete_position:ctx['complete_position']] mark = source.mark + ' ' for candidate in candidates: From a91ea6af6720795242d679b4c02adcbc1a8cba93 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 17 Dec 2017 18:01:06 +0900 Subject: [PATCH 02/40] WIP: add child.py --- rplugin/python3/deoplete/child.py | 385 +++++++++++++++++++++++++++ rplugin/python3/deoplete/deoplete.py | 7 +- 2 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 rplugin/python3/deoplete/child.py diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py new file mode 100644 index 00000000..a4491b66 --- /dev/null +++ b/rplugin/python3/deoplete/child.py @@ -0,0 +1,385 @@ +# ============================================================================ +# FILE: child.py +# AUTHOR: Shougo Matsushita +# License: MIT license +# ============================================================================ + +import re +import copy +import time +import os.path + +from collections import defaultdict + +from deoplete import logger +from deoplete.exceptions import SourceInitError +from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, + find_rplugins, get_buffer_config, get_custom, + get_syn_names, import_plugin, convert2candidates) + + +class Child(logger.LoggingMixin): + + def __init__(self, vim): + self.name = 'child' + + self._vim = vim + self._filters = {} + self._sources = {} + self._custom = [] + self._profile_flag = None + self._profile_start = 0 + self._source_errors = defaultdict(int) + self._filter_errors = defaultdict(int) + self._prev_results = {} + + def enable_logging(self, context): + self.is_debug_enabled = True + + def gather_results(self, context): + results = [] + + for source in [x[1] for x in self.itersource(context)]: + try: + if source.disabled_syntaxes and 'syntax_names' not in context: + context['syntax_names'] = get_syn_names(self._vim) + ctx = copy.deepcopy(context) + + charpos = source.get_complete_position(ctx) + if charpos >= 0 and source.is_bytepos: + charpos = bytepos2charpos( + ctx['encoding'], ctx['input'], charpos) + + ctx['char_position'] = charpos + ctx['complete_position'] = charpos2bytepos( + ctx['encoding'], ctx['input'], charpos) + ctx['complete_str'] = ctx['input'][ctx['char_position']:] + + if charpos < 0 or self.is_skip(ctx, source): + if source.name in self._prev_results: + self._prev_results.pop(source.name) + # Skip + continue + + if (source.name in self._prev_results and + self.use_previous_result( + context, self._prev_results[source.name], + source.is_volatile)): + results.append(self._prev_results[source.name]) + continue + + ctx['is_async'] = False + ctx['is_refresh'] = True + ctx['max_abbr_width'] = min(source.max_abbr_width, + ctx['max_abbr_width']) + ctx['max_kind_width'] = min(source.max_kind_width, + ctx['max_kind_width']) + ctx['max_menu_width'] = min(source.max_menu_width, + ctx['max_menu_width']) + if ctx['max_abbr_width'] > 0: + ctx['max_abbr_width'] = max(20, ctx['max_abbr_width']) + if ctx['max_kind_width'] > 0: + ctx['max_kind_width'] = max(10, ctx['max_kind_width']) + if ctx['max_menu_width'] > 0: + ctx['max_menu_width'] = max(10, ctx['max_menu_width']) + + # Gathering + self.profile_start(ctx, source.name) + ctx['candidates'] = source.gather_candidates(ctx) + self.profile_end(source.name) + + if ctx['candidates'] is None: + continue + + ctx['candidates'] = convert2candidates(ctx['candidates']) + + result = { + 'name': source.name, + 'source': source, + 'context': ctx, + 'is_async': ctx['is_async'], + 'prev_linenr': ctx['position'][1], + 'prev_input': ctx['input'], + } + self._prev_results[source.name] = result + results.append(result) + except Exception: + self._source_errors[source.name] += 1 + if source.is_silent: + continue + if self._source_errors[source.name] > 2: + error(self._vim, 'Too many errors from "%s". ' + 'This source is disabled until Neovim ' + 'is restarted.' % source.name) + self._sources.pop(source.name) + continue + error_tb(self._vim, 'Errors from: %s' % source.name) + + return results + + def gather_async_results(self, result, source): + try: + result['context']['is_refresh'] = False + async_candidates = source.gather_candidates(result['context']) + result['is_async'] = result['context']['is_async'] + if async_candidates is None: + return + result['context']['candidates'] += convert2candidates( + async_candidates) + except Exception: + self._source_errors[source.name] += 1 + if source.is_silent: + return + if self._source_errors[source.name] > 2: + error(self._vim, 'Too many errors from "%s". ' + 'This source is disabled until Neovim ' + 'is restarted.' % source.name) + self._sources.pop(source.name) + else: + error_tb(self._vim, 'Errors from: %s' % source.name) + + def process_filter(self, f, context): + try: + self.profile_start(context, f.name) + if (isinstance(context['candidates'], dict) and + 'sorted_candidates' in context['candidates']): + context_candidates = [] + context['is_sorted'] = True + for candidates in context['candidates']['sorted_candidates']: + context['candidates'] = candidates + context_candidates += f.filter(context) + context['candidates'] = context_candidates + else: + context['candidates'] = f.filter(context) + self.profile_end(f.name) + except Exception: + self._filter_errors[f.name] += 1 + if self._source_errors[f.name] > 2: + error(self._vim, 'Too many errors from "%s". ' + 'This filter is disabled until Neovim ' + 'is restarted.' % f.name) + self._filters.pop(f.name) + return + error_tb(self._vim, 'Errors from: %s' % f) + + def source_result(self, result, context_input): + source = result['source'] + + # Gather async results + if result['is_async']: + self.gather_async_results(result, source) + + if not result['context']['candidates']: + return [] + + # Source context + ctx = copy.deepcopy(result['context']) + + ctx['input'] = context_input + ctx['complete_str'] = context_input[ctx['char_position']:] + ctx['is_sorted'] = False + + # Filtering + ignorecase = ctx['ignorecase'] + smartcase = ctx['smartcase'] + camelcase = ctx['camelcase'] + + # Set ignorecase + if (smartcase or camelcase) and re.search( + r'[A-Z]', ctx['complete_str']): + ctx['ignorecase'] = 0 + + for f in [self._filters[x] for x + in source.matchers + source.sorters + source.converters + if x in self._filters]: + self.process_filter(f, ctx) + + ctx['ignorecase'] = ignorecase + + # On post filter + if hasattr(source, 'on_post_filter'): + ctx['candidates'] = source.on_post_filter(ctx) + + if ctx['candidates']: + return [ctx['candidates'], result] + return [] + + def merge_results(self, context): + results = self.gather_results(context) + + merged_results = [] + for result in [x for x in results + if not self.is_skip(x['context'], x['source'])]: + source_result = self.source_result(result, context['input']) + if source_result: + merged_results.append(source_result) + + is_async = len([x for x in results if x['context']['is_async']]) > 0 + + return (is_async, merged_results) + + def itersource(self, context): + sources = sorted(self._sources.items(), + key=lambda x: get_custom( + context['custom'], + x[1].name, 'rank', x[1].rank), + reverse=True) + filetypes = context['filetypes'] + ignore_sources = set() + for ft in filetypes: + ignore_sources.update( + get_buffer_config(context, ft, + 'deoplete_ignore_sources', + 'deoplete#ignore_sources', + {})) + + for source_name, source in sources: + if source.limit > 0 and context['bufsize'] > source.limit: + continue + if source.filetypes is None or source_name in ignore_sources: + continue + if context['sources'] and source_name not in context['sources']: + continue + if source.filetypes and not any(x in filetypes + for x in source.filetypes): + continue + if not source.is_initialized and hasattr(source, 'on_init'): + self.debug('on_init Source: %s', source.name) + try: + source.on_init(context) + except Exception as exc: + if isinstance(exc, SourceInitError): + error(self._vim, + 'Error when loading source {}: {}. ' + 'Ignoring.'.format(source_name, exc)) + else: + error_tb(self._vim, + 'Error when loading source {}: {}. ' + 'Ignoring.'.format(source_name, exc)) + self._sources.pop(source_name) + continue + else: + source.is_initialized = True + yield source_name, source + + def profile_start(self, context, name): + if self._profile_flag is 0 or not self.is_debug_enabled: + return + + if not self._profile_flag: + self._profile_flag = context['vars']['deoplete#enable_profile'] + if self._profile_flag: + return self.profile_start(context, name) + elif self._profile_flag: + self.debug('Profile Start: {0}'.format(name)) + self._profile_start = time.clock() + + def profile_end(self, name): + if self._profile_start: + self.debug('Profile End : {0:<25} time={1:2.10f}'.format( + name, time.clock() - self._profile_start)) + + def add_source(self, s): + self._sources[s.name] = s + + def add_filter(self, f): + self._filters[f.name] = f + + def set_custom(self, custom): + self._custom = custom + + def load_filters(self, context): + # Load filters from runtimepath + for path in find_rplugins(context, 'filter'): + if path in self._loaded_paths: + continue + self._loaded_paths.add(path) + + name = os.path.splitext(os.path.basename(path))[0] + + f = None + try: + Filter = import_plugin(path, 'filter', 'Filter') + if not Filter: + continue + + f = Filter(self._vim) + f.name = getattr(f, 'name', name) + f.path = path + self._filters[f.name] = f + except Exception: + # Exception occurred when loading a filter. Log stack trace. + error_tb(self._vim, 'Could not load filter: %s' % name) + finally: + if f: + self._filters[f.name] = f + self.debug('Loaded Filter: %s (%s)', f.name, path) + + def use_previous_result(self, context, result, is_volatile): + if context['position'][1] != result['prev_linenr']: + return False + if is_volatile: + return context['input'] == result['prev_input'] + else: + return (re.sub(r'\w*$', '', context['input']) == + re.sub(r'\w*$', '', result['prev_input']) and + context['input'].find(result['prev_input']) == 0) + + def is_skip(self, context, source): + if 'syntax_names' in context and source.disabled_syntaxes: + p = re.compile('(' + '|'.join(source.disabled_syntaxes) + ')$') + if next(filter(p.search, context['syntax_names']), None): + return True + if (source.input_pattern != '' and + re.search('(' + source.input_pattern + ')$', + context['input'])): + return False + if context['event'] == 'Manual': + return False + return not (source.min_pattern_length <= + len(context['complete_str']) <= source.max_pattern_length) + + def set_source_attributes(self, context): + """Set source attributes from the context. + + Each item in `attrs` is the attribute name. If the default value is in + context['vars'] under a different name, use a tuple. + """ + attrs = ( + 'filetypes', + 'disabled_syntaxes', + 'input_pattern', + ('min_pattern_length', 'deoplete#auto_complete_start_length'), + 'max_pattern_length', + ('max_abbr_width', 'deoplete#max_abbr_width'), + ('max_kind_width', 'deoplete#max_menu_width'), + ('max_menu_width', 'deoplete#max_menu_width'), + 'matchers', + 'sorters', + 'converters', + 'mark', + 'is_debug_enabled', + 'is_silent', + ) + + for name, source in self._sources.items(): + for attr in attrs: + if isinstance(attr, tuple): + default_val = context['vars'][attr[1]] + attr = attr[0] + else: + default_val = None + source_attr = getattr(source, attr, default_val) + setattr(source, attr, get_custom(context['custom'], + name, attr, source_attr)) + + def on_event(self, context): + for source_name, source in self.itersource(context): + if hasattr(source, 'on_event'): + self.debug('on_event: Source: %s', source_name) + try: + source.on_event(context) + except Exception as exc: + error_tb(self._vim, 'Exception during {}.on_event ' + 'for event {!r}: {}'.format( + source_name, context['event'], exc)) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 387dbbb7..2839415d 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -3,6 +3,7 @@ # AUTHOR: Shougo Matsushita # License: MIT license # ============================================================================ + import re import copy import time @@ -16,6 +17,7 @@ from deoplete import logger from deoplete.exceptions import SourceInitError +from deoplete.child import Child from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, find_rplugins, get_buffer_config, get_custom, get_syn_names, import_plugin, convert2candidates) @@ -24,6 +26,8 @@ class Deoplete(logger.LoggingMixin): def __init__(self, vim): + self.name = 'core' + self._vim = vim self._filters = {} self._sources = {} @@ -33,9 +37,9 @@ def __init__(self, vim): self._profile_start = 0 self._source_errors = defaultdict(int) self._filter_errors = defaultdict(int) - self.name = 'core' self._loaded_paths = set() self._prev_results = {} + self._children = [Child(vim)] # Enable logging before "Init" for more information, and e.g. # deoplete-jedi picks up the log filename from deoplete's handler in @@ -409,7 +413,6 @@ def load_filters(self, context): f = Filter(self._vim) f.name = getattr(f, 'name', name) f.path = path - self._filters[f.name] = f except Exception: # Exception occurred when loading a filter. Log stack trace. error_tb(self._vim, 'Could not load filter: %s' % name) From 19431f63e255fe62ee588b64142b583c8817665b Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 24 Dec 2017 16:28:15 +0900 Subject: [PATCH 03/40] Splitted to child.py --- rplugin/python3/deoplete/child.py | 32 +-- rplugin/python3/deoplete/deoplete.py | 336 ++------------------------- 2 files changed, 26 insertions(+), 342 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index a4491b66..1a95a465 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -7,15 +7,14 @@ import re import copy import time -import os.path from collections import defaultdict from deoplete import logger from deoplete.exceptions import SourceInitError from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, - find_rplugins, get_buffer_config, get_custom, - get_syn_names, import_plugin, convert2candidates) + get_buffer_config, get_custom, + get_syn_names, convert2candidates) class Child(logger.LoggingMixin): @@ -288,33 +287,6 @@ def add_filter(self, f): def set_custom(self, custom): self._custom = custom - def load_filters(self, context): - # Load filters from runtimepath - for path in find_rplugins(context, 'filter'): - if path in self._loaded_paths: - continue - self._loaded_paths.add(path) - - name = os.path.splitext(os.path.basename(path))[0] - - f = None - try: - Filter = import_plugin(path, 'filter', 'Filter') - if not Filter: - continue - - f = Filter(self._vim) - f.name = getattr(f, 'name', name) - f.path = path - self._filters[f.name] = f - except Exception: - # Exception occurred when loading a filter. Log stack trace. - error_tb(self._vim, 'Could not load filter: %s' % name) - finally: - if f: - self._filters[f.name] = f - self.debug('Loaded Filter: %s (%s)', f.name, path) - def use_previous_result(self, context, result, is_volatile): if context['position'][1] != result['prev_linenr']: return False diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 2839415d..b007facf 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -4,23 +4,16 @@ # License: MIT license # ============================================================================ -import re -import copy import time import os.path -from collections import defaultdict - import deoplete.util # noqa import deoplete.filter # noqa import deoplete.source # noqa from deoplete import logger -from deoplete.exceptions import SourceInitError from deoplete.child import Child -from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, - find_rplugins, get_buffer_config, get_custom, - get_syn_names, import_plugin, convert2candidates) +from deoplete.util import (error_tb, find_rplugins, import_plugin) class Deoplete(logger.LoggingMixin): @@ -29,16 +22,11 @@ def __init__(self, vim): self.name = 'core' self._vim = vim - self._filters = {} - self._sources = {} self._runtimepath = '' self._custom = [] self._profile_flag = None self._profile_start = 0 - self._source_errors = defaultdict(int) - self._filter_errors = defaultdict(int) self._loaded_paths = set() - self._prev_results = {} self._children = [Child(vim)] # Enable logging before "Init" for more information, and e.g. @@ -65,14 +53,12 @@ def completion_begin(self, context): self.check_recache(context) try: - is_async, complete_position, candidates = self.merge_results( - self.gather_results(context), context) - + is_async, position, candidates = self.merge_results(context) except Exception: error_tb(self._vim, 'Error while gathering completions') is_async = False - complete_position = -1 + position = -1 candidates = [] if is_async: @@ -87,191 +73,15 @@ def completion_begin(self, context): # error(self._vim, context['input']) # error(self._vim, candidates) self._vim.vars['deoplete#_context'] = { - 'complete_position': complete_position, + 'complete_position': position, 'candidates': candidates, 'event': context['event'], 'input': context['input'], } self._vim.call('deoplete#handler#_completion_timer_start') - def gather_results(self, context): - results = [] - - for source in [x[1] for x in self.itersource(context)]: - try: - if source.disabled_syntaxes and 'syntax_names' not in context: - context['syntax_names'] = get_syn_names(self._vim) - ctx = copy.deepcopy(context) - - charpos = source.get_complete_position(ctx) - if charpos >= 0 and source.is_bytepos: - charpos = bytepos2charpos( - ctx['encoding'], ctx['input'], charpos) - - ctx['char_position'] = charpos - ctx['complete_position'] = charpos2bytepos( - ctx['encoding'], ctx['input'], charpos) - ctx['complete_str'] = ctx['input'][ctx['char_position']:] - - if charpos < 0 or self.is_skip(ctx, source): - if source.name in self._prev_results: - self._prev_results.pop(source.name) - # Skip - continue - - if (source.name in self._prev_results and - self.use_previous_result( - context, self._prev_results[source.name], - source.is_volatile)): - results.append(self._prev_results[source.name]) - continue - - ctx['is_async'] = False - ctx['is_refresh'] = True - ctx['max_abbr_width'] = min(source.max_abbr_width, - ctx['max_abbr_width']) - ctx['max_kind_width'] = min(source.max_kind_width, - ctx['max_kind_width']) - ctx['max_menu_width'] = min(source.max_menu_width, - ctx['max_menu_width']) - if ctx['max_abbr_width'] > 0: - ctx['max_abbr_width'] = max(20, ctx['max_abbr_width']) - if ctx['max_kind_width'] > 0: - ctx['max_kind_width'] = max(10, ctx['max_kind_width']) - if ctx['max_menu_width'] > 0: - ctx['max_menu_width'] = max(10, ctx['max_menu_width']) - - # Gathering - self.profile_start(ctx, source.name) - ctx['candidates'] = source.gather_candidates(ctx) - self.profile_end(source.name) - - if ctx['candidates'] is None: - continue - - ctx['candidates'] = convert2candidates(ctx['candidates']) - - result = { - 'name': source.name, - 'source': source, - 'context': ctx, - 'is_async': ctx['is_async'], - 'prev_linenr': ctx['position'][1], - 'prev_input': ctx['input'], - } - self._prev_results[source.name] = result - results.append(result) - except Exception: - self._source_errors[source.name] += 1 - if source.is_silent: - continue - if self._source_errors[source.name] > 2: - error(self._vim, 'Too many errors from "%s". ' - 'This source is disabled until Neovim ' - 'is restarted.' % source.name) - self._sources.pop(source.name) - continue - error_tb(self._vim, 'Errors from: %s' % source.name) - - return results - - def gather_async_results(self, result, source): - try: - result['context']['is_refresh'] = False - async_candidates = source.gather_candidates(result['context']) - result['is_async'] = result['context']['is_async'] - if async_candidates is None: - return - result['context']['candidates'] += convert2candidates( - async_candidates) - except Exception: - self._source_errors[source.name] += 1 - if source.is_silent: - return - if self._source_errors[source.name] > 2: - error(self._vim, 'Too many errors from "%s". ' - 'This source is disabled until Neovim ' - 'is restarted.' % source.name) - self._sources.pop(source.name) - else: - error_tb(self._vim, 'Errors from: %s' % source.name) - - def process_filter(self, f, context): - try: - self.profile_start(context, f.name) - if (isinstance(context['candidates'], dict) and - 'sorted_candidates' in context['candidates']): - context_candidates = [] - context['is_sorted'] = True - for candidates in context['candidates']['sorted_candidates']: - context['candidates'] = candidates - context_candidates += f.filter(context) - context['candidates'] = context_candidates - else: - context['candidates'] = f.filter(context) - self.profile_end(f.name) - except Exception: - self._filter_errors[f.name] += 1 - if self._source_errors[f.name] > 2: - error(self._vim, 'Too many errors from "%s". ' - 'This filter is disabled until Neovim ' - 'is restarted.' % f.name) - self._filters.pop(f.name) - return - error_tb(self._vim, 'Errors from: %s' % f) - - def source_result(self, result, context_input): - source = result['source'] - - # Gather async results - if result['is_async']: - self.gather_async_results(result, source) - - if not result['context']['candidates']: - return [] - - # Source context - ctx = copy.deepcopy(result['context']) - - ctx['input'] = context_input - ctx['complete_str'] = context_input[ctx['char_position']:] - ctx['is_sorted'] = False - - # Filtering - ignorecase = ctx['ignorecase'] - smartcase = ctx['smartcase'] - camelcase = ctx['camelcase'] - - # Set ignorecase - if (smartcase or camelcase) and re.search( - r'[A-Z]', ctx['complete_str']): - ctx['ignorecase'] = 0 - - for f in [self._filters[x] for x - in source.matchers + source.sorters + source.converters - if x in self._filters]: - self.process_filter(f, ctx) - - ctx['ignorecase'] = ignorecase - - # On post filter - if hasattr(source, 'on_post_filter'): - ctx['candidates'] = source.on_post_filter(ctx) - - if ctx['candidates']: - return [ctx['candidates'], result] - return [] - - def merge_results(self, results, context): - merged_results = [] - all_candidates = [] - for result in [x for x in results - if not self.is_skip(x['context'], x['source'])]: - source_result = self.source_result(result, context['input']) - if source_result: - merged_results.append(source_result) - - is_async = len([x for x in results if x['context']['is_async']]) > 0 + def merge_results(self, context): + (is_async, merged_results) = self._children[0].merge_results(context) if not merged_results: return (is_async, -1, []) @@ -279,6 +89,7 @@ def merge_results(self, results, context): complete_position = min([x[1]['context']['complete_position'] for x in merged_results]) + all_candidates = [] for [candidates, result] in merged_results: ctx = result['context'] source = result['source'] @@ -300,56 +111,12 @@ def merge_results(self, results, context): all_candidates += candidates # self.debug(candidates) - if context['vars']['deoplete#max_list'] > 0: - all_candidates = all_candidates[ - : context['vars']['deoplete#max_list']] + max_list = context['vars']['deoplete#max_list'] + if max_list > 0: + all_candidates = all_candidates[: max_list] return (is_async, complete_position, all_candidates) - def itersource(self, context): - sources = sorted(self._sources.items(), - key=lambda x: get_custom( - context['custom'], - x[1].name, 'rank', x[1].rank), - reverse=True) - filetypes = context['filetypes'] - ignore_sources = set() - for ft in filetypes: - ignore_sources.update( - get_buffer_config(context, ft, - 'deoplete_ignore_sources', - 'deoplete#ignore_sources', - {})) - - for source_name, source in sources: - if source.limit > 0 and context['bufsize'] > source.limit: - continue - if source.filetypes is None or source_name in ignore_sources: - continue - if context['sources'] and source_name not in context['sources']: - continue - if source.filetypes and not any(x in filetypes - for x in source.filetypes): - continue - if not source.is_initialized and hasattr(source, 'on_init'): - self.debug('on_init Source: %s', source.name) - try: - source.on_init(context) - except Exception as exc: - if isinstance(exc, SourceInitError): - error(self._vim, - 'Error when loading source {}: {}. ' - 'Ignoring.'.format(source_name, exc)) - else: - error_tb(self._vim, - 'Error when loading source {}: {}. ' - 'Ignoring.'.format(source_name, exc)) - self._sources.pop(source_name) - continue - else: - source.is_initialized = True - yield source_name, source - def profile_start(self, context, name): if self._profile_flag is 0 or not self.is_debug_enabled: return @@ -389,11 +156,11 @@ def load_sources(self, context): error_tb(self._vim, 'Could not load source: %s' % name) finally: if source: - self._sources[source.name] = source + self._children[0].add_source(source) self.debug('Loaded Source: %s (%s)', source.name, path) self.set_source_attributes(context) - self._custom = context['custom'] + self.set_custom(context) def load_filters(self, context): # Load filters from runtimepath @@ -418,66 +185,18 @@ def load_filters(self, context): error_tb(self._vim, 'Could not load filter: %s' % name) finally: if f: - self._filters[f.name] = f + for child in self._children: + child.add_filter(f) self.debug('Loaded Filter: %s (%s)', f.name, path) def set_source_attributes(self, context): - """Set source attributes from the context. - - Each item in `attrs` is the attribute name. If the default value is in - context['vars'] under a different name, use a tuple. - """ - attrs = ( - 'filetypes', - 'disabled_syntaxes', - 'input_pattern', - ('min_pattern_length', 'deoplete#auto_complete_start_length'), - 'max_pattern_length', - ('max_abbr_width', 'deoplete#max_abbr_width'), - ('max_kind_width', 'deoplete#max_menu_width'), - ('max_menu_width', 'deoplete#max_menu_width'), - 'matchers', - 'sorters', - 'converters', - 'mark', - 'is_debug_enabled', - 'is_silent', - ) - - for name, source in self._sources.items(): - for attr in attrs: - if isinstance(attr, tuple): - default_val = context['vars'][attr[1]] - attr = attr[0] - else: - default_val = None - source_attr = getattr(source, attr, default_val) - setattr(source, attr, get_custom(context['custom'], - name, attr, source_attr)) - - def use_previous_result(self, context, result, is_volatile): - if context['position'][1] != result['prev_linenr']: - return False - if is_volatile: - return context['input'] == result['prev_input'] - else: - return (re.sub(r'\w*$', '', context['input']) == - re.sub(r'\w*$', '', result['prev_input']) and - context['input'].find(result['prev_input']) == 0) - - def is_skip(self, context, source): - if 'syntax_names' in context and source.disabled_syntaxes: - p = re.compile('(' + '|'.join(source.disabled_syntaxes) + ')$') - if next(filter(p.search, context['syntax_names']), None): - return True - if (source.input_pattern != '' and - re.search('(' + source.input_pattern + ')$', - context['input'])): - return False - if context['event'] == 'Manual': - return False - return not (source.min_pattern_length <= - len(context['complete_str']) <= source.max_pattern_length) + for child in self._children: + child.set_source_attributes(context) + + def set_custom(self, context): + self._custom = context['custom'] + for child in self._children: + child.set_custom(self._custom) def check_recache(self, context): if context['runtimepath'] != self._runtimepath: @@ -489,18 +208,11 @@ def check_recache(self, context): self.on_event(context) elif context['custom'] != self._custom: self.set_source_attributes(context) - self._custom = context['custom'] + self.set_custom(context) def on_event(self, context): self.debug('on_event: %s', context['event']) self.check_recache(context) - for source_name, source in self.itersource(context): - if hasattr(source, 'on_event'): - self.debug('on_event: Source: %s', source_name) - try: - source.on_event(context) - except Exception as exc: - error_tb(self._vim, 'Exception during {}.on_event ' - 'for event {!r}: {}'.format( - source_name, context['event'], exc)) + for child in self._children: + child.on_event(context) From 45652186621744805f32f1a60645ed4abc51c8a2 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 24 Dec 2017 17:59:16 +0900 Subject: [PATCH 04/40] Add max_children --- rplugin/python3/deoplete/deoplete.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index b007facf..c4e7bc55 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -27,7 +27,12 @@ def __init__(self, vim): self._profile_flag = None self._profile_start = 0 self._loaded_paths = set() - self._children = [Child(vim)] + + self._children = [] + self._child_count = 0 + self._max_children = 5 + for n in range(0, self._max_children): + self._children.append(Child(vim)) # Enable logging before "Init" for more information, and e.g. # deoplete-jedi picks up the log filename from deoplete's handler in @@ -81,7 +86,12 @@ def completion_begin(self, context): self._vim.call('deoplete#handler#_completion_timer_start') def merge_results(self, context): - (is_async, merged_results) = self._children[0].merge_results(context) + is_async = False + merged_results = [] + for child in self._children: + result = child.merge_results(context) + is_async = is_async or result[0] + merged_results += result[1] if not merged_results: return (is_async, -1, []) @@ -156,7 +166,9 @@ def load_sources(self, context): error_tb(self._vim, 'Could not load source: %s' % name) finally: if source: - self._children[0].add_source(source) + self._children[self._child_count].add_source(source) + self._child_count += 1 + self._child_count %= self._max_children self.debug('Loaded Source: %s (%s)', source.name, path) self.set_source_attributes(context) From 1858a75d2f8aceafdfdf22d4feacd6694bc0fe25 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Wed, 27 Dec 2017 09:26:37 +0900 Subject: [PATCH 05/40] Fix logging --- rplugin/python3/deoplete/child.py | 2 +- rplugin/python3/deoplete/deoplete.py | 19 ++----------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 1a95a465..d545533c 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -32,7 +32,7 @@ def __init__(self, vim): self._filter_errors = defaultdict(int) self._prev_results = {} - def enable_logging(self, context): + def enable_logging(self): self.is_debug_enabled = True def gather_results(self, context): diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index c4e7bc55..cd706463 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -53,6 +53,8 @@ def enable_logging(self): logging = self._vim.vars['deoplete#_logging'] logger.setup(self._vim, logging['level'], logging['logfile']) self.is_debug_enabled = True + for child in self._children: + child.enable_logging() def completion_begin(self, context): self.check_recache(context) @@ -127,23 +129,6 @@ def merge_results(self, context): return (is_async, complete_position, all_candidates) - def profile_start(self, context, name): - if self._profile_flag is 0 or not self.is_debug_enabled: - return - - if not self._profile_flag: - self._profile_flag = context['vars']['deoplete#enable_profile'] - if self._profile_flag: - return self.profile_start(context, name) - elif self._profile_flag: - self.debug('Profile Start: {0}'.format(name)) - self._profile_start = time.clock() - - def profile_end(self, name): - if self._profile_start: - self.debug('Profile End : {0:<25} time={1:2.10f}'.format( - name, time.clock() - self._profile_start)) - def load_sources(self, context): # Load sources from runtimepath for path in find_rplugins(context, 'source'): From 0c786bfa6c53dfc3ef450c86382fb1fea64244c9 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Wed, 27 Dec 2017 09:27:55 +0900 Subject: [PATCH 06/40] Fix ignorecase --- rplugin/python3/deoplete/child.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index d545533c..762566b6 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -178,16 +178,13 @@ def source_result(self, result, context_input): ctx['complete_str'] = context_input[ctx['char_position']:] ctx['is_sorted'] = False - # Filtering - ignorecase = ctx['ignorecase'] - smartcase = ctx['smartcase'] - camelcase = ctx['camelcase'] - # Set ignorecase - if (smartcase or camelcase) and re.search( - r'[A-Z]', ctx['complete_str']): + case = ctx['smartcase'] or ctx['camelcase'] + if case and re.search(r'[A-Z]', ctx['complete_str']): ctx['ignorecase'] = 0 + ignorecase = ctx['ignorecase'] + # Filtering for f in [self._filters[x] for x in source.matchers + source.sorters + source.converters if x in self._filters]: From 167fe7f1f9ecf3ee91d5d864bb7b7937213675b1 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Wed, 27 Dec 2017 09:28:36 +0900 Subject: [PATCH 07/40] Remove profile flag --- rplugin/python3/deoplete/deoplete.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index cd706463..c1083e59 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -24,8 +24,6 @@ def __init__(self, vim): self._vim = vim self._runtimepath = '' self._custom = [] - self._profile_flag = None - self._profile_start = 0 self._loaded_paths = set() self._children = [] From 79c2d23dd724db55d06ad8b451acd6223a02777a Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Wed, 27 Dec 2017 09:33:51 +0900 Subject: [PATCH 08/40] Add duplicated sources/filters check --- rplugin/python3/deoplete/deoplete.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index c1083e59..0a417877 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -4,7 +4,6 @@ # License: MIT license # ============================================================================ -import time import os.path import deoplete.util # noqa @@ -25,6 +24,8 @@ def __init__(self, vim): self._runtimepath = '' self._custom = [] self._loaded_paths = set() + self._loaded_sources = {} + self._loaded_filters = {} self._children = [] self._child_count = 0 @@ -145,10 +146,17 @@ def load_sources(self, context): source = Source(self._vim) source.name = getattr(source, 'name', name) source.path = path + if source.name in self._loaded_sources: + # Duplicated name + error_tb(self._vim, 'duplicated source: %s' % source.name) + error_tb(self._vim, 'path: "%s" "%s"' % + (path, self._loaded_sources[source.name])) + source = None except Exception: error_tb(self._vim, 'Could not load source: %s' % name) finally: if source: + self._loaded_sources[source.name] = path self._children[self._child_count].add_source(source) self._child_count += 1 self._child_count %= self._max_children @@ -175,11 +183,18 @@ def load_filters(self, context): f = Filter(self._vim) f.name = getattr(f, 'name', name) f.path = path + if f.name in self._loaded_filters: + # Duplicated name + error_tb(self._vim, 'duplicated filter: %s' % f.name) + error_tb(self._vim, 'path: "%s" "%s"' % + (path, self._loaded_filters[f.name])) + f = None except Exception: # Exception occurred when loading a filter. Log stack trace. error_tb(self._vim, 'Could not load filter: %s' % name) finally: if f: + self._loaded_filters[f.name] = path for child in self._children: child.add_filter(f) self.debug('Loaded Filter: %s (%s)', f.name, path) From 89793ba20da582f4ba3950d26d241657cfd0c614 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Wed, 27 Dec 2017 09:42:10 +0900 Subject: [PATCH 09/40] Add the debug messages --- rplugin/python3/deoplete/deoplete.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 0a417877..e3e6db72 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -56,6 +56,8 @@ def enable_logging(self): child.enable_logging() def completion_begin(self, context): + self.debug('completion_begin: %s', context['input']) + self.check_recache(context) try: @@ -76,7 +78,6 @@ def completion_begin(self, context): in context['vars']): self._vim.call('deoplete#mapping#_restore_completeopt') - # error(self._vim, context['input']) # error(self._vim, candidates) self._vim.vars['deoplete#_context'] = { 'complete_position': position, @@ -86,6 +87,8 @@ def completion_begin(self, context): } self._vim.call('deoplete#handler#_completion_timer_start') + self.debug('completion_end: %s', context['input']) + def merge_results(self, context): is_async = False merged_results = [] From c94a488f8977a66d5ffbb7e24e5219b62af12da2 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Wed, 27 Dec 2017 09:45:58 +0900 Subject: [PATCH 10/40] Add thread --- rplugin/python3/deoplete/child.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 762566b6..9f8a106b 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -9,6 +9,9 @@ import time from collections import defaultdict +from threading import Thread +from queue import Queue +from time import time, sleep from deoplete import logger from deoplete.exceptions import SourceInitError @@ -32,6 +35,10 @@ def __init__(self, vim): self._filter_errors = defaultdict(int) self._prev_results = {} + self._thread = None + self._queue_in = Queue() + self._queue_out = Queue() + def enable_logging(self): self.is_debug_enabled = True @@ -276,6 +283,8 @@ def profile_end(self, name): name, time.clock() - self._profile_start)) def add_source(self, s): + if not self._sources: + self._thread = Thread(target=self._main_loop) self._sources[s.name] = s def add_filter(self, f): @@ -352,3 +361,6 @@ def on_event(self, context): error_tb(self._vim, 'Exception during {}.on_event ' 'for event {!r}: {}'.format( source_name, context['event'], exc)) + + def _main_loop(self): + pass From c6d8bc6e577c372bd80830e110790ead85c791d6 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 1 Jan 2018 16:20:24 +0900 Subject: [PATCH 11/40] Improve add_source() --- rplugin/python3/deoplete/child.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 9f8a106b..f6596293 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -283,7 +283,7 @@ def profile_end(self, name): name, time.clock() - self._profile_start)) def add_source(self, s): - if not self._sources: + if not self._thread: self._thread = Thread(target=self._main_loop) self._sources[s.name] = s From 4fbe097183f14e57c6e113770a545f5f341ad794 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 1 Jan 2018 17:41:47 +0900 Subject: [PATCH 12/40] WIP: add main loop --- rplugin/python3/deoplete/child.py | 144 +++++++++++++++++---------- rplugin/python3/deoplete/deoplete.py | 4 +- 2 files changed, 91 insertions(+), 57 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index f6596293..057499de 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -11,7 +11,6 @@ from collections import defaultdict from threading import Thread from queue import Queue -from time import time, sleep from deoplete import logger from deoplete.exceptions import SourceInitError @@ -30,7 +29,7 @@ def __init__(self, vim): self._sources = {} self._custom = [] self._profile_flag = None - self._profile_start = 0 + self._profile_start_time = 0 self._source_errors = defaultdict(int) self._filter_errors = defaultdict(int) self._prev_results = {} @@ -42,10 +41,70 @@ def __init__(self, vim): def enable_logging(self): self.is_debug_enabled = True - def gather_results(self, context): + def add_source(self, s): + if not self._thread: + self._thread = Thread(target=self._main_loop) + self._thread.start() + self._queue_in.put(['add_source', [s]]) + + def add_filter(self, f): + self._queue_in.put(['add_filter', [f]]) + + def set_source_attributes(self, context): + self._queue_in.put(['set_source_attributes', [context]]) + + def set_custom(self, custom): + self._queue_in.put(['set_custom', [custom]]) + + def merge_results(self, context): + self._queue_in.put(['merge_results', [context]]) + return self._queue_out.get() + + def on_event(self, context): + self._queue_in.put(['on_event', [context]]) + + def _main_loop(self): + while 1: + self.debug('main_loop: begin') + [message, args] = self._queue_in.get() + self.debug('main_loop: %s', message) + if message == 'add_source': + self._add_source(args[0]) + elif message == 'add_filter': + self._add_filter(args[0]) + elif message == 'set_source_attributes': + self._set_source_attributes(args[0]) + elif message == 'set_custom': + self._set_custom(args[0]) + elif message == 'on_event': + self._on_event(args[0]) + # elif message == 'merge_results': + # self._merge_results(args[0]) + + def _add_source(self, s): + self._sources[s.name] = s + + def _add_filter(self, f): + self._filters[f.name] = f + + def _merge_results(self, context): + results = self._gather_results(context) + + merged_results = [] + for result in [x for x in results + if not self._is_skip(x['context'], x['source'])]: + source_result = self._source_result(result, context['input']) + if source_result: + merged_results.append(source_result) + + is_async = len([x for x in results if x['context']['is_async']]) > 0 + + self._queue_out.put((is_async, merged_results)) + + def _gather_results(self, context): results = [] - for source in [x[1] for x in self.itersource(context)]: + for source in [x[1] for x in self._itersource(context)]: try: if source.disabled_syntaxes and 'syntax_names' not in context: context['syntax_names'] = get_syn_names(self._vim) @@ -61,14 +120,14 @@ def gather_results(self, context): ctx['encoding'], ctx['input'], charpos) ctx['complete_str'] = ctx['input'][ctx['char_position']:] - if charpos < 0 or self.is_skip(ctx, source): + if charpos < 0 or self._is_skip(ctx, source): if source.name in self._prev_results: self._prev_results.pop(source.name) # Skip continue if (source.name in self._prev_results and - self.use_previous_result( + self._use_previous_result( context, self._prev_results[source.name], source.is_volatile)): results.append(self._prev_results[source.name]) @@ -90,9 +149,9 @@ def gather_results(self, context): ctx['max_menu_width'] = max(10, ctx['max_menu_width']) # Gathering - self.profile_start(ctx, source.name) + self._profile_start(ctx, source.name) ctx['candidates'] = source.gather_candidates(ctx) - self.profile_end(source.name) + self._profile_end(source.name) if ctx['candidates'] is None: continue @@ -123,7 +182,7 @@ def gather_results(self, context): return results - def gather_async_results(self, result, source): + def _gather_async_results(self, result, source): try: result['context']['is_refresh'] = False async_candidates = source.gather_candidates(result['context']) @@ -144,9 +203,9 @@ def gather_async_results(self, result, source): else: error_tb(self._vim, 'Errors from: %s' % source.name) - def process_filter(self, f, context): + def _process_filter(self, f, context): try: - self.profile_start(context, f.name) + self._profile_start(context, f.name) if (isinstance(context['candidates'], dict) and 'sorted_candidates' in context['candidates']): context_candidates = [] @@ -157,7 +216,7 @@ def process_filter(self, f, context): context['candidates'] = context_candidates else: context['candidates'] = f.filter(context) - self.profile_end(f.name) + self._profile_end(f.name) except Exception: self._filter_errors[f.name] += 1 if self._source_errors[f.name] > 2: @@ -168,12 +227,12 @@ def process_filter(self, f, context): return error_tb(self._vim, 'Errors from: %s' % f) - def source_result(self, result, context_input): + def _source_result(self, result, context_input): source = result['source'] # Gather async results if result['is_async']: - self.gather_async_results(result, source) + self._gather_async_results(result, source) if not result['context']['candidates']: return [] @@ -195,7 +254,7 @@ def source_result(self, result, context_input): for f in [self._filters[x] for x in source.matchers + source.sorters + source.converters if x in self._filters]: - self.process_filter(f, ctx) + self._process_filter(f, ctx) ctx['ignorecase'] = ignorecase @@ -207,21 +266,7 @@ def source_result(self, result, context_input): return [ctx['candidates'], result] return [] - def merge_results(self, context): - results = self.gather_results(context) - - merged_results = [] - for result in [x for x in results - if not self.is_skip(x['context'], x['source'])]: - source_result = self.source_result(result, context['input']) - if source_result: - merged_results.append(source_result) - - is_async = len([x for x in results if x['context']['is_async']]) > 0 - - return (is_async, merged_results) - - def itersource(self, context): + def _itersource(self, context): sources = sorted(self._sources.items(), key=lambda x: get_custom( context['custom'], @@ -265,35 +310,24 @@ def itersource(self, context): source.is_initialized = True yield source_name, source - def profile_start(self, context, name): + def _profile_start(self, context, name): if self._profile_flag is 0 or not self.is_debug_enabled: return if not self._profile_flag: self._profile_flag = context['vars']['deoplete#enable_profile'] if self._profile_flag: - return self.profile_start(context, name) + return self._profile_start(context, name) elif self._profile_flag: self.debug('Profile Start: {0}'.format(name)) - self._profile_start = time.clock() + self._profile_start_time = time.clock() - def profile_end(self, name): - if self._profile_start: + def _profile_end(self, name): + if self._profile_start_time: self.debug('Profile End : {0:<25} time={1:2.10f}'.format( - name, time.clock() - self._profile_start)) + name, time.clock() - self._profile_start_time)) - def add_source(self, s): - if not self._thread: - self._thread = Thread(target=self._main_loop) - self._sources[s.name] = s - - def add_filter(self, f): - self._filters[f.name] = f - - def set_custom(self, custom): - self._custom = custom - - def use_previous_result(self, context, result, is_volatile): + def _use_previous_result(self, context, result, is_volatile): if context['position'][1] != result['prev_linenr']: return False if is_volatile: @@ -303,7 +337,7 @@ def use_previous_result(self, context, result, is_volatile): re.sub(r'\w*$', '', result['prev_input']) and context['input'].find(result['prev_input']) == 0) - def is_skip(self, context, source): + def _is_skip(self, context, source): if 'syntax_names' in context and source.disabled_syntaxes: p = re.compile('(' + '|'.join(source.disabled_syntaxes) + ')$') if next(filter(p.search, context['syntax_names']), None): @@ -317,7 +351,7 @@ def is_skip(self, context, source): return not (source.min_pattern_length <= len(context['complete_str']) <= source.max_pattern_length) - def set_source_attributes(self, context): + def _set_source_attributes(self, context): """Set source attributes from the context. Each item in `attrs` is the attribute name. If the default value is in @@ -351,8 +385,11 @@ def set_source_attributes(self, context): setattr(source, attr, get_custom(context['custom'], name, attr, source_attr)) - def on_event(self, context): - for source_name, source in self.itersource(context): + def _set_custom(self, custom): + self._custom = custom + + def _on_event(self, context): + for source_name, source in self._itersource(context): if hasattr(source, 'on_event'): self.debug('on_event: Source: %s', source_name) try: @@ -361,6 +398,3 @@ def on_event(self, context): error_tb(self._vim, 'Exception during {}.on_event ' 'for event {!r}: {}'.format( source_name, context['event'], exc)) - - def _main_loop(self): - pass diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index e3e6db72..f7de5bd7 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -12,7 +12,7 @@ from deoplete import logger from deoplete.child import Child -from deoplete.util import (error_tb, find_rplugins, import_plugin) +from deoplete.util import (error, error_tb, find_rplugins, import_plugin) class Deoplete(logger.LoggingMixin): @@ -29,7 +29,7 @@ def __init__(self, vim): self._children = [] self._child_count = 0 - self._max_children = 5 + self._max_children = 1 for n in range(0, self._max_children): self._children.append(Child(vim)) From b299e7d1f82ad7bda0c6fa35b6085d1abdc1acfc Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 1 Jan 2018 19:31:48 +0900 Subject: [PATCH 13/40] WIP: attach Vim --- autoload/deoplete/init.vim | 2 ++ rplugin/python3/deoplete/child.py | 43 +++++++++++++++++++--------- rplugin/python3/deoplete/deoplete.py | 7 +++-- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/autoload/deoplete/init.vim b/autoload/deoplete/init.vim index 80d02f63..29f3ff3e 100644 --- a/autoload/deoplete/init.vim +++ b/autoload/deoplete/init.vim @@ -221,6 +221,8 @@ function! deoplete#init#_context(event, sources) abort return { \ 'changedtick': b:changedtick, + \ 'serveraddr': (has('nvim') ? + \ v:servername : neovim_rpc#serveraddr()), \ 'event': event, \ 'input': input, \ 'is_windows': ((has('win32') || has('win64')) ? v:true : v:false), diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 057499de..bbffcfff 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -11,6 +11,7 @@ from collections import defaultdict from threading import Thread from queue import Queue +from neovim import attach from deoplete import logger from deoplete.exceptions import SourceInitError @@ -21,10 +22,10 @@ class Child(logger.LoggingMixin): - def __init__(self, vim): + def __init__(self): self.name = 'child' - self._vim = vim + self._vim = None self._filters = {} self._sources = {} self._custom = [] @@ -41,29 +42,45 @@ def __init__(self, vim): def enable_logging(self): self.is_debug_enabled = True - def add_source(self, s): + def add_source(self, s, serveraddr): if not self._thread: - self._thread = Thread(target=self._main_loop) + self._thread = Thread(target=self._main_loop, + args=(serveraddr)) self._thread.start() - self._queue_in.put(['add_source', [s]]) + self._queue_put('add_source', [s]) def add_filter(self, f): - self._queue_in.put(['add_filter', [f]]) + self._queue_put('add_filter', [f]) def set_source_attributes(self, context): - self._queue_in.put(['set_source_attributes', [context]]) + self._queue_put('set_source_attributes', [context]) def set_custom(self, custom): - self._queue_in.put(['set_custom', [custom]]) + self._queue_put('set_custom', [custom]) def merge_results(self, context): - self._queue_in.put(['merge_results', [context]]) + self._queue_put('merge_results', [context]) + if self._queue_out.empty(): + return (False, []) return self._queue_out.get() def on_event(self, context): - self._queue_in.put(['on_event', [context]]) + self._queue_put('on_event', [context]) + + def _queue_put(self, name, args): + self._queue_in.put([name, args]) + + def _attach_vim(self, serveraddr): + if len(serveraddr.split(':')) == 2: + serveraddr, port = serveraddr.split(':') + port = int(port) + self._vim = attach('tcp', address=serveraddr, port=port) + else: + self._vim = attach('socket', address=serveraddr) + + def _main_loop(self, serveraddr): + self._attach_vim(serveraddr) - def _main_loop(self): while 1: self.debug('main_loop: begin') [message, args] = self._queue_in.get() @@ -78,8 +95,8 @@ def _main_loop(self): self._set_custom(args[0]) elif message == 'on_event': self._on_event(args[0]) - # elif message == 'merge_results': - # self._merge_results(args[0]) + elif message == 'merge_results': + self._merge_results(args[0]) def _add_source(self, s): self._sources[s.name] = s diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index f7de5bd7..3d634d80 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -31,7 +31,7 @@ def __init__(self, vim): self._child_count = 0 self._max_children = 1 for n in range(0, self._max_children): - self._children.append(Child(vim)) + self._children.append(Child()) # Enable logging before "Init" for more information, and e.g. # deoplete-jedi picks up the log filename from deoplete's handler in @@ -90,12 +90,14 @@ def completion_begin(self, context): self.debug('completion_end: %s', context['input']) def merge_results(self, context): + error(self._vim, 'merge') is_async = False merged_results = [] for child in self._children: result = child.merge_results(context) is_async = is_async or result[0] merged_results += result[1] + error(self._vim, 'results') if not merged_results: return (is_async, -1, []) @@ -160,7 +162,8 @@ def load_sources(self, context): finally: if source: self._loaded_sources[source.name] = path - self._children[self._child_count].add_source(source) + self._children[self._child_count].add_source( + source, context['serveraddr']) self._child_count += 1 self._child_count %= self._max_children self.debug('Loaded Source: %s (%s)', source.name, path) From 389b6253527f31d2c6d680daad4caca778ac36fe Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 1 Jan 2018 19:43:28 +0900 Subject: [PATCH 14/40] Improve gather_async_results() --- rplugin/python3/deoplete/child.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index bbffcfff..57904d47 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -85,18 +85,18 @@ def _main_loop(self, serveraddr): self.debug('main_loop: begin') [message, args] = self._queue_in.get() self.debug('main_loop: %s', message) - if message == 'add_source': - self._add_source(args[0]) - elif message == 'add_filter': - self._add_filter(args[0]) - elif message == 'set_source_attributes': - self._set_source_attributes(args[0]) - elif message == 'set_custom': - self._set_custom(args[0]) - elif message == 'on_event': - self._on_event(args[0]) - elif message == 'merge_results': - self._merge_results(args[0]) + # if message == 'add_source': + # self._add_source(args[0]) + # elif message == 'add_filter': + # self._add_filter(args[0]) + # elif message == 'set_source_attributes': + # self._set_source_attributes(args[0]) + # elif message == 'set_custom': + # self._set_custom(args[0]) + # elif message == 'on_event': + # self._on_event(args[0]) + # elif message == 'merge_results': + # self._merge_results(args[0]) def _add_source(self, s): self._sources[s.name] = s @@ -201,13 +201,13 @@ def _gather_results(self, context): def _gather_async_results(self, result, source): try: - result['context']['is_refresh'] = False - async_candidates = source.gather_candidates(result['context']) - result['is_async'] = result['context']['is_async'] + context = result['context'] + context['is_refresh'] = False + async_candidates = source.gather_candidates(context) + result['is_async'] = context['is_async'] if async_candidates is None: return - result['context']['candidates'] += convert2candidates( - async_candidates) + context['candidates'] += convert2candidates(async_candidates) except Exception: self._source_errors[source.name] += 1 if source.is_silent: From dedcb83719e0663d70e641f70859df8809915a63 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 1 Jan 2018 19:53:17 +0900 Subject: [PATCH 15/40] Add _start_thread()/_stop_thread() --- rplugin/python3/deoplete/child.py | 42 +++++++++++++++++++------------ 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 57904d47..73d42a9b 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -43,10 +43,7 @@ def enable_logging(self): self.is_debug_enabled = True def add_source(self, s, serveraddr): - if not self._thread: - self._thread = Thread(target=self._main_loop, - args=(serveraddr)) - self._thread.start() + self._start_thread(serveraddr) self._queue_put('add_source', [s]) def add_filter(self, f): @@ -66,6 +63,19 @@ def merge_results(self, context): def on_event(self, context): self._queue_put('on_event', [context]) + if context['event'] == 'VimLeavePre': + self._stop_thread() + + def _start_thread(self, serveraddr): + if not self._thread: + self._thread = Thread(target=self._main_loop, + args=(serveraddr)) + self._thread.start() + + def _stop_thread(self): + if self._thread: + self._thread.join(1.0) + self._thread = None def _queue_put(self, name, args): self._queue_in.put([name, args]) @@ -85,18 +95,18 @@ def _main_loop(self, serveraddr): self.debug('main_loop: begin') [message, args] = self._queue_in.get() self.debug('main_loop: %s', message) - # if message == 'add_source': - # self._add_source(args[0]) - # elif message == 'add_filter': - # self._add_filter(args[0]) - # elif message == 'set_source_attributes': - # self._set_source_attributes(args[0]) - # elif message == 'set_custom': - # self._set_custom(args[0]) - # elif message == 'on_event': - # self._on_event(args[0]) - # elif message == 'merge_results': - # self._merge_results(args[0]) + if message == 'add_source': + self._add_source(args[0]) + elif message == 'add_filter': + self._add_filter(args[0]) + elif message == 'set_source_attributes': + self._set_source_attributes(args[0]) + elif message == 'set_custom': + self._set_custom(args[0]) + elif message == 'on_event': + self._on_event(args[0]) + elif message == 'merge_results': + self._merge_results(args[0]) def _add_source(self, s): self._sources[s.name] = s From 700b7f81593ba0ea2a30e9874d66dee589b24337 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 7 Jan 2018 18:01:53 +0900 Subject: [PATCH 16/40] Add daemon flag --- rplugin/python3/deoplete/child.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 73d42a9b..550a2e69 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -69,7 +69,7 @@ def on_event(self, context): def _start_thread(self, serveraddr): if not self._thread: self._thread = Thread(target=self._main_loop, - args=(serveraddr)) + args=(serveraddr,), daemon=True) self._thread.start() def _stop_thread(self): @@ -89,6 +89,7 @@ def _attach_vim(self, serveraddr): self._vim = attach('socket', address=serveraddr) def _main_loop(self, serveraddr): + self._vim.vars['hoge'] = 1 self._attach_vim(serveraddr) while 1: From ce51f0ce694fedda53a461a58ae2cb23353beda4 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 7 Jan 2018 21:22:09 +0900 Subject: [PATCH 17/40] Add dp_main.py --- autoload/deoplete/init.vim | 5 +++++ rplugin/python3/deoplete/child.py | 29 +++++++++++++++------------- rplugin/python3/deoplete/deoplete.py | 6 ++---- rplugin/python3/deoplete/dp_main.py | 7 +++++++ 4 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 rplugin/python3/deoplete/dp_main.py diff --git a/autoload/deoplete/init.vim b/autoload/deoplete/init.vim index 29f3ff3e..afacdaf7 100644 --- a/autoload/deoplete/init.vim +++ b/autoload/deoplete/init.vim @@ -4,6 +4,9 @@ " License: MIT license "============================================================================= +let s:dp_main = fnamemodify(expand(''), ':h:h:h') + \ . '/rplugin/python3/deoplete/dp_main.py' + if !exists('s:is_enabled') let s:is_enabled = 0 endif @@ -223,6 +226,8 @@ function! deoplete#init#_context(event, sources) abort \ 'changedtick': b:changedtick, \ 'serveraddr': (has('nvim') ? \ v:servername : neovim_rpc#serveraddr()), + \ 'python3': get(g:, 'python3_host_prog', 'python3'), + \ 'dp_main': s:dp_main, \ 'event': event, \ 'input': input, \ 'is_windows': ((has('win32') || has('win64')) ? v:true : v:false), diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 550a2e69..0fcec6ad 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -15,6 +15,7 @@ from deoplete import logger from deoplete.exceptions import SourceInitError +from deoplete.process import Process from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, get_buffer_config, get_custom, get_syn_names, convert2candidates) @@ -22,10 +23,10 @@ class Child(logger.LoggingMixin): - def __init__(self): + def __init__(self, vim): self.name = 'child' - self._vim = None + self._vim = vim self._filters = {} self._sources = {} self._custom = [] @@ -35,15 +36,15 @@ def __init__(self): self._filter_errors = defaultdict(int) self._prev_results = {} - self._thread = None + self._proc = None self._queue_in = Queue() self._queue_out = Queue() def enable_logging(self): self.is_debug_enabled = True - def add_source(self, s, serveraddr): - self._start_thread(serveraddr) + def add_source(self, s, context): + self._start_thread(context, context['serveraddr']) self._queue_put('add_source', [s]) def add_filter(self, f): @@ -66,16 +67,18 @@ def on_event(self, context): if context['event'] == 'VimLeavePre': self._stop_thread() - def _start_thread(self, serveraddr): - if not self._thread: - self._thread = Thread(target=self._main_loop, - args=(serveraddr,), daemon=True) - self._thread.start() + def _start_thread(self, context, serveraddr): + if not self._proc: + self._proc = Process( + [context['python3'], context['dp_main']], + context, context['cwd']) + time.sleep(0.1) + error(self._vim, self._proc.communicate(100)) def _stop_thread(self): - if self._thread: - self._thread.join(1.0) - self._thread = None + if self._proc: + self._proc.kill() + self._proc = None def _queue_put(self, name, args): self._queue_in.put([name, args]) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 3d634d80..6ddd4b8c 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -31,7 +31,7 @@ def __init__(self, vim): self._child_count = 0 self._max_children = 1 for n in range(0, self._max_children): - self._children.append(Child()) + self._children.append(Child(vim)) # Enable logging before "Init" for more information, and e.g. # deoplete-jedi picks up the log filename from deoplete's handler in @@ -90,14 +90,12 @@ def completion_begin(self, context): self.debug('completion_end: %s', context['input']) def merge_results(self, context): - error(self._vim, 'merge') is_async = False merged_results = [] for child in self._children: result = child.merge_results(context) is_async = is_async or result[0] merged_results += result[1] - error(self._vim, 'results') if not merged_results: return (is_async, -1, []) @@ -163,7 +161,7 @@ def load_sources(self, context): if source: self._loaded_sources[source.name] = path self._children[self._child_count].add_source( - source, context['serveraddr']) + source, context) self._child_count += 1 self._child_count %= self._max_children self.debug('Loaded Source: %s (%s)', source.name, path) diff --git a/rplugin/python3/deoplete/dp_main.py b/rplugin/python3/deoplete/dp_main.py new file mode 100644 index 00000000..5a5d67de --- /dev/null +++ b/rplugin/python3/deoplete/dp_main.py @@ -0,0 +1,7 @@ + +def main(): + print('foo') + return + +if __name__ == '__main__': + main() From 38b9a65253780b832881b8939d946ad76b886fca Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 8 Jan 2018 14:48:31 +0900 Subject: [PATCH 18/40] WIP: multi process --- rplugin/python3/deoplete/child.py | 116 +++++++++++++-------------- rplugin/python3/deoplete/deoplete.py | 95 ++++++---------------- rplugin/python3/deoplete/dp_main.py | 37 ++++++++- rplugin/python3/deoplete/parent.py | 63 +++++++++++++++ 4 files changed, 174 insertions(+), 137 deletions(-) create mode 100644 rplugin/python3/deoplete/parent.py diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 0fcec6ad..7771d849 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -4,19 +4,17 @@ # License: MIT license # ============================================================================ +import os.path import re import copy import time from collections import defaultdict -from threading import Thread -from queue import Queue -from neovim import attach from deoplete import logger from deoplete.exceptions import SourceInitError -from deoplete.process import Process from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, + import_plugin, get_buffer_config, get_custom, get_syn_names, convert2candidates) @@ -32,68 +30,17 @@ def __init__(self, vim): self._custom = [] self._profile_flag = None self._profile_start_time = 0 + self._loaded_sources = {} + self._loaded_filters = {} self._source_errors = defaultdict(int) self._filter_errors = defaultdict(int) self._prev_results = {} - self._proc = None - self._queue_in = Queue() - self._queue_out = Queue() - def enable_logging(self): self.is_debug_enabled = True - def add_source(self, s, context): - self._start_thread(context, context['serveraddr']) - self._queue_put('add_source', [s]) - - def add_filter(self, f): - self._queue_put('add_filter', [f]) - - def set_source_attributes(self, context): - self._queue_put('set_source_attributes', [context]) - - def set_custom(self, custom): - self._queue_put('set_custom', [custom]) - - def merge_results(self, context): - self._queue_put('merge_results', [context]) - if self._queue_out.empty(): - return (False, []) - return self._queue_out.get() - - def on_event(self, context): - self._queue_put('on_event', [context]) - if context['event'] == 'VimLeavePre': - self._stop_thread() - - def _start_thread(self, context, serveraddr): - if not self._proc: - self._proc = Process( - [context['python3'], context['dp_main']], - context, context['cwd']) - time.sleep(0.1) - error(self._vim, self._proc.communicate(100)) - - def _stop_thread(self): - if self._proc: - self._proc.kill() - self._proc = None - - def _queue_put(self, name, args): - self._queue_in.put([name, args]) - - def _attach_vim(self, serveraddr): - if len(serveraddr.split(':')) == 2: - serveraddr, port = serveraddr.split(':') - port = int(port) - self._vim = attach('tcp', address=serveraddr, port=port) - else: - self._vim = attach('socket', address=serveraddr) - - def _main_loop(self, serveraddr): + def main_loop(self, serveraddr): self._vim.vars['hoge'] = 1 - self._attach_vim(serveraddr) while 1: self.debug('main_loop: begin') @@ -112,11 +59,56 @@ def _main_loop(self, serveraddr): elif message == 'merge_results': self._merge_results(args[0]) - def _add_source(self, s): - self._sources[s.name] = s + def _add_source(self, path): + source = None + try: + Source = import_plugin(path, 'source', 'Source') + if not Source: + return + + source = Source(self._vim) + name = os.path.splitext(os.path.basename(path))[0] + source.name = getattr(source, 'name', name) + source.path = path + if source.name in self._loaded_sources: + # Duplicated name + error_tb(self._vim, 'duplicated source: %s' % source.name) + error_tb(self._vim, 'path: "%s" "%s"' % + (path, self._loaded_sources[source.name])) + source = None + except Exception: + error_tb(self._vim, 'Could not load source: %s' % name) + finally: + if source: + self._loaded_sources[source.name] = path + self._sources[source.name] = source + self.debug('Loaded Source: %s (%s)', source.name, path) + + def _add_filter(self, path): + f = None + try: + Filter = import_plugin(path, 'filter', 'Filter') + if not Filter: + return - def _add_filter(self, f): - self._filters[f.name] = f + f = Filter(self._vim) + name = os.path.splitext(os.path.basename(path))[0] + f.name = getattr(f, 'name', name) + f.path = path + if f.name in self._loaded_filters: + # Duplicated name + error_tb(self._vim, 'duplicated filter: %s' % f.name) + error_tb(self._vim, 'path: "%s" "%s"' % + (path, self._loaded_filters[f.name])) + f = None + except Exception: + # Exception occurred when loading a filter. Log stack trace. + error_tb(self._vim, 'Could not load filter: %s' % name) + finally: + if f: + self._loaded_filters[f.name] = path + self._filters[f.name] = f + self.debug('Loaded Filter: %s (%s)', f.name, path) def _merge_results(self, context): results = self._gather_results(context) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 6ddd4b8c..1ea33f2c 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -4,15 +4,13 @@ # License: MIT license # ============================================================================ -import os.path - import deoplete.util # noqa import deoplete.filter # noqa import deoplete.source # noqa from deoplete import logger -from deoplete.child import Child -from deoplete.util import (error, error_tb, find_rplugins, import_plugin) +from deoplete.parent import Parent +from deoplete.util import (error, error_tb, find_rplugins) class Deoplete(logger.LoggingMixin): @@ -27,11 +25,11 @@ def __init__(self, vim): self._loaded_sources = {} self._loaded_filters = {} - self._children = [] - self._child_count = 0 - self._max_children = 1 - for n in range(0, self._max_children): - self._children.append(Child(vim)) + self._parents = [] + self._parent_count = 0 + self._max_parents = 1 + for n in range(0, self._max_parents): + self._parents.append(Parent(vim)) # Enable logging before "Init" for more information, and e.g. # deoplete-jedi picks up the log filename from deoplete's handler in @@ -52,8 +50,8 @@ def enable_logging(self): logging = self._vim.vars['deoplete#_logging'] logger.setup(self._vim, logging['level'], logging['logfile']) self.is_debug_enabled = True - for child in self._children: - child.enable_logging() + for parent in self._parents: + parent.enable_logging() def completion_begin(self, context): self.debug('completion_begin: %s', context['input']) @@ -92,8 +90,8 @@ def completion_begin(self, context): def merge_results(self, context): is_async = False merged_results = [] - for child in self._children: - result = child.merge_results(context) + for parent in self._parents: + result = parent.merge_results(context) is_async = is_async or result[0] merged_results += result[1] @@ -138,33 +136,10 @@ def load_sources(self, context): continue self._loaded_paths.add(path) - name = os.path.splitext(os.path.basename(path))[0] - - source = None - try: - Source = import_plugin(path, 'source', 'Source') - if not Source: - continue - - source = Source(self._vim) - source.name = getattr(source, 'name', name) - source.path = path - if source.name in self._loaded_sources: - # Duplicated name - error_tb(self._vim, 'duplicated source: %s' % source.name) - error_tb(self._vim, 'path: "%s" "%s"' % - (path, self._loaded_sources[source.name])) - source = None - except Exception: - error_tb(self._vim, 'Could not load source: %s' % name) - finally: - if source: - self._loaded_sources[source.name] = path - self._children[self._child_count].add_source( - source, context) - self._child_count += 1 - self._child_count %= self._max_children - self.debug('Loaded Source: %s (%s)', source.name, path) + self._parents[self._parent_count].add_source(path, context) + + self._parent_count += 1 + self._parent_count %= self._max_parents self.set_source_attributes(context) self.set_custom(context) @@ -176,41 +151,17 @@ def load_filters(self, context): continue self._loaded_paths.add(path) - name = os.path.splitext(os.path.basename(path))[0] - - f = None - try: - Filter = import_plugin(path, 'filter', 'Filter') - if not Filter: - continue - - f = Filter(self._vim) - f.name = getattr(f, 'name', name) - f.path = path - if f.name in self._loaded_filters: - # Duplicated name - error_tb(self._vim, 'duplicated filter: %s' % f.name) - error_tb(self._vim, 'path: "%s" "%s"' % - (path, self._loaded_filters[f.name])) - f = None - except Exception: - # Exception occurred when loading a filter. Log stack trace. - error_tb(self._vim, 'Could not load filter: %s' % name) - finally: - if f: - self._loaded_filters[f.name] = path - for child in self._children: - child.add_filter(f) - self.debug('Loaded Filter: %s (%s)', f.name, path) + for parent in self._parents: + parent.add_filter(path, context) def set_source_attributes(self, context): - for child in self._children: - child.set_source_attributes(context) + for parent in self._parents: + parent.set_source_attributes(context) def set_custom(self, context): self._custom = context['custom'] - for child in self._children: - child.set_custom(self._custom) + for parent in self._parents: + parent.set_custom(self._custom) def check_recache(self, context): if context['runtimepath'] != self._runtimepath: @@ -228,5 +179,5 @@ def on_event(self, context): self.debug('on_event: %s', context['event']) self.check_recache(context) - for child in self._children: - child.on_event(context) + for parent in self._parents: + parent.on_event(context) diff --git a/rplugin/python3/deoplete/dp_main.py b/rplugin/python3/deoplete/dp_main.py index 5a5d67de..f9937c5c 100644 --- a/rplugin/python3/deoplete/dp_main.py +++ b/rplugin/python3/deoplete/dp_main.py @@ -1,7 +1,38 @@ +# ============================================================================ +# FILE: dp_main.py +# AUTHOR: Shougo Matsushita +# License: MIT license +# ============================================================================ -def main(): - print('foo') +import sys +from neovim import attach + + +def attach_vim(serveraddr): + if len(serveraddr.split(':')) == 2: + serveraddr, port = serveraddr.split(':') + port = int(port) + vim = attach('tcp', address=serveraddr, port=port) + else: + vim = attach('socket', path=serveraddr) + + # sync path + for path in vim.call( + 'globpath', vim.options['runtimepath'], + 'rplugin/python3', 1).split('\n'): + sys.path.append(path) + # Remove current path + del sys.path[0] + + return vim + + +def main(serveraddr): + vim = attach_vim(serveraddr) + from deoplete.util import error + error(vim, 'hoge') return + if __name__ == '__main__': - main() + main(sys.argv[1]) diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py new file mode 100644 index 00000000..72cd7486 --- /dev/null +++ b/rplugin/python3/deoplete/parent.py @@ -0,0 +1,63 @@ +# ============================================================================ +# FILE: parent.py +# AUTHOR: Shougo Matsushita +# License: MIT license +# ============================================================================ + +import time + +from deoplete import logger +from deoplete.process import Process +from deoplete.util import error + + +class Parent(logger.LoggingMixin): + + def __init__(self, vim): + self.name = 'parent' + + self._vim = vim + self._proc = None + + def enable_logging(self): + self.is_debug_enabled = True + + def add_source(self, path, context): + self._start_thread(context, context['serveraddr']) + self._queue_put('add_source', [path]) + + def add_filter(self, path): + self._queue_put('add_filter', [path]) + + def set_source_attributes(self, context): + self._queue_put('set_source_attributes', [context]) + + def set_custom(self, custom): + self._queue_put('set_custom', [custom]) + + def merge_results(self, context): + self._queue_put('merge_results', [context]) + if self._queue_out.empty(): + return (False, []) + return self._queue_out.get() + + def on_event(self, context): + self._queue_put('on_event', [context]) + if context['event'] == 'VimLeavePre': + self._stop_thread() + + def _start_thread(self, context, serveraddr): + if not self._proc: + self._proc = Process( + [context['python3'], context['dp_main'], serveraddr], + context, context['cwd']) + time.sleep(1) + error(self._vim, self._proc.communicate(100)) + + def _stop_thread(self): + if self._proc: + self._proc.kill() + self._proc = None + + def _queue_put(self, name, args): + self._queue_in.put([name, args]) From ab9d23151bfafe84c5a313bb98b856435ef1f49b Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 8 Jan 2018 15:52:07 +0900 Subject: [PATCH 19/40] WIP: multi process --- rplugin/python3/deoplete/deoplete.py | 10 ++---- rplugin/python3/deoplete/dp_main.py | 6 +++- rplugin/python3/deoplete/parent.py | 50 +++++++++++++++++++--------- rplugin/python3/deoplete/process.py | 7 +++- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 1ea33f2c..b88947f0 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -4,10 +4,6 @@ # License: MIT license # ============================================================================ -import deoplete.util # noqa -import deoplete.filter # noqa -import deoplete.source # noqa - from deoplete import logger from deoplete.parent import Parent from deoplete.util import (error, error_tb, find_rplugins) @@ -22,8 +18,6 @@ def __init__(self, vim): self._runtimepath = '' self._custom = [] self._loaded_paths = set() - self._loaded_sources = {} - self._loaded_filters = {} self._parents = [] self._parent_count = 0 @@ -136,7 +130,7 @@ def load_sources(self, context): continue self._loaded_paths.add(path) - self._parents[self._parent_count].add_source(path, context) + self._parents[self._parent_count].add_source(context, path) self._parent_count += 1 self._parent_count %= self._max_parents @@ -152,7 +146,7 @@ def load_filters(self, context): self._loaded_paths.add(path) for parent in self._parents: - parent.add_filter(path, context) + parent.add_filter(path) def set_source_attributes(self, context): for parent in self._parents: diff --git a/rplugin/python3/deoplete/dp_main.py b/rplugin/python3/deoplete/dp_main.py index f9937c5c..40f6575c 100644 --- a/rplugin/python3/deoplete/dp_main.py +++ b/rplugin/python3/deoplete/dp_main.py @@ -30,7 +30,11 @@ def attach_vim(serveraddr): def main(serveraddr): vim = attach_vim(serveraddr) from deoplete.util import error - error(vim, 'hoge') + for queue_id in sys.stdin: + if 'deoplete#_child_in' not in vim.vars: + continue + if queue_id.strip() in vim.vars['deoplete#_child_in']: + error(vim, queue_id) return diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index 72cd7486..665d26a6 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -19,34 +19,41 @@ def __init__(self, vim): self._vim = vim self._proc = None + if 'deoplete#_child_in' not in self._vim.vars: + self._vim.vars['deoplete#_child_in'] = {} + if 'deoplete#_child_out' not in self._vim.vars: + self._vim.vars['deoplete#_child_out'] = {} + def enable_logging(self): self.is_debug_enabled = True - def add_source(self, path, context): - self._start_thread(context, context['serveraddr']) - self._queue_put('add_source', [path]) + def add_source(self, context, path): + self._start_process(context, context['serveraddr']) + self._put('add_source', [path]) def add_filter(self, path): - self._queue_put('add_filter', [path]) + self._put('add_filter', [path]) def set_source_attributes(self, context): - self._queue_put('set_source_attributes', [context]) + self._put('set_source_attributes', [context]) def set_custom(self, custom): - self._queue_put('set_custom', [custom]) + self._put('set_custom', [custom]) def merge_results(self, context): - self._queue_put('merge_results', [context]) - if self._queue_out.empty(): + queue_id = self._put('merge_results', [context]) + if not queue_id: return (False, []) - return self._queue_out.get() + + results = self._get(queue_id) + return results if results else (False, []) def on_event(self, context): - self._queue_put('on_event', [context]) if context['event'] == 'VimLeavePre': - self._stop_thread() + self._stop_process() + self._put('on_event', [context]) - def _start_thread(self, context, serveraddr): + def _start_process(self, context, serveraddr): if not self._proc: self._proc = Process( [context['python3'], context['dp_main'], serveraddr], @@ -54,10 +61,23 @@ def _start_thread(self, context, serveraddr): time.sleep(1) error(self._vim, self._proc.communicate(100)) - def _stop_thread(self): + def _stop_process(self): if self._proc: self._proc.kill() self._proc = None - def _queue_put(self, name, args): - self._queue_in.put([name, args]) + def _put(self, name, args): + if not self._proc: + return None + + queue_id = str(time.time()) + + child_in = self._vim.vars['deoplete#_child_in'] + child_in[queue_id] = { 'name': name, 'args': args } + self._vim.vars['deoplete#_child_in'] = child_in + + self._proc.write(queue_id + '\n') + return queue_id + + def _get(self, queue_id): + return self._vim.vars['deoplete#_child_out'].get(queue_id, None) diff --git a/rplugin/python3/deoplete/process.py b/rplugin/python3/deoplete/process.py index 54b93cf0..3d48d5b4 100644 --- a/rplugin/python3/deoplete/process.py +++ b/rplugin/python3/deoplete/process.py @@ -18,7 +18,7 @@ def __init__(self, commands, context, cwd): startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW self.__proc = subprocess.Popen(commands, - stdin=subprocess.DEVNULL, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo, @@ -42,6 +42,11 @@ def kill(self): self.__thread.join(1.0) self.__thread = None + def write(self, text): + self.__proc.stdin.write(text.encode( + self.__context['encoding'], errors='replace')) + self.__proc.stdin.flush() + def enqueue_output(self): for line in self.__proc.stdout: if not self.__queue_out: From 572fa2c4c38d2080258cade296997c3085aec4c6 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 8 Jan 2018 16:13:46 +0900 Subject: [PATCH 20/40] WIP: main_loop() --- autoload/deoplete/handler.vim | 2 ++ rplugin/python3/deoplete/child.py | 51 ++++++++++++++++++----------- rplugin/python3/deoplete/dp_main.py | 14 ++++---- rplugin/python3/deoplete/parent.py | 4 ++- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/autoload/deoplete/handler.vim b/autoload/deoplete/handler.vim index 1db870a8..f9c0e432 100644 --- a/autoload/deoplete/handler.vim +++ b/autoload/deoplete/handler.vim @@ -213,6 +213,8 @@ endfunction function! s:on_insert_leave() abort call deoplete#mapping#_restore_completeopt() let g:deoplete#_context = {} + let g:deoplete#_child_in = {} + let g:deoplete#_child_out = {} endfunction function! s:on_complete_done() abort diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 7771d849..7e56bee4 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -4,9 +4,10 @@ # License: MIT license # ============================================================================ +import copy import os.path import re -import copy +import sys import time from collections import defaultdict @@ -39,25 +40,31 @@ def __init__(self, vim): def enable_logging(self): self.is_debug_enabled = True - def main_loop(self, serveraddr): - self._vim.vars['hoge'] = 1 + def main_loop(self): + for queue_id in sys.stdin: + queue_id = queue_id.strip() + if queue_id not in self._vim.vars['deoplete#_child_in']: + continue - while 1: self.debug('main_loop: begin') - [message, args] = self._queue_in.get() - self.debug('main_loop: %s', message) - if message == 'add_source': - self._add_source(args[0]) - elif message == 'add_filter': - self._add_filter(args[0]) - elif message == 'set_source_attributes': - self._set_source_attributes(args[0]) - elif message == 'set_custom': - self._set_custom(args[0]) - elif message == 'on_event': - self._on_event(args[0]) - elif message == 'merge_results': - self._merge_results(args[0]) + child_in = self._vim.vars['deoplete#_child_in'] + name = child_in[queue_id]['name'] + args = child_in[queue_id]['args'] + error(self._vim, name) + self.debug('main_loop: %s', name) + + # if name == 'add_source': + # self._add_source(args[0]) + # elif name == 'add_filter': + # self._add_filter(args[0]) + # elif name == 'set_source_attributes': + # self._set_source_attributes(args[0]) + # elif name == 'set_custom': + # self._set_custom(args[0]) + # elif name == 'on_event': + # self._on_event(args[0]) + # elif name == 'merge_results': + # self._merge_results(args[0]) def _add_source(self, path): source = None @@ -110,7 +117,7 @@ def _add_filter(self, path): self._filters[f.name] = f self.debug('Loaded Filter: %s (%s)', f.name, path) - def _merge_results(self, context): + def _merge_results(self, context, queue_id): results = self._gather_results(context) merged_results = [] @@ -122,7 +129,11 @@ def _merge_results(self, context): is_async = len([x for x in results if x['context']['is_async']]) > 0 - self._queue_out.put((is_async, merged_results)) + child_out = self._vim.vars['deoplete#_child_out'] + child_out[queue_id] = { + 'is_async': is_async, 'merged_results': merged_results + } + self._vim.vars['deoplete#_child_out'] = child_out def _gather_results(self, context): results = [] diff --git a/rplugin/python3/deoplete/dp_main.py b/rplugin/python3/deoplete/dp_main.py index 40f6575c..9bb820b9 100644 --- a/rplugin/python3/deoplete/dp_main.py +++ b/rplugin/python3/deoplete/dp_main.py @@ -29,12 +29,14 @@ def attach_vim(serveraddr): def main(serveraddr): vim = attach_vim(serveraddr) - from deoplete.util import error - for queue_id in sys.stdin: - if 'deoplete#_child_in' not in vim.vars: - continue - if queue_id.strip() in vim.vars['deoplete#_child_in']: - error(vim, queue_id) + from deoplete.util import error_tb + child = None + try: + from deoplete.child import Child + child = Child(vim) + child.main_loop() + except Exception: + error_tb(vim, 'Error in child') return diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index 665d26a6..30fa1cd4 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -45,6 +45,8 @@ def merge_results(self, context): if not queue_id: return (False, []) + time.sleep(0.5) + results = self._get(queue_id) return results if results else (False, []) @@ -73,7 +75,7 @@ def _put(self, name, args): queue_id = str(time.time()) child_in = self._vim.vars['deoplete#_child_in'] - child_in[queue_id] = { 'name': name, 'args': args } + child_in[queue_id] = {'name': name, 'args': args} self._vim.vars['deoplete#_child_in'] = child_in self._proc.write(queue_id + '\n') From f3663f112a9c41cce122d1e0146bfbd27a2a2928 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 8 Jan 2018 18:28:41 +0900 Subject: [PATCH 21/40] WIP: Implement multi process --- rplugin/python3/deoplete/child.py | 46 ++++++++++++++++------------ rplugin/python3/deoplete/deoplete.py | 19 ++++++------ rplugin/python3/deoplete/parent.py | 12 +++++--- rplugin/python3/deoplete/process.py | 44 ++------------------------ 4 files changed, 46 insertions(+), 75 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 7e56bee4..89883e81 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -43,6 +43,7 @@ def enable_logging(self): def main_loop(self): for queue_id in sys.stdin: queue_id = queue_id.strip() + if queue_id not in self._vim.vars['deoplete#_child_in']: continue @@ -50,21 +51,20 @@ def main_loop(self): child_in = self._vim.vars['deoplete#_child_in'] name = child_in[queue_id]['name'] args = child_in[queue_id]['args'] - error(self._vim, name) self.debug('main_loop: %s', name) - # if name == 'add_source': - # self._add_source(args[0]) - # elif name == 'add_filter': - # self._add_filter(args[0]) - # elif name == 'set_source_attributes': - # self._set_source_attributes(args[0]) - # elif name == 'set_custom': - # self._set_custom(args[0]) - # elif name == 'on_event': - # self._on_event(args[0]) - # elif name == 'merge_results': - # self._merge_results(args[0]) + if name == 'add_source': + self._add_source(args[0]) + elif name == 'add_filter': + self._add_filter(args[0]) + elif name == 'set_source_attributes': + self._set_source_attributes(args[0]) + elif name == 'set_custom': + self._set_custom(args[0]) + elif name == 'on_event': + self._on_event(args[0]) + elif name == 'merge_results': + self._merge_results(args[0], queue_id) def _add_source(self, path): source = None @@ -125,7 +125,13 @@ def _merge_results(self, context, queue_id): if not self._is_skip(x['context'], x['source'])]: source_result = self._source_result(result, context['input']) if source_result: - merged_results.append(source_result) + merged_results.append({ + 'input': source_result['input'], + 'complete_position': source_result['complete_position'], + 'mark': result['source'].mark, + 'filetypes': result['source'].filetypes, + 'candidates': result['candidates'], + }) is_async = len([x for x in results if x['context']['is_async']]) > 0 @@ -199,6 +205,9 @@ def _gather_results(self, context): 'is_async': ctx['is_async'], 'prev_linenr': ctx['position'][1], 'prev_input': ctx['input'], + 'input': ctx['input'], + 'complete_position': ctx['complete_position'], + 'candidates': ctx['candidates'], } self._prev_results[source.name] = result results.append(result) @@ -268,8 +277,8 @@ def _source_result(self, result, context_input): if result['is_async']: self._gather_async_results(result, source) - if not result['context']['candidates']: - return [] + if not result['candidates']: + return None # Source context ctx = copy.deepcopy(result['context']) @@ -296,9 +305,8 @@ def _source_result(self, result, context_input): if hasattr(source, 'on_post_filter'): ctx['candidates'] = source.on_post_filter(ctx) - if ctx['candidates']: - return [ctx['candidates'], result] - return [] + result['candidates'] = ctx['candidates'] + return result if result['candidates'] else None def _itersource(self, context): sources = sorted(self._sources.items(), diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index b88947f0..13cca3b1 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -6,7 +6,8 @@ from deoplete import logger from deoplete.parent import Parent -from deoplete.util import (error, error_tb, find_rplugins) +from deoplete.util import (error_tb, find_rplugins) +# from deoplete.util import error class Deoplete(logger.LoggingMixin): @@ -92,26 +93,26 @@ def merge_results(self, context): if not merged_results: return (is_async, -1, []) - complete_position = min([x[1]['context']['complete_position'] + complete_position = min([x['complete_position'] for x in merged_results]) all_candidates = [] - for [candidates, result] in merged_results: - ctx = result['context'] - source = result['source'] - prefix = ctx['input'][complete_position:ctx['complete_position']] + for result in merged_results: + candidates = result['candidates'] + prefix = result['input'][ + complete_position:result['complete_position']] - mark = source.mark + ' ' + mark = result['mark'] + ' ' for candidate in candidates: # Add prefix candidate['word'] = prefix + candidate['word'] # Set default menu and icase candidate['icase'] = 1 - if (source.mark != '' and + if (mark != ' ' and candidate.get('menu', '').find(mark) != 0): candidate['menu'] = mark + candidate.get('menu', '') - if source.filetypes: + if result['filetypes']: candidate['dup'] = 1 all_candidates += candidates diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index 30fa1cd4..e85a2e19 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -8,7 +8,7 @@ from deoplete import logger from deoplete.process import Process -from deoplete.util import error +# from deoplete.util import error class Parent(logger.LoggingMixin): @@ -45,10 +45,14 @@ def merge_results(self, context): if not queue_id: return (False, []) - time.sleep(0.5) + time.sleep(1.0) results = self._get(queue_id) - return results if results else (False, []) + if not results: + return (False, []) + self._vim.vars['deoplete#_child_out'] = {} + return (results['is_async'], + results['merged_results']) if results else (False, []) def on_event(self, context): if context['event'] == 'VimLeavePre': @@ -60,8 +64,6 @@ def _start_process(self, context, serveraddr): self._proc = Process( [context['python3'], context['dp_main'], serveraddr], context, context['cwd']) - time.sleep(1) - error(self._vim, self._proc.communicate(100)) def _stop_process(self): if self._proc: diff --git a/rplugin/python3/deoplete/process.py b/rplugin/python3/deoplete/process.py index 3d48d5b4..d2f981fa 100644 --- a/rplugin/python3/deoplete/process.py +++ b/rplugin/python3/deoplete/process.py @@ -5,9 +5,6 @@ # ============================================================================ import subprocess -from threading import Thread -from queue import Queue -from time import time import os @@ -19,15 +16,12 @@ def __init__(self, commands, context, cwd): startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW self.__proc = subprocess.Popen(commands, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, startupinfo=startupinfo, cwd=cwd) self.__eof = False self.__context = context - self.__queue_out = Queue() - self.__thread = Thread(target=self.enqueue_output) - self.__thread.start() def eof(self): return self.__eof @@ -38,42 +32,8 @@ def kill(self): self.__proc.kill() self.__proc.wait() self.__proc = None - self.__queue_out = None - self.__thread.join(1.0) - self.__thread = None def write(self, text): self.__proc.stdin.write(text.encode( self.__context['encoding'], errors='replace')) self.__proc.stdin.flush() - - def enqueue_output(self): - for line in self.__proc.stdout: - if not self.__queue_out: - return - self.__queue_out.put( - line.decode(self.__context['encoding'], - errors='replace').strip('\r\n')) - - def communicate(self, timeout): - if not self.__proc: - return ([], []) - - start = time() - outs = [] - - while not self.__queue_out.empty() and time() < start + timeout: - outs.append(self.__queue_out.get_nowait()) - - if self.__thread.is_alive() or not self.__queue_out.empty(): - return (outs, []) - - _, errs = self.__proc.communicate(timeout=timeout) - errs = errs.decode(self.__context['encoding'], - errors='replace').splitlines() - self.__eof = True - self.__proc = None - self.__thread = None - self.__queue = None - - return (outs, errs) From 4d7f09eb566e7f4c3e89741d690fde600a67850d Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 8 Jan 2018 18:54:54 +0900 Subject: [PATCH 22/40] Clear child_in variable --- rplugin/python3/deoplete/child.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 89883e81..6709e175 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -53,6 +53,10 @@ def main_loop(self): args = child_in[queue_id]['args'] self.debug('main_loop: %s', name) + # Clear child_in + child_in[queue_id] = {} + self._vim.vars['deoplete#_child_in'] = child_in + if name == 'add_source': self._add_source(args[0]) elif name == 'add_filter': From e913bb1c0efae208140029e6f1771163ede84041 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 09:39:34 +0900 Subject: [PATCH 23/40] Fix error check --- rplugin/python3/deoplete/child.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 6709e175..4959ae41 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -88,7 +88,7 @@ def _add_source(self, path): (path, self._loaded_sources[source.name])) source = None except Exception: - error_tb(self._vim, 'Could not load source: %s' % name) + error_tb(self._vim, 'Could not load source: %s' % path) finally: if source: self._loaded_sources[source.name] = path @@ -114,7 +114,7 @@ def _add_filter(self, path): f = None except Exception: # Exception occurred when loading a filter. Log stack trace. - error_tb(self._vim, 'Could not load filter: %s' % name) + error_tb(self._vim, 'Could not load filter: %s' % path) finally: if f: self._loaded_filters[f.name] = path From 933df0f5358b988162192f3dc50571670d7a4ab4 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 09:40:37 +0900 Subject: [PATCH 24/40] Fix import error --- rplugin/python3/deoplete/child.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 4959ae41..c099bb75 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -12,6 +12,9 @@ from collections import defaultdict +import deoplete.source # noqa +import deoplete.filter # noqa + from deoplete import logger from deoplete.exceptions import SourceInitError from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb, From b9888877cd56f5b8632b26953c8a829ff848e44b Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 10:03:46 +0900 Subject: [PATCH 25/40] Use msgpack instead --- autoload/deoplete/handler.vim | 2 - rplugin/python3/deoplete/child.py | 62 ++++++++++++++--------------- rplugin/python3/deoplete/parent.py | 18 ++------- rplugin/python3/deoplete/process.py | 42 +++++++++++-------- 4 files changed, 58 insertions(+), 66 deletions(-) diff --git a/autoload/deoplete/handler.vim b/autoload/deoplete/handler.vim index f9c0e432..1db870a8 100644 --- a/autoload/deoplete/handler.vim +++ b/autoload/deoplete/handler.vim @@ -213,8 +213,6 @@ endfunction function! s:on_insert_leave() abort call deoplete#mapping#_restore_completeopt() let g:deoplete#_context = {} - let g:deoplete#_child_in = {} - let g:deoplete#_child_out = {} endfunction function! s:on_complete_done() abort diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index c099bb75..0c2fbbf2 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -9,6 +9,7 @@ import re import sys import time +import msgpack from collections import defaultdict @@ -44,34 +45,33 @@ def enable_logging(self): self.is_debug_enabled = True def main_loop(self): - for queue_id in sys.stdin: - queue_id = queue_id.strip() - - if queue_id not in self._vim.vars['deoplete#_child_in']: - continue - - self.debug('main_loop: begin') - child_in = self._vim.vars['deoplete#_child_in'] - name = child_in[queue_id]['name'] - args = child_in[queue_id]['args'] - self.debug('main_loop: %s', name) - - # Clear child_in - child_in[queue_id] = {} - self._vim.vars['deoplete#_child_in'] = child_in - - if name == 'add_source': - self._add_source(args[0]) - elif name == 'add_filter': - self._add_filter(args[0]) - elif name == 'set_source_attributes': - self._set_source_attributes(args[0]) - elif name == 'set_custom': - self._set_custom(args[0]) - elif name == 'on_event': - self._on_event(args[0]) - elif name == 'merge_results': - self._merge_results(args[0], queue_id) + unpacker = msgpack.Unpacker(encoding='utf-8') + + while 1: + b = sys.stdin.buffer.read(1) + unpacker.feed(b) + + for child_in in unpacker: + self.debug('main_loop: begin') + name = child_in['name'] + args = child_in['args'] + self.debug('main_loop: %s', name) + + if name == 'add_source': + self._add_source(args[0]) + elif name == 'add_filter': + self._add_filter(args[0]) + elif name == 'set_source_attributes': + self._set_source_attributes(args[0]) + elif name == 'set_custom': + self._set_custom(args[0]) + elif name == 'on_event': + self._on_event(args[0]) + elif name == 'merge_results': + result = self._merge_results(args[0]) + sys.stdout.buffer.write(msgpack.packb( + result, use_bin_type=True)) + sys.stdout.flush() def _add_source(self, path): source = None @@ -124,7 +124,7 @@ def _add_filter(self, path): self._filters[f.name] = f self.debug('Loaded Filter: %s (%s)', f.name, path) - def _merge_results(self, context, queue_id): + def _merge_results(self, context): results = self._gather_results(context) merged_results = [] @@ -142,11 +142,9 @@ def _merge_results(self, context, queue_id): is_async = len([x for x in results if x['context']['is_async']]) > 0 - child_out = self._vim.vars['deoplete#_child_out'] - child_out[queue_id] = { + return { 'is_async': is_async, 'merged_results': merged_results } - self._vim.vars['deoplete#_child_out'] = child_out def _gather_results(self, context): results = [] diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index e85a2e19..a91f9b96 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -8,7 +8,7 @@ from deoplete import logger from deoplete.process import Process -# from deoplete.util import error +from deoplete.util import error class Parent(logger.LoggingMixin): @@ -19,11 +19,6 @@ def __init__(self, vim): self._vim = vim self._proc = None - if 'deoplete#_child_in' not in self._vim.vars: - self._vim.vars['deoplete#_child_in'] = {} - if 'deoplete#_child_out' not in self._vim.vars: - self._vim.vars['deoplete#_child_out'] = {} - def enable_logging(self): self.is_debug_enabled = True @@ -45,12 +40,9 @@ def merge_results(self, context): if not queue_id: return (False, []) - time.sleep(1.0) - results = self._get(queue_id) if not results: return (False, []) - self._vim.vars['deoplete#_child_out'] = {} return (results['is_async'], results['merged_results']) if results else (False, []) @@ -76,12 +68,8 @@ def _put(self, name, args): queue_id = str(time.time()) - child_in = self._vim.vars['deoplete#_child_in'] - child_in[queue_id] = {'name': name, 'args': args} - self._vim.vars['deoplete#_child_in'] = child_in - - self._proc.write(queue_id + '\n') + self._proc.write({'name': name, 'args': args, 'queue_id': queue_id}) return queue_id def _get(self, queue_id): - return self._vim.vars['deoplete#_child_out'].get(queue_id, None) + return self._proc.read() diff --git a/rplugin/python3/deoplete/process.py b/rplugin/python3/deoplete/process.py index d2f981fa..d59ac23c 100644 --- a/rplugin/python3/deoplete/process.py +++ b/rplugin/python3/deoplete/process.py @@ -6,6 +6,7 @@ import subprocess import os +import msgpack class Process(object): @@ -14,26 +15,33 @@ def __init__(self, commands, context, cwd): if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - self.__proc = subprocess.Popen(commands, - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - startupinfo=startupinfo, - cwd=cwd) - self.__eof = False - self.__context = context + self._proc = subprocess.Popen(commands, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + startupinfo=startupinfo, + cwd=cwd) + self._eof = False + self._context = context + self._unpacker = msgpack.Unpacker(encoding='utf-8') def eof(self): - return self.__eof + return self._eof def kill(self): - if not self.__proc: + if not self._proc: return - self.__proc.kill() - self.__proc.wait() - self.__proc = None + self._proc.kill() + self._proc.wait() + self._proc = None - def write(self, text): - self.__proc.stdin.write(text.encode( - self.__context['encoding'], errors='replace')) - self.__proc.stdin.flush() + def read(self): + while 1: + b = self._proc.stdout.read(1) + self._unpacker.feed(b) + for child_out in self._unpacker: + return child_out + + def write(self, expr): + self._proc.stdin.write(msgpack.packb(expr, use_bin_type=True)) + self._proc.stdin.flush() From e1560dd8b785c337bc25bcafbfaed73e399e8ef2 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 11:46:06 +0900 Subject: [PATCH 26/40] Change private attributes --- .../python3/deoplete/filter/matcher_cpsm.py | 12 ++++----- rplugin/python3/deoplete/source/buffer.py | 14 +++++----- rplugin/python3/deoplete/source/dictionary.py | 22 ++++++++-------- rplugin/python3/deoplete/source/file.py | 22 ++++++++-------- rplugin/python3/deoplete/source/member.py | 26 +++++++++---------- rplugin/python3/deoplete/source/omni.py | 12 ++++----- rplugin/python3/deoplete/source/tag.py | 22 ++++++++-------- 7 files changed, 65 insertions(+), 65 deletions(-) diff --git a/rplugin/python3/deoplete/filter/matcher_cpsm.py b/rplugin/python3/deoplete/filter/matcher_cpsm.py index fc1a68f6..04172cc6 100644 --- a/rplugin/python3/deoplete/filter/matcher_cpsm.py +++ b/rplugin/python3/deoplete/filter/matcher_cpsm.py @@ -18,15 +18,15 @@ def __init__(self, vim): self.name = 'matcher_cpsm' self.description = 'cpsm matcher' - self.__initialized = False - self.__disabled = False + self._initialized = False + self._disabled = False def filter(self, context): if not context['candidates'] or not context[ - 'input'] or self.__disabled: + 'input'] or self._disabled: return context['candidates'] - if not self.__initialized: + if not self._initialized: # cpsm installation check ext = '.pyd' if context['is_windows'] else '.so' if globruntime(context['runtimepath'], 'bin/cpsm_py' + ext): @@ -34,13 +34,13 @@ def filter(self, context): sys.path.append(os.path.dirname( globruntime(context['runtimepath'], 'bin/cpsm_py' + ext)[0])) - self.__initialized = True + self._initialized = True else: error(self.vim, 'matcher_cpsm: bin/cpsm_py' + ext + ' is not found in your runtimepath.') error(self.vim, 'matcher_cpsm: You must install/build' + ' Python3 support enabled cpsm.') - self.__disabled = True + self._disabled = True return [] cpsm_result = self._get_cpsm_result( diff --git a/rplugin/python3/deoplete/source/buffer.py b/rplugin/python3/deoplete/source/buffer.py index 06168a7e..cf5b196d 100644 --- a/rplugin/python3/deoplete/source/buffer.py +++ b/rplugin/python3/deoplete/source/buffer.py @@ -17,14 +17,14 @@ def __init__(self, vim): self.name = 'buffer' self.mark = '[B]' self.limit = 1000000 - self.__buffers = {} - self.__max_lines = 5000 + self._buffers = {} + self._max_lines = 5000 def on_event(self, context): bufnr = context['bufnr'] - if (bufnr not in self.__buffers or + if (bufnr not in self._buffers or context['event'] == 'BufWritePost'): - self.__make_cache(context, bufnr) + self._make_cache(context, bufnr) def gather_candidates(self, context): self.on_event(context) @@ -32,16 +32,16 @@ def gather_candidates(self, context): same_filetype = context['vars'].get( 'deoplete#buffer#require_same_filetype', True) return {'sorted_candidates': [ - x['candidates'] for x in self.__buffers.values() + x['candidates'] for x in self._buffers.values() if not same_filetype or x['filetype'] in context['filetypes'] or x['filetype'] in context['same_filetypes'] or x['bufnr'] in tab_bufnrs ]} - def __make_cache(self, context, bufnr): + def _make_cache(self, context, bufnr): try: - self.__buffers[bufnr] = { + self._buffers[bufnr] = { 'bufnr': bufnr, 'filetype': self.vim.eval('&l:filetype'), 'candidates': [ diff --git a/rplugin/python3/deoplete/source/dictionary.py b/rplugin/python3/deoplete/source/dictionary.py index 78387b0c..73f661b8 100644 --- a/rplugin/python3/deoplete/source/dictionary.py +++ b/rplugin/python3/deoplete/source/dictionary.py @@ -19,32 +19,32 @@ def __init__(self, vim): self.name = 'dictionary' self.mark = '[D]' - self.__cache = {} + self._cache = {} def on_event(self, context): - self.__make_cache(context) + self._make_cache(context) def gather_candidates(self, context): - self.__make_cache(context) + self._make_cache(context) candidates = [] - for filename in [x for x in self.__get_dictionaries(context) - if x in self.__cache]: - candidates.append(self.__cache[filename].candidates) + for filename in [x for x in self._get_dictionaries(context) + if x in self._cache]: + candidates.append(self._cache[filename].candidates) return {'sorted_candidates': candidates} - def __make_cache(self, context): - for filename in self.__get_dictionaries(context): + def _make_cache(self, context): + for filename in self._get_dictionaries(context): mtime = getmtime(filename) - if filename in self.__cache and self.__cache[ + if filename in self._cache and self._cache[ filename].mtime == mtime: continue with open(filename, 'r', errors='replace') as f: - self.__cache[filename] = DictCacheItem( + self._cache[filename] = DictCacheItem( mtime, [{'word': x} for x in sorted( [x.strip() for x in f], key=str.lower)] ) - def __get_dictionaries(self, context): + def _get_dictionaries(self, context): return [x for x in context['dict__dictionary'].split(',') if exists(x)] diff --git a/rplugin/python3/deoplete/source/file.py b/rplugin/python3/deoplete/source/file.py index b5c6ca80..34bd36b7 100644 --- a/rplugin/python3/deoplete/source/file.py +++ b/rplugin/python3/deoplete/source/file.py @@ -21,14 +21,14 @@ def __init__(self, vim): self.mark = '[F]' self.min_pattern_length = 0 self.rank = 150 - self.__isfname = '' + self._isfname = '' def on_init(self, context): - self.__buffer_path = context['vars'].get( + self._buffer_path = context['vars'].get( 'deoplete#file#enable_buffer_path', 0) def on_event(self, context): - self.__isfname = self.vim.call( + self._isfname = self.vim.call( 'deoplete#util#vimoption2python_not', self.vim.options['isfname']) @@ -37,13 +37,13 @@ def get_complete_position(self, context): return pos if pos < 0 else pos + 1 def gather_candidates(self, context): - if not self.__isfname: + if not self._isfname: return [] - p = self.__longest_path_that_exists(context, context['input']) + p = self._longest_path_that_exists(context, context['input']) if p in (None, []) or p == '/' or re.search('//+$', p): return [] - complete_str = self.__substitute_path(context, dirname(p) + '/') + complete_str = self._substitute_path(context, dirname(p) + '/') if not os.path.isdir(complete_str): return [] hidden = context['complete_str'].find('.') == 0 @@ -60,19 +60,19 @@ def gather_candidates(self, context): return [{'word': x, 'abbr': x + '/'} for x in dirs ] + [{'word': x} for x in files] - def __longest_path_that_exists(self, context, input_str): + def _longest_path_that_exists(self, context, input_str): input_str = re.sub(r'[^/]*$', '', input_str) data = re.split(r'((?:%s+|(?:(?']) - set_pattern(self.__prefix_patterns, + set_pattern(self._prefix_patterns, 'cpp,objcpp', ['\.', '->', '::']) - set_pattern(self.__prefix_patterns, + set_pattern(self._prefix_patterns, 'perl,php', ['->']) - set_pattern(self.__prefix_patterns, + set_pattern(self._prefix_patterns, 'ruby', ['\.', '::']) - set_pattern(self.__prefix_patterns, + set_pattern(self._prefix_patterns, 'lua', ['\.', ':']) def get_complete_position(self, context): @@ -44,12 +44,12 @@ def get_complete_position(self, context): get_buffer_config(context, context['filetype'], 'deoplete_member_prefix_patterns', 'deoplete#member#prefix_patterns', - self.__prefix_patterns)): - m = re.search(self.__object_pattern + prefix_pattern + r'\w*$', + self._prefix_patterns)): + m = re.search(self._object_pattern + prefix_pattern + r'\w*$', context['input']) if m is None or prefix_pattern == '': continue - self.__prefix = re.sub(r'\w*$', '', m.group(0)) + self._prefix = re.sub(r'\w*$', '', m.group(0)) return re.search(r'\w*$', context['input']).start() return -1 @@ -57,6 +57,6 @@ def gather_candidates(self, context): return [{'word': x} for x in parse_buffer_pattern( getlines(self.vim), - r'(?<=' + re.escape(self.__prefix) + r')\w+' + r'(?<=' + re.escape(self._prefix) + r')\w+' ) if x != context['complete_str']] diff --git a/rplugin/python3/deoplete/source/omni.py b/rplugin/python3/deoplete/source/omni.py index 3163c53e..fc48c6e7 100644 --- a/rplugin/python3/deoplete/source/omni.py +++ b/rplugin/python3/deoplete/source/omni.py @@ -48,7 +48,7 @@ def _get_complete_position(self, context, current_ft, filetype): omnifunc = context['omni__omnifunc'] if omnifunc == '': continue - self.__omnifunc = omnifunc + self._omnifunc = omnifunc for input_pattern in convert2list( get_buffer_config(context, filetype, 'deoplete_omni_input_patterns', @@ -62,7 +62,7 @@ def _get_complete_position(self, context, current_ft, filetype): 'Manual' and m is None): continue - if filetype == current_ft and self.__omnifunc in [ + if filetype == current_ft and self._omnifunc in [ 'ccomplete#Complete', 'htmlcomplete#CompleteTags', 'LanguageClient#complete', @@ -70,24 +70,24 @@ def _get_complete_position(self, context, current_ft, filetype): # In the blacklist return -1 try: - complete_pos = self.vim.call(self.__omnifunc, 1, '') + complete_pos = self.vim.call(self._omnifunc, 1, '') except Exception as e: self.print_error('Error occurred calling omnifunction: ' + - self.__omnifunc) + self._omnifunc) return -1 return complete_pos return -1 def gather_candidates(self, context): try: - candidates = self.vim.call(self.__omnifunc, 0, '') + candidates = self.vim.call(self._omnifunc, 0, '') if isinstance(candidates, dict): candidates = candidates['words'] elif isinstance(candidates, int): candidates = [] except Exception as e: self.print_error('Error occurred calling omnifunction: ' + - self.__omnifunc) + self._omnifunc) candidates = [] candidates = convert2candidates(candidates) diff --git a/rplugin/python3/deoplete/source/tag.py b/rplugin/python3/deoplete/source/tag.py index 510071f1..1dcec7f3 100644 --- a/rplugin/python3/deoplete/source/tag.py +++ b/rplugin/python3/deoplete/source/tag.py @@ -23,26 +23,26 @@ def __init__(self, vim): self.name = 'tag' self.mark = '[T]' - self.__cache = {} + self._cache = {} def on_init(self, context): - self.__limit = context['vars'].get( + self._limit = context['vars'].get( 'deoplete#tag#cache_limit_size', 500000) def on_event(self, context): - self.__make_cache(context) + self._make_cache(context) def gather_candidates(self, context): - self.__make_cache(context) + self._make_cache(context) candidates = [] - for c in self.__cache.values(): + for c in self._cache.values(): candidates.extend(c.candidates) return candidates - def __make_cache(self, context): - for filename in self.__get_tagfiles(context): + def _make_cache(self, context): + for filename in self._get_tagfiles(context): mtime = getmtime(filename) - if filename in self.__cache and self.__cache[ + if filename in self._cache and self._cache[ filename].mtime == mtime: continue @@ -66,14 +66,14 @@ def __make_cache(self, context): if not items: continue - self.__cache[filename] = TagsCacheItem( + self._cache[filename] = TagsCacheItem( mtime, sorted(items, key=lambda x: x['word'].lower())) - def __get_tagfiles(self, context): + def _get_tagfiles(self, context): include_files = self.vim.call( 'neoinclude#include#get_tag_files') if self.vim.call( 'exists', '*neoinclude#include#get_tag_files') else [] return [x for x in self.vim.call( 'map', self.vim.call('tagfiles') + include_files, 'fnamemodify(v:val, ":p")') - if exists(x) and getsize(x) < self.__limit] + if exists(x) and getsize(x) < self._limit] From e320da659ca2901d5702d6cc070ae459a1b09d9e Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 12:37:46 +0900 Subject: [PATCH 27/40] [RFC] parallel feature --- autoload/deoplete/handler.vim | 19 +++++++------ autoload/deoplete/init.vim | 7 +++++ doc/deoplete.txt | 8 ++++++ rplugin/python3/deoplete/child.py | 9 ++++-- rplugin/python3/deoplete/deoplete.py | 10 ++++++- rplugin/python3/deoplete/parent.py | 41 +++++++++++++++++++--------- rplugin/python3/deoplete/process.py | 26 ++++++++++++++++-- 7 files changed, 93 insertions(+), 27 deletions(-) diff --git a/autoload/deoplete/handler.vim b/autoload/deoplete/handler.vim index 1db870a8..792508c9 100644 --- a/autoload/deoplete/handler.vim +++ b/autoload/deoplete/handler.vim @@ -51,15 +51,16 @@ function! s:do_complete(timer) abort return endif + let prev = g:deoplete#_prev_completion if context.event !=# 'Manual' - \ && s:prev_completion.complete_position == getpos('.') - \ && s:prev_completion.candidates ==# context.candidates + \ && prev.complete_position == getpos('.') + \ && prev.candidates ==# context.candidates return endif - let s:prev_completion.event = context.event - let s:prev_completion.candidates = context.candidates - let s:prev_completion.complete_position = getpos('.') + let prev.event = context.event + let prev.candidates = context.candidates + let prev.complete_position = getpos('.') if context.event ==# 'Manual' let context.event = '' @@ -86,8 +87,10 @@ function! deoplete#handler#_completion_timer_start() abort let delay = max([50, g:deoplete#auto_complete_delay]) let s:completion_timer = timer_start(delay, function('s:do_complete')) - let s:prev_completion = { - \ 'complete_position': [], 'candidates': [], 'event': '' + let g:deoplete#_prev_completion = { + \ 'complete_position': [], + \ 'candidates': [], + \ 'event': '', \ } endfunction function! s:completion_timer_stop() abort @@ -132,7 +135,7 @@ function! s:completion_begin(event) abort return endif - if exists('s:prev_completion') && s:prev_completion.event !=# 'Manual' + if g:deoplete#_prev_completion.event !=# 'Manual' " Call omni completion for filetype in context.filetypes for pattern in deoplete#util#convert2list( diff --git a/autoload/deoplete/init.vim b/autoload/deoplete/init.vim index afacdaf7..6b25dbc0 100644 --- a/autoload/deoplete/init.vim +++ b/autoload/deoplete/init.vim @@ -103,6 +103,11 @@ function! deoplete#init#_disable() abort endfunction function! deoplete#init#_variables() abort + let g:deoplete#_prev_completion = { + \ 'complete_position': [], + \ 'candidates': [], + \ 'event': '', + \ } let g:deoplete#_context = {} let g:deoplete#_rank = {} if !exists('g:deoplete#_logging') @@ -147,6 +152,8 @@ function! deoplete#init#_variables() abort \ 'g:deoplete#skip_chars', []) call deoplete#util#set_default( \ 'g:deoplete#complete_method', 'complete') + call deoplete#util#set_default( + \ 'g:deoplete#max_processes', 4) call deoplete#util#set_default( \ 'g:deoplete#keyword_patterns', {}) diff --git a/doc/deoplete.txt b/doc/deoplete.txt index 1c748c68..5d0b392f 100644 --- a/doc/deoplete.txt +++ b/doc/deoplete.txt @@ -195,6 +195,14 @@ g:deoplete#max_list Default value: 100 + *g:deoplete#max_processes* +g:deoplete#max_processes + The max number of processes used for the asynchronous + completion. + If it is less than equal 1, this feature is disabled. + + Default value: 4 + *g:deoplete#max_abbr_width* g:deoplete#max_abbr_width If the candidate abbr length exceeds the length it will be cut diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 0c2fbbf2..19a291aa 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -55,6 +55,7 @@ def main_loop(self): self.debug('main_loop: begin') name = child_in['name'] args = child_in['args'] + queue_id = child_in['queue_id'] self.debug('main_loop: %s', name) if name == 'add_source': @@ -68,7 +69,7 @@ def main_loop(self): elif name == 'on_event': self._on_event(args[0]) elif name == 'merge_results': - result = self._merge_results(args[0]) + result = self._merge_results(args[0], queue_id) sys.stdout.buffer.write(msgpack.packb( result, use_bin_type=True)) sys.stdout.flush() @@ -124,7 +125,7 @@ def _add_filter(self, path): self._filters[f.name] = f self.debug('Loaded Filter: %s (%s)', f.name, path) - def _merge_results(self, context): + def _merge_results(self, context, queue_id): results = self._gather_results(context) merged_results = [] @@ -143,7 +144,9 @@ def _merge_results(self, context): is_async = len([x for x in results if x['context']['is_async']]) > 0 return { - 'is_async': is_async, 'merged_results': merged_results + 'queue_id': queue_id, + 'is_async': is_async, + 'merged_results': merged_results, } def _gather_results(self, context): diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index 13cca3b1..f940ecf8 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -22,7 +22,8 @@ def __init__(self, vim): self._parents = [] self._parent_count = 0 - self._max_parents = 1 + self._max_parents = max( + [1, self._vim.vars['deoplete#max_processes']]) for n in range(0, self._max_parents): self._parents.append(Parent(vim)) @@ -71,12 +72,19 @@ def completion_begin(self, context): in context['vars']): self._vim.call('deoplete#mapping#_restore_completeopt') + # Check the previous completion + prev_candidates = context['vars'][ + 'deoplete#_prev_completion']['candidates'] + if context['event'] == 'Async' and candidates == prev_candidates: + return + # error(self._vim, candidates) self._vim.vars['deoplete#_context'] = { 'complete_position': position, 'candidates': candidates, 'event': context['event'], 'input': context['input'], + 'is_async': is_async, } self._vim.call('deoplete#handler#_completion_timer_start') diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index a91f9b96..cdf9d897 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -8,7 +8,7 @@ from deoplete import logger from deoplete.process import Process -from deoplete.util import error +# from deoplete.util import error class Parent(logger.LoggingMixin): @@ -18,6 +18,7 @@ def __init__(self, vim): self._vim = vim self._proc = None + self._queue_id = '' def enable_logging(self): self.is_debug_enabled = True @@ -36,13 +37,24 @@ def set_custom(self, custom): self._put('set_custom', [custom]) def merge_results(self, context): - queue_id = self._put('merge_results', [context]) - if not queue_id: - return (False, []) - - results = self._get(queue_id) - if not results: - return (False, []) + prev_pos = context['vars'][ + 'deoplete#_prev_completion']['complete_position'] + if context['position'] == prev_pos and ( + self._queue_id or context['event'] == 'Async'): + # Use previous id + queue_id = self._queue_id + else: + queue_id = self._put('merge_results', [context]) + if not queue_id: + return (False, []) + + get = self._get(queue_id) + if not get: + # Skip the next merge_results + self._queue_id = queue_id + return (True, []) + self._queue_id = '' + results = get[0] return (results['is_async'], results['merged_results']) if results else (False, []) @@ -52,10 +64,12 @@ def on_event(self, context): self._put('on_event', [context]) def _start_process(self, context, serveraddr): - if not self._proc: - self._proc = Process( - [context['python3'], context['dp_main'], serveraddr], - context, context['cwd']) + if self._proc: + return + + self._proc = Process( + [context['python3'], context['dp_main'], serveraddr], + context, context['cwd']) def _stop_process(self): if self._proc: @@ -72,4 +86,5 @@ def _put(self, name, args): return queue_id def _get(self, queue_id): - return self._proc.read() + return [x for x in self._proc.communicate(50) + if x['queue_id'] == queue_id] diff --git a/rplugin/python3/deoplete/process.py b/rplugin/python3/deoplete/process.py index d59ac23c..89f05da8 100644 --- a/rplugin/python3/deoplete/process.py +++ b/rplugin/python3/deoplete/process.py @@ -7,6 +7,9 @@ import subprocess import os import msgpack +from threading import Thread +from queue import Queue +from time import time, sleep class Process(object): @@ -24,6 +27,9 @@ def __init__(self, commands, context, cwd): self._eof = False self._context = context self._unpacker = msgpack.Unpacker(encoding='utf-8') + self._queue_out = Queue() + self._thread = Thread(target=self.enqueue_output) + self._thread.start() def eof(self): return self._eof @@ -34,13 +40,29 @@ def kill(self): self._proc.kill() self._proc.wait() self._proc = None + self._queue_out = None + self._thread.join(1.0) + self._thread = None - def read(self): + def enqueue_output(self): while 1: b = self._proc.stdout.read(1) self._unpacker.feed(b) for child_out in self._unpacker: - return child_out + self._queue_out.put(child_out) + + def communicate(self, timeout): + if not self._proc: + return [] + + start = time() + outs = [] + + if self._queue_out.empty(): + sleep(timeout / 1000.0) + while not self._queue_out.empty() and time() < start + timeout: + outs.append(self._queue_out.get_nowait()) + return outs def write(self, expr): self._proc.stdin.write(msgpack.packb(expr, use_bin_type=True)) From bc5deb95a70d80171373d283cf08d2c41c6cc7f3 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 13:50:42 +0900 Subject: [PATCH 28/40] Fix sorting problem --- rplugin/python3/deoplete/child.py | 15 +++++++-------- rplugin/python3/deoplete/deoplete.py | 28 +++++++++++++++++++++------- rplugin/python3/deoplete/parent.py | 6 +++--- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 19a291aa..509cdfc7 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -133,12 +133,16 @@ def _merge_results(self, context, queue_id): if not self._is_skip(x['context'], x['source'])]: source_result = self._source_result(result, context['input']) if source_result: + rank = get_custom(self._custom, + result['source'].name, 'rank', + result['source'].rank) merged_results.append({ - 'input': source_result['input'], 'complete_position': source_result['complete_position'], 'mark': result['source'].mark, - 'filetypes': result['source'].filetypes, + 'dup': bool(result['source'].filetypes), 'candidates': result['candidates'], + 'source_name': result['source'].name, + 'rank': rank, }) is_async = len([x for x in results if x['context']['is_async']]) > 0 @@ -317,11 +321,6 @@ def _source_result(self, result, context_input): return result if result['candidates'] else None def _itersource(self, context): - sources = sorted(self._sources.items(), - key=lambda x: get_custom( - context['custom'], - x[1].name, 'rank', x[1].rank), - reverse=True) filetypes = context['filetypes'] ignore_sources = set() for ft in filetypes: @@ -331,7 +330,7 @@ def _itersource(self, context): 'deoplete#ignore_sources', {})) - for source_name, source in sources: + for source_name, source in self._sources.items(): if source.limit > 0 and context['bufsize'] > source.limit: continue if source.filetypes is None or source_name in ignore_sources: diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index f940ecf8..db38bb4c 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -19,6 +19,8 @@ def __init__(self, vim): self._runtimepath = '' self._custom = [] self._loaded_paths = set() + self._prev_merged_results = {} + self._prev_pos = [] self._parents = [] self._parent_count = 0 @@ -91,12 +93,23 @@ def completion_begin(self, context): self.debug('completion_end: %s', context['input']) def merge_results(self, context): + use_prev = context['position'] == self._prev_pos + if not use_prev: + self._prev_merged_results = {} + is_async = False merged_results = [] - for parent in self._parents: - result = parent.merge_results(context) - is_async = is_async or result[0] - merged_results += result[1] + for cnt, parent in enumerate(self._parents): + if use_prev and cnt in self._prev_merged_results: + # Use previous result + merged_results += self._prev_merged_results[cnt] + else: + result = parent.merge_results(context) + is_async = is_async or result[0] + if not result[0]: + self._prev_merged_results[cnt] = result[1] + merged_results += result[1] + self._prev_pos = context['position'] if not merged_results: return (is_async, -1, []) @@ -105,9 +118,10 @@ def merge_results(self, context): for x in merged_results]) all_candidates = [] - for result in merged_results: + for result in sorted(merged_results, + key=lambda x: x['rank'], reverse=True): candidates = result['candidates'] - prefix = result['input'][ + prefix = context['input'][ complete_position:result['complete_position']] mark = result['mark'] + ' ' @@ -120,7 +134,7 @@ def merge_results(self, context): if (mark != ' ' and candidate.get('menu', '').find(mark) != 0): candidate['menu'] = mark + candidate.get('menu', '') - if result['filetypes']: + if result['dup']: candidate['dup'] = 1 all_candidates += candidates diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index cdf9d897..f1824eeb 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -19,6 +19,7 @@ def __init__(self, vim): self._vim = vim self._proc = None self._queue_id = '' + self._prev_pos = [] def enable_logging(self): self.is_debug_enabled = True @@ -37,9 +38,7 @@ def set_custom(self, custom): self._put('set_custom', [custom]) def merge_results(self, context): - prev_pos = context['vars'][ - 'deoplete#_prev_completion']['complete_position'] - if context['position'] == prev_pos and ( + if context['position'] == self._prev_pos and ( self._queue_id or context['event'] == 'Async'): # Use previous id queue_id = self._queue_id @@ -52,6 +51,7 @@ def merge_results(self, context): if not get: # Skip the next merge_results self._queue_id = queue_id + self._prev_pos = context['position'] return (True, []) self._queue_id = '' results = get[0] From e832f25a5c735b3e93cf2a275c4ba4f65d17f3e7 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 19:20:12 +0900 Subject: [PATCH 29/40] Fix surrogate pair errors --- rplugin/python3/deoplete/child.py | 10 +++++++--- rplugin/python3/deoplete/process.py | 9 +++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index 509cdfc7..a46e2660 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -45,7 +45,12 @@ def enable_logging(self): self.is_debug_enabled = True def main_loop(self): - unpacker = msgpack.Unpacker(encoding='utf-8') + unpacker = msgpack.Unpacker( + encoding='utf-8', + unicode_errors='surrogateescape') + packer = msgpack.Packer( + use_bin_type=True, + unicode_errors='surrogateescape') while 1: b = sys.stdin.buffer.read(1) @@ -70,8 +75,7 @@ def main_loop(self): self._on_event(args[0]) elif name == 'merge_results': result = self._merge_results(args[0], queue_id) - sys.stdout.buffer.write(msgpack.packb( - result, use_bin_type=True)) + sys.stdout.buffer.write(packer.pack(result)) sys.stdout.flush() def _add_source(self, path): diff --git a/rplugin/python3/deoplete/process.py b/rplugin/python3/deoplete/process.py index 89f05da8..ca880f88 100644 --- a/rplugin/python3/deoplete/process.py +++ b/rplugin/python3/deoplete/process.py @@ -26,7 +26,12 @@ def __init__(self, commands, context, cwd): cwd=cwd) self._eof = False self._context = context - self._unpacker = msgpack.Unpacker(encoding='utf-8') + self._packer = msgpack.Packer( + use_bin_type=True, + unicode_errors='surrogateescape') + self._unpacker = msgpack.Unpacker( + encoding='utf-8', + unicode_errors='surrogateescape') self._queue_out = Queue() self._thread = Thread(target=self.enqueue_output) self._thread.start() @@ -65,5 +70,5 @@ def communicate(self, timeout): return outs def write(self, expr): - self._proc.stdin.write(msgpack.packb(expr, use_bin_type=True)) + self._proc.stdin.write(self._packer.pack(expr)) self._proc.stdin.flush() From b317482e55421041b1e7ea395b9a458432588ac4 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 28 Jan 2018 22:33:00 +0900 Subject: [PATCH 30/40] Improve main loop --- rplugin/python3/deoplete/child.py | 65 +++++++++++++++-------------- rplugin/python3/deoplete/dp_main.py | 3 +- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index a46e2660..d1c88e93 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -40,43 +40,44 @@ def __init__(self, vim): self._source_errors = defaultdict(int) self._filter_errors = defaultdict(int) self._prev_results = {} - - def enable_logging(self): - self.is_debug_enabled = True - - def main_loop(self): - unpacker = msgpack.Unpacker( + self._unpacker = msgpack.Unpacker( encoding='utf-8', unicode_errors='surrogateescape') - packer = msgpack.Packer( + self._packer = msgpack.Packer( use_bin_type=True, unicode_errors='surrogateescape') - while 1: - b = sys.stdin.buffer.read(1) - unpacker.feed(b) - - for child_in in unpacker: - self.debug('main_loop: begin') - name = child_in['name'] - args = child_in['args'] - queue_id = child_in['queue_id'] - self.debug('main_loop: %s', name) - - if name == 'add_source': - self._add_source(args[0]) - elif name == 'add_filter': - self._add_filter(args[0]) - elif name == 'set_source_attributes': - self._set_source_attributes(args[0]) - elif name == 'set_custom': - self._set_custom(args[0]) - elif name == 'on_event': - self._on_event(args[0]) - elif name == 'merge_results': - result = self._merge_results(args[0], queue_id) - sys.stdout.buffer.write(packer.pack(result)) - sys.stdout.flush() + def enable_logging(self): + self.is_debug_enabled = True + + def main(self): + for child_in in self._read(): + self.debug('main_loop: begin') + name = child_in['name'] + args = child_in['args'] + queue_id = child_in['queue_id'] + self.debug('main_loop: %s', name) + + if name == 'add_source': + self._add_source(args[0]) + elif name == 'add_filter': + self._add_filter(args[0]) + elif name == 'set_source_attributes': + self._set_source_attributes(args[0]) + elif name == 'set_custom': + self._set_custom(args[0]) + elif name == 'on_event': + self._on_event(args[0]) + elif name == 'merge_results': + self._write(self._merge_results(args[0], queue_id)) + + def _read(self): + self._unpacker.feed(sys.stdin.buffer.read(1)) + return self._unpacker + + def _write(self, expr): + sys.stdout.buffer.write(self._packer.pack(expr)) + sys.stdout.flush() def _add_source(self, path): source = None diff --git a/rplugin/python3/deoplete/dp_main.py b/rplugin/python3/deoplete/dp_main.py index 9bb820b9..1b04603d 100644 --- a/rplugin/python3/deoplete/dp_main.py +++ b/rplugin/python3/deoplete/dp_main.py @@ -34,7 +34,8 @@ def main(serveraddr): try: from deoplete.child import Child child = Child(vim) - child.main_loop() + while 1: + child.main() except Exception: error_tb(vim, 'Error in child') return From 1ea0d5fde19231b307fc97abccd79d6a9dbf0158 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Tue, 30 Jan 2018 10:11:13 +0900 Subject: [PATCH 31/40] Optimized --- autoload/deoplete/handler.vim | 4 ++-- autoload/deoplete/init.vim | 2 +- rplugin/python3/deoplete/parent.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autoload/deoplete/handler.vim b/autoload/deoplete/handler.vim index 792508c9..9553a5b8 100644 --- a/autoload/deoplete/handler.vim +++ b/autoload/deoplete/handler.vim @@ -84,7 +84,7 @@ function! deoplete#handler#_completion_timer_start() abort call s:completion_timer_stop() endif - let delay = max([50, g:deoplete#auto_complete_delay]) + let delay = max([20, g:deoplete#auto_complete_delay]) let s:completion_timer = timer_start(delay, function('s:do_complete')) let g:deoplete#_prev_completion = { @@ -109,7 +109,7 @@ function! deoplete#handler#_async_timer_start() abort let s:async_timer = { 'event': 'Async', 'changedtick': b:changedtick } let s:async_timer.id = timer_start( - \ max([50, g:deoplete#auto_refresh_delay]), + \ max([20, g:deoplete#auto_refresh_delay]), \ function('s:completion_async'), {'repeat': -1}) endfunction function! deoplete#handler#_async_timer_stop() abort diff --git a/autoload/deoplete/init.vim b/autoload/deoplete/init.vim index 6b25dbc0..43bb9e01 100644 --- a/autoload/deoplete/init.vim +++ b/autoload/deoplete/init.vim @@ -143,7 +143,7 @@ function! deoplete#init#_variables() abort call deoplete#util#set_default( \ 'g:deoplete#auto_complete_delay', 50) call deoplete#util#set_default( - \ 'g:deoplete#auto_refresh_delay', 500) + \ 'g:deoplete#auto_refresh_delay', 50) call deoplete#util#set_default( \ 'g:deoplete#max_abbr_width', 80) call deoplete#util#set_default( diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index f1824eeb..bf977029 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -86,5 +86,5 @@ def _put(self, name, args): return queue_id def _get(self, queue_id): - return [x for x in self._proc.communicate(50) + return [x for x in self._proc.communicate(15) if x['queue_id'] == queue_id] From 2d2e10103d1e5f0be1e1c7759441a6577c14d138 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Tue, 30 Jan 2018 10:33:42 +0900 Subject: [PATCH 32/40] Fix debug feature --- rplugin/python3/deoplete/child.py | 12 ++++++++---- rplugin/python3/deoplete/deoplete.py | 11 +++++++---- rplugin/python3/deoplete/parent.py | 7 ++++--- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/rplugin/python3/deoplete/child.py b/rplugin/python3/deoplete/child.py index d1c88e93..4246d508 100644 --- a/rplugin/python3/deoplete/child.py +++ b/rplugin/python3/deoplete/child.py @@ -47,9 +47,6 @@ def __init__(self, vim): use_bin_type=True, unicode_errors='surrogateescape') - def enable_logging(self): - self.is_debug_enabled = True - def main(self): for child_in in self._read(): self.debug('main_loop: begin') @@ -58,7 +55,9 @@ def main(self): queue_id = child_in['queue_id'] self.debug('main_loop: %s', name) - if name == 'add_source': + if name == 'enable_logging': + self._enable_logging() + elif name == 'add_source': self._add_source(args[0]) elif name == 'add_filter': self._add_filter(args[0]) @@ -79,6 +78,11 @@ def _write(self, expr): sys.stdout.buffer.write(self._packer.pack(expr)) sys.stdout.flush() + def _enable_logging(self): + logging = self._vim.vars['deoplete#_logging'] + logger.setup(self._vim, logging['level'], logging['logfile']) + self.is_debug_enabled = True + def _add_source(self, path): source = None try: diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index db38bb4c..c2a506f6 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -26,8 +26,13 @@ def __init__(self, vim): self._parent_count = 0 self._max_parents = max( [1, self._vim.vars['deoplete#max_processes']]) + + # Init context + context = self._vim.call('deoplete#init#_context', 'Init', []) + context['rpc'] = 'deoplete_on_event' + for n in range(0, self._max_parents): - self._parents.append(Parent(vim)) + self._parents.append(Parent(vim, context)) # Enable logging before "Init" for more information, and e.g. # deoplete-jedi picks up the log filename from deoplete's handler in @@ -36,8 +41,6 @@ def __init__(self, vim): self.enable_logging() # on_init() call - context = self._vim.call('deoplete#init#_context', 'Init', []) - context['rpc'] = 'deoplete_on_event' self.on_event(context) self._vim.vars['deoplete#_initialized'] = True @@ -153,7 +156,7 @@ def load_sources(self, context): continue self._loaded_paths.add(path) - self._parents[self._parent_count].add_source(context, path) + self._parents[self._parent_count].add_source(path) self._parent_count += 1 self._parent_count %= self._max_parents diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index bf977029..52b8ae0f 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -13,19 +13,20 @@ class Parent(logger.LoggingMixin): - def __init__(self, vim): + def __init__(self, vim, context): self.name = 'parent' self._vim = vim self._proc = None self._queue_id = '' self._prev_pos = [] + self._start_process(context, context['serveraddr']) def enable_logging(self): + self._put('enable_logging', []) self.is_debug_enabled = True - def add_source(self, context, path): - self._start_process(context, context['serveraddr']) + def add_source(self, path): self._put('add_source', [path]) def add_filter(self, path): From 2e29bb63621df45bbf2d198f38618e4df9a40a53 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Tue, 30 Jan 2018 11:24:19 +0900 Subject: [PATCH 33/40] Update README --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fcab7d3a..affb05a7 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,8 @@ To view the current options, please consult the ## Installation **Note:** deoplete requires Neovim(latest is recommended) or Vim8 with Python3 and -timers(neovim ver.0.1.5+) enabled. See [requirements](#requirements) if you -aren't sure whether you have this. - -1. Extract the files and put them in your Neovim or .vim directory - (usually `$XDG_CONFIG_HOME/nvim/`). -2. Write `call deoplete#enable()` or `let g:deoplete#enable_at_startup = 1` in - your `init.vim` +timers enabled. See [requirements](#requirements) if you aren't sure whether +you have this. For vim-plug @@ -34,6 +29,7 @@ else Plug 'roxma/nvim-yarp' Plug 'roxma/vim-hug-neovim-rpc' endif +let g:deoplete#enable_at_startup = 1 ``` For dein.vim @@ -44,13 +40,22 @@ if !has('nvim') call dein#add('roxma/nvim-yarp') call dein#add('roxma/vim-hug-neovim-rpc') endif +let g:deoplete#enable_at_startup = 1 ``` +For manual installation(not recommended) + +1. Extract the files and put them in your Neovim or .vim directory + (usually `$XDG_CONFIG_HOME/nvim/`). + +2. Write `call deoplete#enable()` or `let g:deoplete#enable_at_startup = 1` in + your `init.vim` + ## Requirements -deoplete requires Neovim or Vim8 with if\_python3. -If `:echo has("python3")` returns `1`, then you have python 3 support; otherwise, see below. +deoplete requires Neovim or Vim8 with if\_python3. If `:echo has("python3")` +returns `1`, then you have python 3 support; otherwise, see below. You can enable Python3 interface with pip: @@ -76,6 +81,7 @@ auto-completion. let g:deoplete#enable_at_startup = 1 ``` + ## Sources deoplete will display completions via `complete()` by default. @@ -122,5 +128,4 @@ https://www.youtube.com/watch?v=oanoPTpiSF4 [Register/Extract list completions](https://camo.githubusercontent.com/6a6df993ad0e05c014c72c8f8702447f9b34ad90/68747470733a2f2f692e696d6775722e636f6d2f5131663731744a2e676966) -### deoplete-fsharp sample ( Enjoy! ) ![FSharp completion using deopletefs](https://github.com/callmekohei/deoplete-fsharp/blob/master/pic/sample.gif) From 1783e4a871eabb943083c81f8fd5d2d843ab1e1f Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Tue, 30 Jan 2018 11:29:03 +0900 Subject: [PATCH 34/40] Fix the documentation --- doc/deoplete.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/deoplete.txt b/doc/deoplete.txt index 5d0b392f..6f992910 100644 --- a/doc/deoplete.txt +++ b/doc/deoplete.txt @@ -1289,9 +1289,9 @@ phpcd.vim: another PHP omnifunc. Faster. https://github.com/php-vim/phpcd.vim ============================================================================== -FREQUENTLY ASKED QUESTIONS (FAQ) *deoplete-faq* +FREQUENTLY ASKED QUESTIONS (FAQ) *deoplete-faq* - *deoplete-faq-trouble* + *deoplete-faq-trouble* 1. Troubleshooting~ Q: deoplete does not work. @@ -1378,7 +1378,7 @@ A: Please enable logging feature like this. > call deoplete#enable_logging('DEBUG', 'deoplete.log') call deoplete#custom#source('jedi', 'is_debug_enabled', 1) < - *deoplete-faq-config* + *deoplete-faq-config* 2. Configuration~ Q: I want to silence the |ins-completion-menu| messages in the command line @@ -1523,7 +1523,7 @@ Q: What does each part of a line on the pop up mean? For example I see: A: It is the source mark. [~] is from around source. A is from |deoplete-source-around|. - *deoplete-faq-ft-specific* + *deoplete-faq-ft-specific* 3. Filetype Specific Questions~ Q: I want to disable the auto completion for certain filetypes. @@ -1587,7 +1587,7 @@ Q: I want to complete AAA using deoplete. A: You can create the source for it. Why don't create the source if you need? - *deoplete-faq-general* + *deoplete-faq-general* 4. General Questions~ Q: How to donate money to you? From 3c02a2f0d8a5b491c9799fbb1da3b46c27d3fc25 Mon Sep 17 00:00:00 2001 From: Ryo Susami Date: Sun, 4 Feb 2018 13:16:57 +0900 Subject: [PATCH 35/40] wait to kill python processes. --- autoload/deoplete/util.vim | 5 +++++ rplugin/python3/deoplete/deoplete.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/autoload/deoplete/util.vim b/autoload/deoplete/util.vim index 5cb8fddf..17cc2cb0 100644 --- a/autoload/deoplete/util.vim +++ b/autoload/deoplete/util.vim @@ -218,6 +218,11 @@ function! deoplete#util#rpcnotify(event, context) abort return '' endif call s:notify(a:event, a:context) + if a:event == 'deoplete_on_event' && a:context['event'] == 'VimLeavePre' + while !exists('g:deoplete#_process_stopped') + sleep 50m + endwhile + endif return '' endfunction diff --git a/rplugin/python3/deoplete/deoplete.py b/rplugin/python3/deoplete/deoplete.py index c2a506f6..a56cb144 100644 --- a/rplugin/python3/deoplete/deoplete.py +++ b/rplugin/python3/deoplete/deoplete.py @@ -201,3 +201,6 @@ def on_event(self, context): for parent in self._parents: parent.on_event(context) + + if context['event'] == 'VimLeavePre': + self._vim.vars['deoplete#_process_stopped'] = True From bb05c6537e87299ec535d74cb885461c0fe06cf9 Mon Sep 17 00:00:00 2001 From: Ryo Susami Date: Sun, 4 Feb 2018 17:11:17 +0900 Subject: [PATCH 36/40] wait to kill python processes use only nvim. --- autoload/deoplete/util.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoload/deoplete/util.vim b/autoload/deoplete/util.vim index 17cc2cb0..c6cb9cbc 100644 --- a/autoload/deoplete/util.vim +++ b/autoload/deoplete/util.vim @@ -218,7 +218,7 @@ function! deoplete#util#rpcnotify(event, context) abort return '' endif call s:notify(a:event, a:context) - if a:event == 'deoplete_on_event' && a:context['event'] == 'VimLeavePre' + if has('nvim') && a:event == 'deoplete_on_event' && a:context['event'] == 'VimLeavePre' while !exists('g:deoplete#_process_stopped') sleep 50m endwhile From 27a8872473a772ccb52c4e91ec6cfb4a0267beca Mon Sep 17 00:00:00 2001 From: Ryo Susami Date: Sun, 4 Feb 2018 17:31:43 +0900 Subject: [PATCH 37/40] use ==# to pass the Travis CI. --- autoload/deoplete/util.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoload/deoplete/util.vim b/autoload/deoplete/util.vim index c6cb9cbc..02fe2a99 100644 --- a/autoload/deoplete/util.vim +++ b/autoload/deoplete/util.vim @@ -218,7 +218,7 @@ function! deoplete#util#rpcnotify(event, context) abort return '' endif call s:notify(a:event, a:context) - if has('nvim') && a:event == 'deoplete_on_event' && a:context['event'] == 'VimLeavePre' + if has('nvim') && a:event ==# 'deoplete_on_event' && a:context['event'] ==# 'VimLeavePre' while !exists('g:deoplete#_process_stopped') sleep 50m endwhile From 0a073d4fb8c7d5226c12be2802552cba06ef4a39 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 4 Feb 2018 17:47:25 +0900 Subject: [PATCH 38/40] Fix logging error --- rplugin/python3/deoplete/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rplugin/python3/deoplete/__init__.py b/rplugin/python3/deoplete/__init__.py index 0e5f4100..7f21f03b 100644 --- a/rplugin/python3/deoplete/__init__.py +++ b/rplugin/python3/deoplete/__init__.py @@ -28,7 +28,7 @@ def init_channel(self, args): self._deoplete = Deoplete(self._vim) @neovim.rpc_export('deoplete_enable_logging') - def enable_logging(self): + def enable_logging(self, context): self._deoplete.enable_logging() @neovim.rpc_export('deoplete_auto_completion_begin') @@ -51,7 +51,7 @@ def on_event(self, context): def deoplete_init(): pass - def deoplete_enable_logging(): + def deoplete_enable_logging(context): global_deoplete.enable_logging() def deoplete_auto_completion_begin(context): From 6cac2641aee2d8271252e114a08ca390b2086579 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 4 Feb 2018 17:47:55 +0900 Subject: [PATCH 39/40] Fix documentation --- doc/deoplete.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/deoplete.txt b/doc/deoplete.txt index 6f992910..50e9cda6 100644 --- a/doc/deoplete.txt +++ b/doc/deoplete.txt @@ -230,7 +230,7 @@ g:deoplete#auto_complete_delay g:deoplete#auto_refresh_delay Delay the refresh when asynchronous. - Default value: 500 + Default value: 50 *g:deoplete#skip_chars* g:deoplete#skip_chars From e37de74af2da442ffe38035110ba8f6ffe028d27 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 4 Feb 2018 17:50:38 +0900 Subject: [PATCH 40/40] Fix start_process() error --- rplugin/python3/deoplete/parent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rplugin/python3/deoplete/parent.py b/rplugin/python3/deoplete/parent.py index 52b8ae0f..1c5bf8a8 100644 --- a/rplugin/python3/deoplete/parent.py +++ b/rplugin/python3/deoplete/parent.py @@ -5,10 +5,11 @@ # ============================================================================ import time +import os from deoplete import logger from deoplete.process import Process -# from deoplete.util import error +from deoplete.util import error class Parent(logger.LoggingMixin): @@ -68,6 +69,11 @@ def _start_process(self, context, serveraddr): if self._proc: return + if not os.access(context['python3'], os.X_OK): + error(self._vim, str(context['python3']) + ' is not executble.') + error(self._vim, 'You need to set g:python3_host_prog.') + return + self._proc = Process( [context['python3'], context['dp_main'], serveraddr], context, context['cwd'])