diff --git a/src/cl/op-node/op_node_builder_launcher.star b/src/cl/op-node/op_node_builder_launcher.star index 38db8d9a..18d258c3 100644 --- a/src/cl/op-node/op_node_builder_launcher.star +++ b/src/cl/op-node/op_node_builder_launcher.star @@ -17,6 +17,8 @@ ethereum_package_input_parser = import_module( constants = import_module("../../package_io/constants.star") util = import_module("../../util.star") +observability = import_module("../../observability/observability.star") +interop_constants = import_module("../../interop/constants.star") # ---------------------------------- Beacon client ------------------------------------- @@ -73,6 +75,9 @@ def launch( existing_cl_clients, l1_config_env_vars, sequencer_enabled, + observability_helper, + interop_params, + da_server_context, ): beacon_node_identity_recipe = PostHttpRequestRecipe( endpoint="/", @@ -104,6 +109,9 @@ def launch( l1_config_env_vars, beacon_node_identity_recipe, sequencer_enabled, + observability_helper, + interop_params, + da_server_context, ) beacon_service = plan.add_service(service_name, config) @@ -113,6 +121,8 @@ def launch( beacon_service.ip_address, beacon_http_port.number ) + metrics_info = observability.new_metrics_info(observability_helper, beacon_service) + response = plan.request( recipe=beacon_node_identity_recipe, service_name=service_name ) @@ -127,7 +137,7 @@ def launch( ip_addr=beacon_service.ip_address, http_port=beacon_http_port.number, beacon_http_url=beacon_http_url, - cl_nodes_metrics_info=None, + cl_nodes_metrics_info=[metrics_info], beacon_service_name=service_name, multiaddr=beacon_multiaddr, peer_id=beacon_peer_id, @@ -148,29 +158,33 @@ def get_beacon_config( l1_config_env_vars, beacon_node_identity_recipe, sequencer_enabled, + observability_helper, + interop_params, + da_server_context, ): + ports = dict(get_used_ports(BEACON_DISCOVERY_PORT_NUM)) + EXECUTION_ENGINE_ENDPOINT = "http://{0}:{1}".format( el_context.ip_addr, el_context.engine_rpc_port_num, ) - used_ports = get_used_ports(BEACON_DISCOVERY_PORT_NUM) - cmd = [ "op-node", "--l2={0}".format(EXECUTION_ENGINE_ENDPOINT), "--l2.jwt-secret=" + ethereum_package_constants.JWT_MOUNT_PATH_ON_CONTAINER, - "--verifier.l1-confs=4", + "--verifier.l1-confs=1", "--rollup.config=" - + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS - + "/rollup-{0}.json".format(launcher.network_params.network_id), + + "{0}/rollup-{1}.json".format( + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, + launcher.network_params.network_id, + ), "--rpc.addr=0.0.0.0", "--rpc.port={0}".format(BEACON_HTTP_PORT_NUM), "--rpc.enable-admin", "--l1={0}".format(l1_config_env_vars["L1_RPC_URL"]), "--l1.rpckind={0}".format(l1_config_env_vars["L1_RPC_KIND"]), "--l1.beacon={0}".format(l1_config_env_vars["CL_RPC_URL"]), - "--l1.trustrpc", "--p2p.advertise.ip=" + ethereum_package_constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, "--p2p.advertise.tcp={0}".format(BEACON_DISCOVERY_PORT_NUM), @@ -178,19 +192,73 @@ def get_beacon_config( "--p2p.listen.ip=0.0.0.0", "--p2p.listen.tcp={0}".format(BEACON_DISCOVERY_PORT_NUM), "--p2p.listen.udp={0}".format(BEACON_DISCOVERY_PORT_NUM), + "--safedb.path={0}".format(BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER), + "--altda.enabled=" + str(da_server_context.enabled), + "--altda.da-server=" + da_server_context.http_url, ] - sequencer_private_key = util.read_network_config_value( - plan, - launcher.deployment_output, - "sequencer-{0}".format(launcher.network_params.network_id), - ".privateKey", - ) + # configure files + + files = { + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.deployment_output, + ethereum_package_constants.JWT_MOUNTPOINT_ON_CLIENTS: launcher.jwt_file, + } + + if persistent: + files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + size=int(participant.cl_builder_volume_size) + if int(participant.cl_builder_volume_size) > 0 + else constants.VOLUME_SIZE[launcher.network][ + constants.CL_TYPE.hildr + "_volume_size" + ], + ) + + # configure environment variables + + env_vars = dict(participant.cl_builder_extra_env_vars) + + # apply customizations + + if observability_helper.enabled: + cmd += [ + "--metrics.enabled=true", + "--metrics.addr=0.0.0.0", + "--metrics.port={0}".format(observability.METRICS_PORT_NUM), + ] + + observability.expose_metrics_port(ports) + + if interop_params.enabled: + ports[ + interop_constants.INTEROP_WS_PORT_ID + ] = ethereum_package_shared_utils.new_port_spec( + interop_constants.INTEROP_WS_PORT_NUM, + ethereum_package_shared_utils.TCP_PROTOCOL, + ) + + env_vars.update( + { + # "OP_NODE_INTEROP_SUPERVISOR": interop_constants.SUPERVISOR_ENDPOINT, + "OP_NODE_INTEROP_RPC_ADDR": "0.0.0.0", + "OP_NODE_INTEROP_RPC_PORT": str(interop_constants.INTEROP_WS_PORT_NUM), + "OP_NODE_INTEROP_JWT_SECRET": ethereum_package_constants.JWT_MOUNT_PATH_ON_CONTAINER, + } + ) if sequencer_enabled: - cmd.append("--p2p.sequencer.key=" + sequencer_private_key) - cmd.append("--sequencer.enabled") - cmd.append("--sequencer.l1-confs=5") + sequencer_private_key = util.read_network_config_value( + plan, + launcher.deployment_output, + "sequencer-{0}".format(launcher.network_params.network_id), + ".privateKey", + ) + + cmd += [ + "--p2p.sequencer.key=" + sequencer_private_key, + "--sequencer.enabled", + "--sequencer.l1-confs=2", + ] if len(existing_cl_clients) > 0: cmd.append( @@ -207,25 +275,6 @@ def get_beacon_config( cmd += participant.cl_builder_extra_params - files = { - ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.deployment_output, - ethereum_package_constants.JWT_MOUNTPOINT_ON_CLIENTS: launcher.jwt_file, - } - - if persistent: - files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( - persistent_key="data-{0}".format(service_name), - size=int(participant.cl_builder_volume_size) - if int(participant.cl_builder_volume_size) > 0 - else constants.VOLUME_SIZE[launcher.network][ - constants.CL_TYPE.hildr + "_volume_size" - ], - ) - - ports = {} - ports.update(used_ports) - - env_vars = participant.cl_builder_extra_env_vars config_args = { "image": participant.cl_builder_image, "ports": ports, @@ -251,6 +300,8 @@ def get_beacon_config( "node_selectors": node_selectors, } + # configure resources + if participant.cl_builder_min_cpu > 0: config_args["min_cpu"] = participant.cl_builder_min_cpu if participant.cl_builder_max_cpu > 0: @@ -259,6 +310,7 @@ def get_beacon_config( config_args["min_memory"] = participant.cl_builder_min_mem if participant.cl_builder_max_mem > 0: config_args["max_memory"] = participant.cl_builder_max_mem + return ServiceConfig(**config_args) diff --git a/src/el/op-geth/op_geth_builder_launcher.star b/src/el/op-geth/op_geth_builder_launcher.star index 4f17e828..ef2b7866 100644 --- a/src/el/op-geth/op_geth_builder_launcher.star +++ b/src/el/op-geth/op_geth_builder_launcher.star @@ -22,14 +22,14 @@ ethereum_package_constants = import_module( ) constants = import_module("../../package_io/constants.star") - +observability = import_module("../../observability/observability.star") +interop_constants = import_module("../../interop/constants.star") util = import_module("../../util.star") RPC_PORT_NUM = 8545 WS_PORT_NUM = 8546 DISCOVERY_PORT_NUM = 30303 ENGINE_RPC_PORT_NUM = 8551 -METRICS_PORT_NUM = 9001 # The min/max CPU/memory that the execution node can use EXECUTION_MIN_CPU = 300 @@ -42,13 +42,11 @@ TCP_DISCOVERY_PORT_ID = "tcp-discovery" UDP_DISCOVERY_PORT_ID = "udp-discovery" ENGINE_RPC_PORT_ID = "engine-rpc" ENGINE_WS_PORT_ID = "engineWs" -METRICS_PORT_ID = "metrics" + # TODO(old) Scale this dynamically based on CPUs available and Geth nodes mining NUM_MINING_THREADS = 1 -METRICS_PATH = "/debug/metrics/prometheus" - # The dirpath of the execution data directory on the client container EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/geth/execution-data" @@ -73,9 +71,6 @@ def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): ENGINE_RPC_PORT_NUM, ethereum_package_shared_utils.TCP_PROTOCOL, ), - METRICS_PORT_ID: ethereum_package_shared_utils.new_port_spec( - METRICS_PORT_NUM, ethereum_package_shared_utils.TCP_PROTOCOL - ), } return used_ports @@ -128,6 +123,8 @@ def launch( cl_client_name, sequencer_enabled, sequencer_context, + observability_helper, + interop_params, ) service = plan.add_service(service_name, config) @@ -136,13 +133,10 @@ def launch( plan, service_name, RPC_PORT_ID ) - metrics_url = "{0}:{1}".format(service.ip_address, METRICS_PORT_NUM) - geth_metrics_info = ethereum_package_node_metrics.new_node_metrics_info( - service_name, METRICS_PATH, metrics_url - ) - http_url = "http://{0}:{1}".format(service.ip_address, RPC_PORT_NUM) + metrics_info = observability.new_metrics_info(observability_helper, service) + return ethereum_package_el_context.new_el_context( client_name="op-geth", enode=enode, @@ -153,7 +147,7 @@ def launch( rpc_http_url=http_url, enr=enr, service_name=service_name, - el_metrics_info=[geth_metrics_info], + el_metrics_info=[metrics_info], ) @@ -170,15 +164,13 @@ def get_config( cl_client_name, sequencer_enabled, sequencer_context, + observability_helper, + interop_params, ): - init_datadir_cmd_str = "geth init --datadir={0} --state.scheme=hash {1}".format( - EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, - ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS - + "/genesis-{0}.json".format(launcher.network_id), - ) - discovery_port = DISCOVERY_PORT_NUM - used_ports = get_used_ports(discovery_port) + ports = dict(get_used_ports(discovery_port)) + + subcommand_strs = [] cmd = [ "geth", @@ -205,13 +197,56 @@ def get_config( "--syncmode=full", "--nat=extip:" + ethereum_package_constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, "--rpc.allow-unprotected-txs", - "--metrics", - "--metrics.addr=0.0.0.0", - "--metrics.port={0}".format(METRICS_PORT_NUM), "--discovery.port={0}".format(discovery_port), "--port={0}".format(discovery_port), ] + # configure files + + files = { + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.deployment_output, + ethereum_package_constants.JWT_MOUNTPOINT_ON_CLIENTS: launcher.jwt_file, + } + + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + size=int(participant.el_builder_volume_size) + if int(participant.el_builder_volume_size) > 0 + else constants.VOLUME_SIZE[launcher.network][ + constants.EL_TYPE.op_geth + "_volume_size" + ], + ) + + if launcher.network not in ethereum_package_constants.PUBLIC_NETWORKS: + init_datadir_cmd_str = "geth init --datadir={0} --state.scheme=hash {1}".format( + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, + "{0}/genesis-{1}.json".format( + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, + launcher.network_id, + ), + ) + + subcommand_strs.append(init_datadir_cmd_str) + + # configure environment variables + + env_vars = dict(participant.el_builder_extra_env_vars) + + # apply customizations + + if observability_helper.enabled: + cmd += [ + "--metrics", + "--metrics.addr=0.0.0.0", + "--metrics.port={0}".format(observability.METRICS_PORT_NUM), + ] + + observability.expose_metrics_port(ports) + + if interop_params.enabled: + env_vars["GETH_ROLLUP_INTEROPRPC"] = interop_constants.SUPERVISOR_ENDPOINT + if not sequencer_enabled: cmd.append("--rollup.sequencerhttp={0}".format(sequencer_context.rpc_http_url)) @@ -228,34 +263,18 @@ def get_config( ) ) + # construct command string + cmd += participant.el_builder_extra_params - cmd_str = " ".join(cmd) - if launcher.network not in ethereum_package_constants.PUBLIC_NETWORKS: - subcommand_strs = [ - init_datadir_cmd_str, - cmd_str, - ] - command_str = " && ".join(subcommand_strs) - else: - command_str = cmd_str + subcommand_strs.append(" ".join(cmd)) + command_str = " && ".join(subcommand_strs) + + plan.print(">>>", participant.el_builder_image) + plan.print(">>>", util.label_from_image(participant.el_builder_image)) - files = { - ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.deployment_output, - ethereum_package_constants.JWT_MOUNTPOINT_ON_CLIENTS: launcher.jwt_file, - } - if persistent: - files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( - persistent_key="data-{0}".format(service_name), - size=int(participant.el_builder_volume_size) - if int(participant.el_builder_volume_size) > 0 - else constants.VOLUME_SIZE[launcher.network][ - constants.EL_TYPE.op_geth + "_volume_size" - ], - ) - env_vars = participant.el_builder_extra_env_vars config_args = { "image": participant.el_builder_image, - "ports": used_ports, + "ports": ports, "cmd": [command_str], "files": files, "entrypoint": ENTRYPOINT_ARGS, @@ -272,6 +291,8 @@ def get_config( "node_selectors": node_selectors, } + # configure resources + if participant.el_builder_min_cpu > 0: config_args["min_cpu"] = participant.el_builder_min_cpu if participant.el_builder_max_cpu > 0: @@ -280,6 +301,7 @@ def get_config( config_args["min_memory"] = participant.el_builder_min_mem if participant.el_builder_max_mem > 0: config_args["max_memory"] = participant.el_builder_max_mem + return ServiceConfig(**config_args) diff --git a/src/el/op-reth/op_reth_builder_launcher.star b/src/el/op-reth/op_reth_builder_launcher.star index 1a56816b..733866ba 100644 --- a/src/el/op-reth/op_reth_builder_launcher.star +++ b/src/el/op-reth/op_reth_builder_launcher.star @@ -21,14 +21,13 @@ ethereum_package_input_parser = import_module( ) constants = import_module("../../package_io/constants.star") - +observability = import_module("../../observability/observability.star") util = import_module("../../util.star") RPC_PORT_NUM = 8545 WS_PORT_NUM = 8546 DISCOVERY_PORT_NUM = 30303 ENGINE_RPC_PORT_NUM = 9551 -METRICS_PORT_NUM = 9001 # The min/max CPU/memory that the execution node can use EXECUTION_MIN_CPU = 100 @@ -40,7 +39,6 @@ WS_PORT_ID = "ws" TCP_DISCOVERY_PORT_ID = "tcp-discovery" UDP_DISCOVERY_PORT_ID = "udp-discovery" ENGINE_RPC_PORT_ID = "engine-rpc" -METRICS_PORT_ID = "metrics" # Paths METRICS_PATH = "/metrics" @@ -68,9 +66,6 @@ def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): ENGINE_RPC_PORT_ID: ethereum_package_shared_utils.new_port_spec( ENGINE_RPC_PORT_NUM, ethereum_package_shared_utils.TCP_PROTOCOL ), - METRICS_PORT_ID: ethereum_package_shared_utils.new_port_spec( - METRICS_PORT_NUM, ethereum_package_shared_utils.TCP_PROTOCOL - ), } return used_ports @@ -96,6 +91,8 @@ def launch( existing_el_clients, sequencer_enabled, sequencer_context, + observability_helper, + interop_params, ): log_level = ethereum_package_input_parser.get_client_log_level_or_default( participant.el_builder_log_level, global_log_level, VERBOSITY_LEVELS @@ -116,6 +113,7 @@ def launch( cl_client_name, sequencer_enabled, sequencer_context, + observability_helper, ) service = plan.add_service(service_name, config) @@ -124,13 +122,12 @@ def launch( plan, service_name, RPC_PORT_ID ) - metric_url = "{0}:{1}".format(service.ip_address, METRICS_PORT_NUM) - op_reth_metrics_info = ethereum_package_node_metrics.new_node_metrics_info( - service_name, METRICS_PATH, metric_url - ) - http_url = "http://{0}:{1}".format(service.ip_address, RPC_PORT_NUM) + metrics_info = observability.new_metrics_info( + observability_helper, service, METRICS_PATH + ) + return ethereum_package_el_context.new_el_context( client_name="reth", enode=enode, @@ -140,7 +137,7 @@ def launch( engine_rpc_port_num=ENGINE_RPC_PORT_NUM, rpc_http_url=http_url, service_name=service_name, - el_metrics_info=[op_reth_metrics_info], + el_metrics_info=[metrics_info], ) @@ -157,10 +154,10 @@ def get_config( cl_client_name, sequencer_enabled, sequencer_context, + observability_helper, ): - public_ports = {} discovery_port = DISCOVERY_PORT_NUM - used_ports = get_used_ports(discovery_port) + ports = dict(get_used_ports(discovery_port)) cmd = [ "node", @@ -187,12 +184,38 @@ def get_config( "--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM), "--authrpc.jwtsecret=" + ethereum_package_constants.JWT_MOUNT_PATH_ON_CONTAINER, "--authrpc.addr=0.0.0.0", - "--metrics=0.0.0.0:{0}".format(METRICS_PORT_NUM), "--discovery.port={0}".format(discovery_port), "--port={0}".format(discovery_port), "--rpc.eth-proof-window=302400", ] + # configure files + + files = { + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.deployment_output, + ethereum_package_constants.JWT_MOUNTPOINT_ON_CLIENTS: launcher.jwt_file, + } + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + size=int(participant.el_builder_volume_size) + if int(participant.el_builder_volume_size) > 0 + else constants.VOLUME_SIZE[launcher.network][ + constants.EL_TYPE.op_reth + "_volume_size" + ], + ) + + # configure environment variables + + env_vars = participant.el_builder_extra_env_vars + + # apply customizations + + if observability_helper.enabled: + cmd.append("--metrics=0.0.0.0:{0}".format(observability.METRICS_PORT_NUM)) + + observability.expose_metrics_port(ports) + if not sequencer_enabled: cmd.append("--rollup.sequencer-http={0}".format(sequencer_context.rpc_http_url)) @@ -209,25 +232,11 @@ def get_config( ) ) - files = { - ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.deployment_output, - ethereum_package_constants.JWT_MOUNTPOINT_ON_CLIENTS: launcher.jwt_file, - } - if persistent: - files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( - persistent_key="data-{0}".format(service_name), - size=int(participant.el_builder_volume_size) - if int(participant.el_builder_volume_size) > 0 - else constants.VOLUME_SIZE[launcher.network][ - constants.EL_TYPE.op_reth + "_volume_size" - ], - ) - cmd += participant.el_builder_extra_params - env_vars = participant.el_builder_extra_env_vars + config_args = { "image": participant.el_builder_image, - "ports": used_ports, + "ports": ports, "cmd": cmd, "files": files, "private_ip_address_placeholder": ethereum_package_constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, @@ -243,7 +252,9 @@ def get_config( "node_selectors": node_selectors, } - if participant.el_min_cpu > 0: + # configure resources + + if participant.el_builder_min_cpu > 0: config_args["min_cpu"] = participant.el_builder_min_cpu if participant.el_builder_max_cpu > 0: config_args["max_cpu"] = participant.el_builder_max_cpu diff --git a/src/el_cl_launcher.star b/src/el_cl_launcher.star index 86ddcf4d..6012751f 100644 --- a/src/el_cl_launcher.star +++ b/src/el_cl_launcher.star @@ -155,6 +155,7 @@ def launch( all_el_contexts = [] sequencer_enabled = True rollup_boost_enabled = "rollup-boost" in additional_services + external_builder = mev_params.builder_host != "" and mev_params.builder_port != "" for index, participant in enumerate(participants): cl_type = participant.cl_type @@ -273,7 +274,17 @@ def launch( if rollup_boost_enabled and sequencer_enabled: plan.print("Starting rollup boost") - if mev_params.builder_host == "" or mev_params.builder_port == "": + if external_builder: + el_builder_context = struct( + ip_addr=mev_params.builder_host, + engine_rpc_port_num=mev_params.builder_port, + rpc_port_num=mev_params.builder_port, + rpc_http_url="http://{0}:{1}".format( + mev_params.builder_host, mev_params.builder_port + ), + client_name="external-builder", + ) + else: el_builder_context = el_builder_launch_method( plan, el_builder_launcher, @@ -289,17 +300,15 @@ def launch( observability_helper, interop_params, ) - else: - el_builder_context = struct( - ip_addr=mev_params.builder_host, - engine_rpc_port_num=mev_params.builder_port, - rpc_port_num=mev_params.builder_port, - rpc_http_url="http://{0}:{1}".format( - mev_params.builder_host, mev_params.builder_port - ), - client_name="external-builder", - ) - + for metrics_info in [ + x for x in el_builder_context.el_metrics_info if x != None + ]: + observability.register_node_metrics_job( + observability_helper, + el_builder_context.client_name, + "execution-builder", + metrics_info, + ) rollup_boost_image = ( mev_params.rollup_boost_image if mev_params.rollup_boost_image != "" @@ -351,12 +360,8 @@ def launch( }, ) - sequencer_enabled = False - - all_el_contexts.append(el_context) - all_cl_contexts.append(cl_context) - - if rollup_boost_enabled and sequencer_enabled: + # We don't deploy CL for external builder + if rollup_boost_enabled and sequencer_enabled and not external_builder: cl_builder_context = cl_builder_launch_method( plan, cl_builder_launcher, @@ -372,8 +377,26 @@ def launch( False, observability_helper, interop_params, + da_server_context, ) + for metrics_info in [ + x for x in cl_builder_context.cl_nodes_metrics_info if x != None + ]: + observability.register_node_metrics_job( + observability_helper, + cl_builder_context.client_name, + "beacon-builder", + metrics_info, + { + "supernode": str(cl_builder_context.supernode), + }, + ) all_cl_contexts.append(cl_builder_context) + # We need to make sure that el_context and cl_context are first in the list, as down the line all_el_contexts[0] + # and all_cl_contexts[0] are used + all_el_contexts.insert(0, el_context) + all_cl_contexts.insert(0, cl_context) + plan.print("Successfully added {0} EL/CL participants".format(num_participants)) return all_el_contexts, all_cl_contexts