Replace HTTP server in controller with RPC (#27)

Split RPC stuff into separate directory.
Controller now has a unified API trait matching worker.
Load generator now simplified using these objects.

---------

Co-authored-by: Alex Fuerst <alfuerst@iu.edu>

src/Ilúvatar/Cargo.toml

@@ -7,7 +7,8 @@ members = [
   "iluvatar_controller",
   "iluvatar_library",
   "iluvatar_worker_library",
-  "iluvatar_energy_mon"
+  "iluvatar_energy_mon",
+  "iluvatar_rpc"
 ]
 resolver = "2"
 

src/Ilúvatar/ansible/controller.yml

@@ -1,19 +1,20 @@
 - hosts: controller
   vars:
+    expected: "Connection failure: [Errno 104] Connection reset by peer"
     host_group: "{{ groups['controller'] }}"
     host_group_size: "{{ host_group | length }}"
     bin_dir: "/tmp/iluvatar/bin"
     exe_dest: "{{bin_dir}}/{{ controller.bin_name }}"
     config_dest: "{{bin_dir}}/{{ controller.config_name }}"
     __architecture: "{{ servers[ansible_host].architecture | default('x86_64-unknown-linux-gnu') }}"
     def_bin_src: "{{iluvatar_home}}/target/{{__architecture}}/{{target}}"
-    __bin_src: "{{ worker_bin_src | default(def_bin_src) }}"
+    __bin_src: "{{ controller_bin_src | default(def_bin_src) }}"
     __remote_bin_src: "{{ remote_bin_src | default(false) }}"
-    __worker_address: "{{ worker_address | default(servers[ansible_host].internal_ip) }}"
+    __controller_address: "{{ controller_address | default(servers[ansible_host].internal_ip) }}"
     coded_proxy_env:
       "ILUVATAR_CONTROLLER__name": "{{ controller.host }}"
       "ILUVATAR_CONTROLLER__port": "{{ controller.port }}"
-      "ILUVATAR_CONTROLLER__address": "{{ __worker_address | default('localhost') }}"
+      "ILUVATAR_CONTROLLER__address": "{{ __controller_address | default('localhost') }}"
       "ILUVATAR_CONTROLLER__logging__directory" : "{{ controller_log_dir | default('/tmp/iluvatar/logs') }}"
       "ILUVATAR_CONTROLLER__logging__level" : "{{ controller_log_level | default('info') }}"
       "ILUVATAR_CONTROLLER__load_balancer__algorithm" : "{{ controller.algorithm }}"
@@ -89,10 +90,14 @@
   - name: wait until the controller on this host is up and running
     uri:
       url:
-        "http://{{__worker_address}}:{{controller.port}}/ping"
+        "http://{{__controller_address}}:{{controller.port}}/ping"
       validate_certs: "no"
+      # RPC server doesn't accept basic HTTP connections
+      # but we can try connecting to it once it's up and know it's alive
+      status_code: [-1] #[-1, 0.9, 1.1]
     register: result
-    until: result.status == 200
+    until: result.msg == expected
     retries: 10
     delay: 5
     when: mode == "deploy"
+

src/Ilúvatar/docs/TODOs/TODO.md

@@ -119,6 +119,8 @@ Logs for workers in a cluster simulation scenario are put into one file.
 These become impossible to separate without any identifying factor.
 Either split up the logs into separate files on worker name, or include the name in each log message so it can be split in post-processing.
 
+Probably use a filter in `tracing-subscriber`, see [here](https://stackoverflow.com/questions/76939805/tracing-how-to-filter-logs-under-specified-levels-for-layer) and [here](https://docs.rs/tracing-subscriber/0.3.16/tracing_subscriber/layer/index.html#filtering-with-layers).
+
 ## CI Testing
 
 Run automated tests in Github actions CI.

src/Ilúvatar/docs/examples/sample_setup.sh

@@ -58,5 +58,5 @@ EOT
 ILU_HOME="../.."
 ret=$(pwd)
 cd $ILU_HOME
-make
+make release
 cd $ret

src/Ilúvatar/iluvatar_controller/Cargo.toml

@@ -11,10 +11,11 @@ repository.workspace = true
 categories.workspace = true
 
 [dependencies]
-actix-web = "4"
 iluvatar_library = { path = "../iluvatar_library" }
+iluvatar_rpc = { path = "../iluvatar_rpc" }
 iluvatar_controller_library = { path = "../iluvatar_controller_library" }
 clap = { version = "4.1", features = ["derive"] }
 tracing = "0.1"
 tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "test-util", "parking_lot"] }
 anyhow = "1.0.13"
+tonic = "0.11"

src/Ilúvatar/iluvatar_controller/src/main.rs

@@ -1,10 +1,14 @@
-use actix_web::{web::Data, App, HttpServer};
+use std::time::Duration;
+
+// use actix_web::{web::Data, App, HttpServer};
 use clap::{command, Parser};
-use iluvatar_controller_library::server::{config::Configuration, controller::Controller, web_server::*};
+use iluvatar_controller_library::server::{config::Configuration, controller::Controller};
 use iluvatar_library::logging::start_tracing;
 use iluvatar_library::transaction::{TransactionId, LOAD_BALANCER_TID};
 use iluvatar_library::utils::wait_for_exit_signal;
-use tracing::info;
+use iluvatar_rpc::rpc::iluvatar_controller_server::IluvatarControllerServer;
+use tonic::transport::Server;
+use tracing::{debug, info};
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -23,25 +27,16 @@ async fn main() -> anyhow::Result<()> {
     let config = Configuration::boxed(&args.config).unwrap();
     let _guard = start_tracing(config.logging.clone(), &config.name, tid).unwrap();
 
-    let server = Controller::new(config.clone(), tid).await?;
-    let server_data = Data::new(server);
+    let controller = Controller::new(config.clone(), tid).await?;
 
     info!(tid=%tid, "Controller started!");
-
-    let _handle = tokio::spawn(
-        HttpServer::new(move || {
-            App::new()
-                .app_data(server_data.clone())
-                .service(ping)
-                .service(invoke_api)
-                .service(invoke_async_api)
-                .service(invoke_async_check_api)
-                .service(prewarm_api)
-                .service(register_function_api)
-                .service(register_worker_api)
-        })
-        .bind((config.address.clone(), config.port))?
-        .run(),
+    debug!(config=?config, "Controller configuration");
+    let addr = std::net::SocketAddr::new(config.address.clone().parse()?, config.port);
+    let _j = tokio::spawn(
+        Server::builder()
+            .timeout(Duration::from_secs(config.timeout_sec))
+            .add_service(IluvatarControllerServer::new(controller))
+            .serve(addr),
     );
 
     wait_for_exit_signal(tid).await?;

src/Ilúvatar/iluvatar_controller_library/Cargo.toml

@@ -13,7 +13,7 @@ categories.workspace = true
 [dependencies]
 iluvatar_library = { path = "../iluvatar_library" }
 iluvatar_worker_library = { path = "../iluvatar_worker_library" }
-tonic = "0.7.2"
+iluvatar_rpc = { path = "../iluvatar_rpc" }
 tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "test-util", "parking_lot"] }
 lazy_static = "1.4.0"
 anyhow = "1.0.13"
@@ -25,7 +25,7 @@ clap = "4.1"
 reqwest = { version = "0.12.4", default-features = false, features = ["json", "rustls-tls"] }
 dashmap = "5.3.4"
 tracing = "0.1"
-actix-web = "4"
+tonic = "0.11"
 
 [dev-dependencies]
 rstest = "0.13.0"