From 002e55a4092fa926b7fe008e630b3870bef57285 Mon Sep 17 00:00:00 2001 From: Francesco Cogno Date: Mon, 12 Dec 2022 11:29:08 +0000 Subject: [PATCH 1/3] docker support --- .dockerignore | 2 ++ Cargo.toml | 2 +- Dockerfile | 16 ++++++++++------ src/options.rs | 8 ++++---- 4 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8bdebe3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +extra/ +target/ diff --git a/Cargo.toml b/Cargo.toml index 8cf8f45..49b2bfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" [dependencies] anyhow = "1.0.66" chrono = "0.4.22" -clap = { version = "4.0.22", features = ["derive"] } +clap = { version = "4.0.22", features = ["derive", "env"] } csv = "1.1.6" goauth = "0.13.1" google-cloud-googleapis = { version = "0.6.0", features = ["pubsub"] } diff --git a/Dockerfile b/Dockerfile index ec0eb5a..4c4e9fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,11 @@ -FROM rustlang/rust:nightly as build +FROM ubuntu:latest as build + +RUN apt-get update && apt-get install build-essential pkg-config libssl-dev curl -y +# install rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y # create a new empty shell project -RUN USER=root cargo new --bin pgdelaytest +RUN $HOME/.cargo/bin/cargo new --bin pgdelaytest WORKDIR /pgdelaytest # copy over your manifests @@ -12,17 +16,17 @@ COPY ./Cargo.toml ./Cargo.toml COPY ./src ./src # build for release -RUN cargo build --release +RUN $HOME/.cargo/bin/cargo build --release +RUN $HOME/.cargo/bin/cargo build # our final base FROM ubuntu:latest -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt-get install ca-certificates libssl-dev tini tzdata -y +RUN apt-get update && apt-get dist-upgrade -y && apt-get install ca-certificates libssl-dev tini tzdata -y # copy the build artifact from the build stage COPY --from=build /pgdelaytest/target/release/pgdelaytest /usr/local/bin/. +COPY --from=build /pgdelaytest/target/release/pgdelaytest /usr/local/bin/pgdelaytest-debug ENV RUST_LOG=info ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/pgdelaytest"] diff --git a/src/options.rs b/src/options.rs index dc35c25..14adc58 100644 --- a/src/options.rs +++ b/src/options.rs @@ -5,7 +5,7 @@ pub enum Commands { /// publish events to Google Cloud Pub/Sub Publish { /// Pub/sub topic to publish to - #[arg(long)] + #[arg(long, env = "PUB_SUB_TOPIC")] pub_sub_topic: String, }, } @@ -15,15 +15,15 @@ pub enum Commands { #[command(author, version, about, long_about = None)] pub struct Options { /// Primary instance (RW) - #[arg(short, long)] + #[arg(short, long, env = "PRIMARY_CONNECTION_STRING")] pub primary_connection_string: String, /// Secondary instance (RO) - #[arg(short, long)] + #[arg(short, long, env = "SECONDARY_CONNECTION_STRING")] pub secondary_connection_string: String, /// Sleep time in ms after each evaluation - #[arg(long, default_value_t = 1000)] + #[arg(long, default_value_t = 1000, env = "SLEEP_MS")] pub sleep_ms: u64, #[command(subcommand)] From 36255630656af2c5cf88eda1cebfd50022d2cdfd Mon Sep 17 00:00:00 2001 From: Francesco Cogno Date: Mon, 12 Dec 2022 13:40:09 +0000 Subject: [PATCH 2/3] Only parames via env variables --- src/main.rs | 16 +++++++++++----- src/options.rs | 17 ++++------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main.rs b/src/main.rs index 01be666..07c2ef0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Error}; use clap::Parser; use event::Event; use google_cloud_pubsub::client::Client as PubSubClient; -use options::{Commands, Options}; +use options::Options; use tokio_postgres::*; const ID: i32 = 1; @@ -15,13 +15,19 @@ async fn main() -> Result<(), Error> { let options = Options::parse(); // println!("Using options {:?}", options); - let topic = if let Some(Commands::Publish { pub_sub_topic }) = options.command { - let pubsub = PubSubClient::default().await?; - let topic = pubsub.topic(&pub_sub_topic).new_publisher(None); - Some(topic) + let topic = if let Some(pub_sub_topic) = options.pub_sub_topic { + if pub_sub_topic.is_empty() { + None + } else { + println!("Events will be published to {} topic", pub_sub_topic); + let pubsub = PubSubClient::default().await?; + let topic = pubsub.topic(&pub_sub_topic).new_publisher(None); + Some(topic) + } } else { None }; + println!("Openinig connection to primary..."); let (pri_client, pri_connection) = tokio_postgres::connect(&options.primary_connection_string, NoTls).await?; diff --git a/src/options.rs b/src/options.rs index 14adc58..36ef590 100644 --- a/src/options.rs +++ b/src/options.rs @@ -1,14 +1,4 @@ -use clap::{Parser, Subcommand}; - -#[derive(Subcommand, Debug)] -pub enum Commands { - /// publish events to Google Cloud Pub/Sub - Publish { - /// Pub/sub topic to publish to - #[arg(long, env = "PUB_SUB_TOPIC")] - pub_sub_topic: String, - }, -} +use clap::Parser; /// Simple program to greet a person #[derive(Parser, Debug)] @@ -26,6 +16,7 @@ pub struct Options { #[arg(long, default_value_t = 1000, env = "SLEEP_MS")] pub sleep_ms: u64, - #[command(subcommand)] - pub command: Option, + /// Pub/sub topic to publish to + #[arg(long, env = "PUB_SUB_TOPIC")] + pub pub_sub_topic: Option, } From c78dab126024f925ee3e52b0c3e7c0a8b396a571 Mon Sep 17 00:00:00 2001 From: Francesco Cogno Date: Mon, 12 Dec 2022 13:54:00 +0000 Subject: [PATCH 3/3] updated readme --- Cargo.toml | 2 -- README.md | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 49b2bfb..ad36fa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,8 +4,6 @@ authors = ["Francesco Cogno "] version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] anyhow = "1.0.66" chrono = "0.4.22" diff --git a/README.md b/README.md index 76f5178..cb7d03d 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,7 @@ CREATE TABLE tbl(id INT, value INT); INSERT INTO tbl(id, value) VALUES(1, 900); ``` -Required parameters for the tool are the two connection strings (in the format specificed here: [https://docs.rs/tokio-postgres/latest/tokio_postgres/config/struct.Config.html](https://docs.rs/tokio-postgres/latest/tokio_postgres/config/struct.Config.html)). You can also optionally change the sleep time between tries and the pub/sub topic to write to. If you specify the pub/sub topic you need to specify a service account key file path that has enough privileges to write to the topic. - +Required parameters for the tool are the two connection strings (in the format specificed here: [https://docs.rs/tokio-postgres/latest/tokio_postgres/config/struct.Config.html](https://docs.rs/tokio-postgres/latest/tokio_postgres/config/struct.Config.html)). You can also optionally change the sleep time between tries and the pub/sub topic to write to. For example this command: @@ -27,6 +26,16 @@ Tests the latency between `primary` and `secondary`, publishing the results both Note that, in order to publish to pub/sub, a valid GCP identity must be available and proper permissions must be granted. +## Usage (Docker) + +You can either build the container with `docker build . -t pgdelaytest:latest` or pull it from Docker.io. Then execute it passing env variables. For example: + +```bash +docker run -e PRIMARY_CONNECTION_STRING="host=host user=test password=password" -e SECONDARY_CONNECTION_STRING="host=secondary user=test password=password" -e PUB_SUB_TOPIC=topic -e GOOGLE_APPLICATION_CREDENTIALS=/service_account_pvk.json -v /service_account_pvk.json:/service_account_pvk.json pgdelaytest:latest +``` + +*Note*: this example uses a service account key file, it is not necessary if you don't want to publish to pub/sub or you have default credentials at hand. + ## Methodology The tool updates a row on the primary and right away tries to get the same row from the secondary. If the value matches, the reported latency is zero. If not, the tool keeps querying the same row until the value matches and then reports the time taken as *replication latency*.