diff --git a/.dockerignore b/.dockerignore index 21c5356..e3c57a2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,5 @@ .tools-versions Dockerfile docker-compose.* -docker-helper.sh \ No newline at end of file +docker-helper.sh +docs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7de7872..4dc5b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +- [Changelog](#changelog) + - [Structure](#structure) + - [Unreleased](#unreleased) + - [v1.1.0](#v110) + - [General](#general) + - [Known issues](#known-issues) + - [Added](#added) + - [Changed](#changed) + - [Environment Variables](#environment-variables) + - [Manager/Pool-related](#managerpool-related) + - [Server-related](#server-related) + - [v1.0.0](#v100) + + +## Structure + - All notable changes to this project will be documented in this file. - The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - We try to follow [semantic versioning](https://semver.org/). @@ -8,9 +24,107 @@ ## Unreleased ## v1.1.0 -- First of all, even though this is a **major refactor**, the clientside API is still the same. **There should be no breaking changes!** -- The main goal of this release is to bring IntegreSQL's performance on par with our previous Node.js implementation. Specifially we wanted to eliminate any long-running mutex locks and make the codebase more maintainable and easier to extend in the future. -- ... + +> Special thanks to [Anna - @anjankow](https://github.com/anjankow) for her contributions to this release! + +### General +- Major refactor of the pool manager, while the API should still be backwards-compatible. There should not be any breaking changes when it comes to using the client libraries. +- The main goal of this release is to bring IntegreSQL's performance on par with our previous native Node.js implementation. + - Specifially we wanted to eliminate some long-running mutex locks (especially when the pool hits the configured pool limit) and make the codebase more maintainable and easier to extend in the future. + - While the above should be already visible in CI-environment, the subjective performance gain while developing locally could be even bigger. + +### Known issues +- We still have no mechanism to limit the global (cross-pool) number of test-databases. + - This is especially problematic if you have many pools running at the same time. + - This could lead to situations where the pool manager is unable to create a new test-databases because the limit (e.g. disk size) is reached even tough some pools/test-databases will probably never be used again. + - This is a **known issue** and will be addressed in a future release. +- OpenAPI/Swagger API documentation is still missing, we are working on it. + +### Added +- GitHub Packages + - Going forward, images are built via GitHub Actions and published to GitHub packages. +- ARM Docker images + - Arm64 is now supported (Apple Silicon M1/M2/M3), we publish a multi-arch image (`linux/amd64,linux/arm64`). +- We added the `POST /api/v1/templates/:hash/tests/:id/recreate` endpoint to the API. + - You can use it to express that you no longer using this database and it can be recreated and returned to the pool. + - Using this endpoint means you want to break out of our FIFO (first in, first out) recreating queue and get your test-database recreated as soon as possible. + - Explicitly calling recreate is **optional** of course! +- Minor: Added woodpecker/drone setup (internal allaboutapps CI/CD) + +### Changed +- Redesigned Database Pool Logic and Template Management + - Reimplemented pool and template logic, separated DB template management from test DB pool, and added per pool workers for preparing test DBs in the background. +- Soft-deprecated the `DELETE /api/v1/templates/:hash/tests/:id` endpoint in favor of `POST /api/v1/templates/:hash/tests/:id/unlock`. + - We did a bad job describing the previous functionality of this endpoint: It's really only deleting the lock, so the exact same test-database can be used again. + - The new `POST /api/v1/templates/:hash/tests/:id/recreate` vs. `POST /api/v1/templates/:hash/tests/:id/unlock` endpoint naming is way more explicit in what it does. + - Closes [#13](https://github.com/allaboutapps/integresql/issues/13) +- Logging and Debugging Improvements + - Introduced zerolog for better logging in the pool and manager modules. Debug statements were refined, and unnecessary print debugging was disabled. + +### Environment Variables + +There have been quite a few additions and changes, thus we have the in-depth details here. + +#### Manager/Pool-related + +- Added `INTEGRESQL_TEST_MAX_POOL_SIZE`: + - Maximal pool size that won't be exceeded + - Defaults to "your number of CPU cores 4 times" [`runtime.NumCPU()*4`](https://pkg.go.dev/runtime#NumCPU) + - Previous default was `500` (hardcoded) + - This might be a **significant change** for some usecases, please adjust accordingly. The pooling recreation logic is now much faster, there is typically no need to have such a high limit of test-databases **per pool**! +- Added `INTEGRESQL_TEST_INITIAL_POOL_SIZE`: + - Initial number of ready DBs prepared in background. The pool is configured to always try to have this number of ready DBs available (it actively tries to recreate databases within the pool in a FIFO manner). + - Defaults to [`runtime.NumCPU()`](https://pkg.go.dev/runtime#NumCPU) + - Previous default was `10` (hardcoded) +- Added `INTEGRESQL_POOL_MAX_PARALLEL_TASKS`: + - Maximal number of pool tasks running in parallel. Must be a number greater or equal 1. + - Defaults to [`runtime.NumCPU()`](https://pkg.go.dev/runtime#NumCPU) +- Added `INTEGRESQL_TEST_DB_RETRY_RECREATE_SLEEP_MIN_MS`: + - Minimal time to wait after a test db recreate has failed (e.g. as client is still connected). Subsequent retries multiply this values until the maximum (below) is reached. + - Defaults to `250`ms +- Added `INTEGRESQL_TEST_DB_RETRY_RECREATE_SLEEP_MAX_MS`: + - The maximum possible sleep time between recreation retries (e.g. 3 seconds), see above. + - Defaults to `3000`ms +- Added `INTEGRESQL_TEST_DB_MINIMAL_LIFETIME_MS`: + - After a test-database transitions from `ready` to `dirty`, always block auto-recreation (FIFO) for this duration (expect `POST /api/v1/templates/:hash/tests/:id/recreate` was called manually). + - Defaults to `250`ms +- Added `INTEGRESQL_TEMPLATE_FINALIZE_TIMEOUT_MS`: + - Internal time to wait for a template-database to transition into the 'finalized' state + - Defaults to `60000`ms (1 minute, same as `INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS`) +- Added `INTEGRESQL_TEST_DB_GET_TIMEOUT_MS`: + - Internal time to wait for a ready database (requested via `/api/v1/templates/:hash/tests`) + - Defaults to `60000`ms (1 minute, same as `INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS`) + - Previous default `10` (was hardcoded) + +#### Server-related + +- Added `INTEGRESQL_DEBUG_ENDPOINTS` + - Enables [pprof debug endpoints](https://golang.org/pkg/net/http/pprof/) under `/debug/*` + - Defaults to `false` +- Added `INTEGRESQL_ECHO_DEBUG` + - Enables [echo framework debug mode](https://echo.labstack.com/docs/customization) + - Defaults to `false` +- Added middlewares, all default to `true` + - `INTEGRESQL_ECHO_ENABLE_CORS_MIDDLEWARE`: [enables CORS](https://echo.labstack.com/docs/middleware/cors) + - `INTEGRESQL_ECHO_ENABLE_LOGGER_MIDDLEWARE`: [enables logger](https://echo.labstack.com/docs/middleware/logger) + - `INTEGRESQL_ECHO_ENABLE_RECOVER_MIDDLEWARE`: [enables recover](https://echo.labstack.com/docs/middleware/recover) + - `INTEGRESQL_ECHO_ENABLE_REQUEST_ID_MIDDLEWARE`: [sets request_id to context](https://echo.labstack.com/docs/middleware/request-id) + - `INTEGRESQL_ECHO_ENABLE_TRAILING_SLASH_MIDDLEWARE`: [auto-adds trailing slash](https://echo.labstack.com/docs/middleware/trailing-slash) + - `INTEGRESQL_ECHO_ENABLE_REQUEST_TIMEOUT_MIDDLEWARE`: [enables timeout middleware](https://echo.labstack.com/docs/middleware/timeout) +- Added `INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS` + - Generic timeout handling for most endpoints. + - Defaults to `60000`ms (1 minute, same as `INTEGRESQL_TEMPLATE_FINALIZE_TIMEOUT_MS` and `INTEGRESQL_TEST_DB_GET_TIMEOUT_MS`) +- Added `INTEGRESQL_LOGGER_LEVEL` + - Defaults to `info` +- Added `INTEGRESQL_LOGGER_REQUEST_LEVEL` + - Defaults to `info` +- Added the following logging settings, which all default to `false` + - `INTEGRESQL_LOGGER_LOG_REQUEST_BODY`: Should the request-log include the body? + - `INTEGRESQL_LOGGER_LOG_REQUEST_HEADER`: Should the request-log include headers? + - `INTEGRESQL_LOGGER_LOG_REQUEST_QUERY`: Should the request-log include the query? + - `INTEGRESQL_LOGGER_LOG_RESPONSE_BODY`: Should the request-log include the response body? + - `INTEGRESQL_LOGGER_LOG_RESPONSE_HEADER`: Should the request-log include the response header? + - `INTEGRESQL_LOGGER_PRETTY_PRINT_CONSOLE`: Should the console logger pretty-print the log (instead of json)? ## v1.0.0 - Initial release May 2020 diff --git a/README.md b/README.md index a6e4387..f09cdd5 100644 --- a/README.md +++ b/README.md @@ -35,32 +35,14 @@ Do your engineers a favour by allowing them to write fast executing, parallel an - [Previous maintainers](#previous-maintainers) - [License](#license) -#### Integrate by client lib - -The flow above might look intimidating at first glance, but trust us, it's simple to integrate especially if there is already an client library available for your specific language. We currently have those: - -* Go: [integresql-client-go](https://github.com/allaboutapps/integresql-client-go) by [Nick Müller - @MorpheusXAUT](https://github.com/MorpheusXAUT) -* Python: [integresql-client-python](https://github.com/msztolcman/integresql-client-python) by [Marcin Sztolcman - @msztolcman](https://github.com/msztolcman) -* .NET: [IntegreSQL.EF](https://github.com/mcctomsk/IntegreSql.EF) by [Artur Drobinskiy - @Shaddix](https://github.com/Shaddix) -* JavaScript/TypeScript: [@devoxa/integresql-client](https://github.com/devoxa/integresql-client) by [Devoxa - @devoxa](https://github.com/devoxa) -* ... *Add your link here and make a PR* - -#### Integrate by RESTful JSON calls - -A really good starting point to write your own integresql-client for a specific language can be found [here (go code)](https://github.com/allaboutapps/integresql-client-go/blob/master/client.go) and [here (godoc)](https://pkg.go.dev/github.com/allaboutapps/integresql-client-go?tab=doc). It's just RESTful JSON after all. - -#### Demo - -If you want to take a look on how we integrate IntegreSQL - 🤭 - please just try our [go-starter](https://github.com/allaboutapps/go-starter) project or take a look at our [test_database setup code](https://github.com/allaboutapps/go-starter/blob/master/internal/test/test_database.go). - ## Install ### Install using Docker (preferred) -A minimal Docker image containing a pre-built `IntegreSQL` executable is available at [Docker Hub](https://hub.docker.com/r/allaboutapps/integresql). +A minimal Docker image containing a pre-built `IntegreSQL` executable is available at [Github Packages](https://github.com/allaboutapps/integresql/releases). ```bash -docker pull allaboutapps/integresql +docker pull ghcr.io/allaboutapps/integresql ``` ### Install locally @@ -71,29 +53,6 @@ Installing `IntegreSQL` locally requires a working [Go](https://golang.org/dl/) go get github.com/allaboutapps/integresql/cmd/server ``` -## Configuration - -`IntegreSQL` requires little configuration, all of which has to be provided via environment variables (due to the intended usage in a Docker environment). The following settings are available: - -| Description | Environment variable | Default | Required | -| ----------------------------------------------------------------- | ------------------------------------- | -------------------- | -------- | -| IntegreSQL: listen address (defaults to all if empty) | `INTEGRESQL_ADDRESS` | `""` | | -| IntegreSQL: port | `INTEGRESQL_PORT` | `5000` | | -| PostgreSQL: host | `INTEGRESQL_PGHOST`, `PGHOST` | `"127.0.0.1"` | Yes | -| PostgreSQL: port | `INTEGRESQL_PGPORT`, `PGPORT` | `5432` | | -| PostgreSQL: username | `INTEGRESQL_PGUSER`, `PGUSER`, `USER` | `"postgres"` | Yes | -| PostgreSQL: password | `INTEGRESQL_PGPASSWORD`, `PGPASSWORD` | `""` | Yes | -| PostgreSQL: database for manager | `INTEGRESQL_PGDATABASE` | `"postgres"` | | -| PostgreSQL: template database to use | `INTEGRESQL_ROOT_TEMPLATE` | `"template0"` | | -| Managed databases: prefix | `INTEGRESQL_DB_PREFIX` | `"integresql"` | | -| Managed *template* databases: prefix `integresql_template_` | `INTEGRESQL_TEMPLATE_DB_PREFIX` | `"template"` | | -| Managed *test* databases: prefix `integresql_test__` | `INTEGRESQL_TEST_DB_PREFIX` | `"test"` | | -| Managed *test* databases: username | `INTEGRESQL_TEST_PGUSER` | PostgreSQL: username | | -| Managed *test* databases: password | `INTEGRESQL_TEST_PGPASSWORD` | PostgreSQL: password | | -| Managed *test* databases: minimal test pool size | `INTEGRESQL_TEST_INITIAL_POOL_SIZE` | `10` | | -| Managed *test* databases: maximal test pool size | `INTEGRESQL_TEST_MAX_POOL_SIZE` | `500` | | - - ## Usage ### Run using Docker (preferred) @@ -180,6 +139,97 @@ export PGPASSWORD=testpass integresql ``` +## Configuration + +`IntegreSQL` requires little configuration, all of which has to be provided via environment variables (due to the intended usage in a Docker environment). The following settings are available: + +| Description | Environment variable | Default | Required | +| ----------------------------------------------------------------- | ------------------------------------- | -------------------- | -------- | +| IntegreSQL: listen address (defaults to all if empty) | `INTEGRESQL_ADDRESS` | `""` | | +| IntegreSQL: port | `INTEGRESQL_PORT` | `5000` | | +| PostgreSQL: host | `INTEGRESQL_PGHOST`, `PGHOST` | `"127.0.0.1"` | Yes | +| PostgreSQL: port | `INTEGRESQL_PGPORT`, `PGPORT` | `5432` | | +| PostgreSQL: username | `INTEGRESQL_PGUSER`, `PGUSER`, `USER` | `"postgres"` | Yes | +| PostgreSQL: password | `INTEGRESQL_PGPASSWORD`, `PGPASSWORD` | `""` | Yes | +| PostgreSQL: database for manager | `INTEGRESQL_PGDATABASE` | `"postgres"` | | +| PostgreSQL: template database to use | `INTEGRESQL_ROOT_TEMPLATE` | `"template0"` | | +| Managed databases: prefix | `INTEGRESQL_DB_PREFIX` | `"integresql"` | | +| Managed *template* databases: prefix `integresql_template_` | `INTEGRESQL_TEMPLATE_DB_PREFIX` | `"template"` | | +| Managed *test* databases: prefix `integresql_test__` | `INTEGRESQL_TEST_DB_PREFIX` | `"test"` | | +| Managed *test* databases: username | `INTEGRESQL_TEST_PGUSER` | PostgreSQL: username | | +| Managed *test* databases: password | `INTEGRESQL_TEST_PGPASSWORD` | PostgreSQL: password | | +| Managed *test* databases: minimal test pool size | `INTEGRESQL_TEST_INITIAL_POOL_SIZE` | `10` | | +| Managed *test* databases: maximal test pool size | `INTEGRESQL_TEST_MAX_POOL_SIZE` | `500` | | + + +## Integrate + +IntegreSQL is a RESTful JSON API distributed as Docker image or go cli. It's language agnostic and manages multiple [PostgreSQL templates](https://supabase.io/blog/2020/07/09/postgresql-templates/) and their separate pool of test databases for your tests. It keeps the pool of test databases warm (as it's running in the background) and is fit for parallel test execution with multiple test runners / processes. + +You will typically want to integrate by a client lib (see below), but you can also integrate by RESTful JSON calls directly. The flow is introducd below. + +### Integrate by RESTful JSON calls + +You development/testing flow should look like this: + +* **Start IntegreSQL** and leave it running **in the background** (your PostgreSQL template and test database pool will always be warm) +* ... +* You trigger your test command. 1..n test runners/processes start in parallel +* **Once** per test runner/process: + * Get migrations/fixtures files `hash` over all related database files + * `InitializeTemplate: POST /templates`: attempt to create a new PostgreSQL template database identifying though the above hash `payload: {"hash": "string"}` + * `StatusOK: 200` + * Truncate + * Apply all migrations + * Seed all fixtures + * `FinalizeTemplate: PUT /api/v1/templates/:hash` + * If you encountered any template setup errors call `DiscardTemplate: DELETE /api/v1/templates/:hash` + * `StatusLocked: 423` + * Some other process has already recreated a PostgreSQL template database for this `hash` (or is currently doing it), you can just consider the template ready at this point. + * `StatusServiceUnavailable: 503` + * Typically happens if IntegreSQL cannot communicate with PostgreSQL, fail the test runner process +* **Before each** test `GetTestDatabase: GET /api/v1/templates/:hash/tests` + * Blocks until the template database is finalized (via `FinalizeTemplate`) + * `StatusOK: 200` + * You get a fully isolated PostgreSQL database from our already migrated/seeded template database to use within your test + * `StatusNotFound: 404` + * Well, seems like someone forgot to call `InitializeTemplate` or it errored out. + * `StatusGone: 410` + * There was an error during test setup with our fixtures, someone called `DiscardTemplate`, thus this template cannot be used. + * `StatusServiceUnavailable: 503` + * Well, typically a PostgreSQL connectivity problem +* Utilizing the isolated PostgreSQL test database received from IntegreSQL for each (parallel) test: + * **Run your test code** +* **After each** test **optional**: + * `RecreateTestDatabase: POST /api/v1/templates/:hash/tests/:id/recreate` + * Recreates the test DB according to the template and returns it back to the pool. + * **This is optional!** If you don't call this endpoint, the test database will be recreated in a FIFO manner (first in, first out) as soon as possible. + * This is useful if you have parallel testing with a mix of very long and super short tests. Our auto–FIFO recreation handling might block there. + * `ReturnTestDatabase: POST /api/v1/templates/:hash/tests/:id/unlock` (previously and soft-deprecated `DELETE /api/v1/templates/:hash/tests/:id`) + * Returns the given test DB directly to the pool, without cleaning (recreating it). + * **This is optional!** If you don't call this endpoints, the test database will be recreated in a FIFO manner (first in, first out) as soon as possible, even though it actually had no changes. + * This is useful if you are sure, you did not do any changes to the database and thus want to skip the recreation process by returning it to the pool directly. + +* 1..n test runners end +* ... +* Subsequent 1..n test runners start/end in parallel and reuse the above logic + +A really good starting point to write your own integresql-client for a specific language can be found [here (go code)](https://github.com/allaboutapps/integresql-client-go/blob/master/client.go) and [here (godoc)](https://pkg.go.dev/github.com/allaboutapps/integresql-client-go?tab=doc). It's just RESTful JSON after all. + +### Integrate by client lib + +The flow above might look intimidating at first glance, but trust us, it's simple to integrate especially if there is already an client library available for your specific language. We currently have those: + +* Go: [integresql-client-go](https://github.com/allaboutapps/integresql-client-go) by [Nick Müller - @MorpheusXAUT](https://github.com/MorpheusXAUT) +* Python: [integresql-client-python](https://github.com/msztolcman/integresql-client-python) by [Marcin Sztolcman - @msztolcman](https://github.com/msztolcman) +* .NET: [IntegreSQL.EF](https://github.com/mcctomsk/IntegreSql.EF) by [Artur Drobinskiy - @Shaddix](https://github.com/Shaddix) +* JavaScript/TypeScript: [@devoxa/integresql-client](https://github.com/devoxa/integresql-client) by [Devoxa - @devoxa](https://github.com/devoxa) +* ... *Add your link here and make a PR* + +#### Demo + +If you want to take a look on how we integrate IntegreSQL - 🤭 - please just try our [go-starter](https://github.com/allaboutapps/go-starter) project or take a look at our [test_database setup code](https://github.com/allaboutapps/go-starter/blob/master/internal/test/test_database.go). + ## Background We came a long way to realize that something just did not feel right with our PostgreSQL integration testing strategies. @@ -370,43 +420,86 @@ We realized that having the above pool logic directly within the test runner is As we switched to Go as our primary backend engineering language, we needed to rewrite the above logic anyways and decided to provide a safe and language agnostic way to utilize this testing strategy with PostgreSQL. -IntegreSQL is a RESTful JSON API distributed as Docker image or go cli. It's language agnostic and manages multiple [PostgreSQL templates](https://supabase.io/blog/2020/07/09/postgresql-templates/) and their separate pool of test databases for your tests. It keeps the pool of test databases warm (as it's running in the background) and is fit for parallel test execution with multiple test runners / processes. +This is how `IntegreSQL` was born. -Our flow now finally changed to this: +## Benchmarks -* **Start IntegreSQL** and leave it running **in the background** (your PostgreSQL template and test database pool will always be warm) -* ... -* 1..n test runners start in parallel -* Once per test runner process - * Get migrations/fixtures files `hash` over all related database files - * `InitializeTemplate: POST /templates`: attempt to create a new PostgreSQL template database identifying though the above hash `payload: {"hash": "string"}` - * `StatusOK: 200` - * Truncate - * Apply all migrations - * Seed all fixtures - * `FinalizeTemplate: PUT /templates/{hash}` - * If you encountered any template setup errors call `DiscardTemplate: DELETE /templates/{hash}` - * `StatusLocked: 423` - * Some other process has already recreated a PostgreSQL template database for this `hash` (or is currently doing it), you can just consider the template ready at this point. - * `StatusServiceUnavailable: 503` - * Typically happens if IntegreSQL cannot communicate with PostgreSQL, fail the test runner process -* **Before each** test `GetTestDatabase: GET /templates/{hash}/tests` - * Blocks until the template database is finalized (via `FinalizeTemplate`) - * `StatusOK: 200` - * You get a fully isolated PostgreSQL database from our already migrated/seeded template database to use within your test - * `StatusNotFound: 404` - * Well, seems like someone forgot to call `InitializeTemplate` or it errored out. - * `StatusGone: 410` - * There was an error during test setup with our fixtures, someone called `DiscardTemplate`, thus this template cannot be used. - * `StatusServiceUnavailable: 503` - * Well, typically a PostgreSQL connectivity problem -* Utilizing the isolated PostgreSQL test database received from IntegreSQL for each (parallel) test: - * **Run your test code** -* **After each** test optional: `ReturnTestDatabase: DELETE /templates/{hash}/tests/{test-database-id}` - * Marks the test database that it can be wiped early on pool limit overflow (or reused if `true` is submitted) -* 1..n test runners end -* ... -* Subsequent 1..n test runners start/end in parallel and reuse the above logic +### Benchmark v1.1.0 vs v1.0.0 + +We focued on improving the pool manager performance in v1.1.0, especially when it comes to locking and thus request latency. + +![benchmark comparison v1.1.0](docs/benchmark_v1_1_0.png) + +The main goal was to bring IntegreSQL's performance on par with our previous native Node.js implementation, which we also benchmarked: + +```bash + +# Previous Node.js implementation +--- ----------------------------------- --- + replicas switched: 563 avg=14ms min=6ms max=316ms + replicas awaited: 1 prebuffer=8 avg=301ms max=301ms + background replicas: 571 avg=-ms min=-ms max=1180ms + - warm up: 32% 4041ms + * drop/cache check: 4% 561ms + * migrate/cache reuse: 25% 3177ms + * fixtures: 2% 302ms + * special: 0% 0ms + * create pool: 0% 1ms + - switching: 67% 8294ms + * disconnect: 1% 139ms + * switch slave: 4% 591ms + - resolve next: 2% 290ms + - await next: 2% 301ms + * reinitialize: 61% 7563ms + strategy related time: 12335ms + vs total executed time: 11% 106184ms +--- --------------------------------- --- +Done in 106.60s. + +# IntegreSQL v1.1.0 (next version) +--- ----------------------------------- --- + replicas switched: 563 avg=70ms min=58ms max=603ms + replicas awaited: 1 prebuffer=8 avg=72ms max=72ms + background replicas: 571 avg=58ms min=49ms max=520ms + - warm up: 9% 4101ms + * drop/cache check: 0% 1ms + * migrate/cache reuse: 8% 3520ms + * fixtures: 0% 296ms + * special: 0% 0ms + * create pool: 0% 284ms + - switching: 90% 39865ms + * disconnect: 0% 120ms + * switch replica: 0% 261ms (563x min=0ms q25=0ms q50=0ms q75=1ms q95=1ms max=72ms) + - resolve next: 0% 189ms + - await next: 0% 72ms + * reinitialize: 89% 39478ms (563x min=58ms q25=66ms q50=68ms q75=71ms q95=80ms max=531ms) + strategy related time: 43966ms + vs total executed time: 40% 109052ms +--- --------------------------------- --- +Done in 109.45s. + +# IntegreSQL v1.0.0 (previous version) +--- ----------------------------------- --- + replicas switched: 563 avg=131ms min=9ms max=2019ms + replicas awaited: 94 prebuffer=8 avg=590ms max=1997ms + background replicas: 571 avg=1292ms min=52ms max=3817ms + - warm up: 7% 6144ms + * drop/cache check: 0% 0ms + * migrate/cache reuse: 4% 3587ms + * fixtures: 0% 298ms + * special: 0% 0ms + * create pool: 2% 2259ms + - switching: 92% 73837ms + * disconnect: 0% 112ms + * switch replica: 64% 51552ms (563x min=0ms q25=0ms q50=0ms q75=1ms q95=773ms max=1997ms) + - resolve next: 5% 3922ms + - await next: 69% 55474ms + * reinitialize: 27% 22169ms (563x min=9ms q25=12ms q50=15ms q75=19ms q95=187ms max=1201ms) + strategy related time: 79981ms + vs total executed time: 51% 153889ms +--- --------------------------------- --- +Done in 154.29s. +``` ## Contributing diff --git a/docs/benchmark_v1_1_0.png b/docs/benchmark_v1_1_0.png new file mode 100644 index 0000000..ab4cb06 Binary files /dev/null and b/docs/benchmark_v1_1_0.png differ diff --git a/internal/api/server_config.go b/internal/api/server_config.go index 316cace..8840e63 100644 --- a/internal/api/server_config.go +++ b/internal/api/server_config.go @@ -1,6 +1,8 @@ package api import ( + "time" + "github.com/allaboutapps/integresql/pkg/util" "github.com/rs/zerolog" ) @@ -21,6 +23,8 @@ type EchoConfig struct { EnableRecoverMiddleware bool EnableRequestIDMiddleware bool EnableTrailingSlashMiddleware bool + EnableTimeoutMiddleware bool + RequestTimeout time.Duration } type LoggerConfig struct { @@ -38,7 +42,7 @@ func DefaultServerConfigFromEnv() ServerConfig { return ServerConfig{ Address: util.GetEnv("INTEGRESQL_ADDRESS", ""), Port: util.GetEnvAsInt("INTEGRESQL_PORT", 5000), - DebugEndpoints: util.GetEnvAsBool("INTEGRESQL_DEBUG_ENDPOINTS", true), // https://golang.org/pkg/net/http/pprof/ + DebugEndpoints: util.GetEnvAsBool("INTEGRESQL_DEBUG_ENDPOINTS", false), // https://golang.org/pkg/net/http/pprof/ Echo: EchoConfig{ Debug: util.GetEnvAsBool("INTEGRESQL_ECHO_DEBUG", false), EnableCORSMiddleware: util.GetEnvAsBool("INTEGRESQL_ECHO_ENABLE_CORS_MIDDLEWARE", true), @@ -46,10 +50,15 @@ func DefaultServerConfigFromEnv() ServerConfig { EnableRecoverMiddleware: util.GetEnvAsBool("INTEGRESQL_ECHO_ENABLE_RECOVER_MIDDLEWARE", true), EnableRequestIDMiddleware: util.GetEnvAsBool("INTEGRESQL_ECHO_ENABLE_REQUEST_ID_MIDDLEWARE", true), EnableTrailingSlashMiddleware: util.GetEnvAsBool("INTEGRESQL_ECHO_ENABLE_TRAILING_SLASH_MIDDLEWARE", true), + EnableTimeoutMiddleware: util.GetEnvAsBool("INTEGRESQL_ECHO_ENABLE_REQUEST_TIMEOUT_MIDDLEWARE", true), + + // typically these timeouts should be the same as INTEGRESQL_TEMPLATE_FINALIZE_TIMEOUT_MS and INTEGRESQL_TEST_DB_GET_TIMEOUT_MS + // pkg/manager/manager_config.go + RequestTimeout: time.Millisecond * time.Duration(util.GetEnvAsInt("INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS", 60*1000 /*1 min*/)), // affects INTEGRESQL_TEMPLATE_FINALIZE_TIMEOUT_MS and INTEGRESQL_TEST_DB_GET_TIMEOUT_MS }, Logger: LoggerConfig{ Level: util.LogLevelFromString(util.GetEnv("INTEGRESQL_LOGGER_LEVEL", zerolog.InfoLevel.String())), - RequestLevel: util.LogLevelFromString(util.GetEnv("INTEGRESQL_LOGGER_REQUEST_LEVEL", zerolog.DebugLevel.String())), + RequestLevel: util.LogLevelFromString(util.GetEnv("INTEGRESQL_LOGGER_REQUEST_LEVEL", zerolog.InfoLevel.String())), LogRequestBody: util.GetEnvAsBool("INTEGRESQL_LOGGER_LOG_REQUEST_BODY", false), LogRequestHeader: util.GetEnvAsBool("INTEGRESQL_LOGGER_LOG_REQUEST_HEADER", false), LogRequestQuery: util.GetEnvAsBool("INTEGRESQL_LOGGER_LOG_REQUEST_QUERY", false), diff --git a/internal/api/templates/templates.go b/internal/api/templates/templates.go index a81d1db..bc10c84 100644 --- a/internal/api/templates/templates.go +++ b/internal/api/templates/templates.go @@ -1,11 +1,9 @@ package templates import ( - "context" "errors" "net/http" "strconv" - "time" "github.com/allaboutapps/integresql/internal/api" "github.com/allaboutapps/integresql/pkg/manager" @@ -29,10 +27,7 @@ func postInitializeTemplate(s *api.Server) echo.HandlerFunc { return echo.NewHTTPError(http.StatusBadRequest, "hash is required") } - ctx, cancel := context.WithTimeout(c.Request().Context(), 30*time.Second) - defer cancel() - - template, err := s.Manager.InitializeTemplateDatabase(ctx, payload.Hash) + template, err := s.Manager.InitializeTemplateDatabase(c.Request().Context(), payload.Hash) if err != nil { if errors.Is(err, manager.ErrManagerNotReady) { return echo.ErrServiceUnavailable @@ -52,10 +47,7 @@ func putFinalizeTemplate(s *api.Server) echo.HandlerFunc { return func(c echo.Context) error { hash := c.Param("hash") - ctx, cancel := context.WithTimeout(c.Request().Context(), 10*time.Second) - defer cancel() - - if _, err := s.Manager.FinalizeTemplateDatabase(ctx, hash); err != nil { + if _, err := s.Manager.FinalizeTemplateDatabase(c.Request().Context(), hash); err != nil { if errors.Is(err, manager.ErrTemplateAlreadyInitialized) { // template is initialized, we ignore this error return c.NoContent(http.StatusNoContent) @@ -77,10 +69,7 @@ func deleteDiscardTemplate(s *api.Server) echo.HandlerFunc { return func(c echo.Context) error { hash := c.Param("hash") - ctx, cancel := context.WithTimeout(c.Request().Context(), 10*time.Second) - defer cancel() - - if err := s.Manager.DiscardTemplateDatabase(ctx, hash); err != nil { + if err := s.Manager.DiscardTemplateDatabase(c.Request().Context(), hash); err != nil { if errors.Is(err, manager.ErrManagerNotReady) { return echo.ErrServiceUnavailable } else if errors.Is(err, manager.ErrTemplateNotFound) { @@ -96,13 +85,11 @@ func deleteDiscardTemplate(s *api.Server) echo.HandlerFunc { } func getTestDatabase(s *api.Server) echo.HandlerFunc { + return func(c echo.Context) error { hash := c.Param("hash") - ctx, cancel := context.WithTimeout(c.Request().Context(), 1*time.Minute) - defer cancel() - - test, err := s.Manager.GetTestDatabase(ctx, hash) + test, err := s.Manager.GetTestDatabase(c.Request().Context(), hash) if err != nil { if errors.Is(err, manager.ErrManagerNotReady) { @@ -134,10 +121,7 @@ func postUnlockTestDatabase(s *api.Server) echo.HandlerFunc { return echo.NewHTTPError(http.StatusBadRequest, "invalid test database ID") } - ctx, cancel := context.WithTimeout(c.Request().Context(), 10*time.Second) - defer cancel() - - if err := s.Manager.ReturnTestDatabase(ctx, hash, id); err != nil { + if err := s.Manager.ReturnTestDatabase(c.Request().Context(), hash, id); err != nil { if errors.Is(err, manager.ErrManagerNotReady) { return echo.ErrServiceUnavailable } else if errors.Is(err, manager.ErrTemplateNotFound) { diff --git a/internal/router/router.go b/internal/router/router.go index 4fac61d..0ff08c3 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -61,6 +61,12 @@ func Init(s *api.Server) { log.Warn().Msg("Disabling logger middleware due to environment config") } + if s.Config.Echo.EnableTimeoutMiddleware { + s.Echo.Use(echoMiddleware.TimeoutWithConfig(echoMiddleware.TimeoutConfig{ + Timeout: s.Config.Echo.RequestTimeout, + })) + } + // enable debug endpoints only if requested if s.Config.DebugEndpoints { s.Echo.GET("/debug/*", echo.WrapHandler(http.DefaultServeMux)) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 67f13f0..554ab52 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -157,6 +157,10 @@ func (m Manager) Ready() bool { return m.db != nil } +func (m Manager) Config() ManagerConfig { + return m.config +} + func (m *Manager) Initialize(ctx context.Context) error { log := m.getManagerLogger(ctx, "Initialize") diff --git a/pkg/manager/manager_config.go b/pkg/manager/manager_config.go index d59f000..a37a13e 100644 --- a/pkg/manager/manager_config.go +++ b/pkg/manager/manager_config.go @@ -53,8 +53,11 @@ func DefaultManagerConfigFromEnv() ManagerConfig { // we reuse the same user (PGUSER) and passwort (PGPASSWORT) for the test / template databases by default TestDatabaseOwner: util.GetEnv("INTEGRESQL_TEST_PGUSER", util.GetEnv("INTEGRESQL_PGUSER", util.GetEnv("PGUSER", "postgres"))), TestDatabaseOwnerPassword: util.GetEnv("INTEGRESQL_TEST_PGPASSWORD", util.GetEnv("INTEGRESQL_PGPASSWORD", util.GetEnv("PGPASSWORD", ""))), - TemplateFinalizeTimeout: time.Millisecond * time.Duration(util.GetEnvAsInt("INTEGRESQL_TEMPLATE_FINALIZE_TIMEOUT_MS", 5*60*1000 /*5 min*/)), - TestDatabaseGetTimeout: time.Millisecond * time.Duration(util.GetEnvAsInt("INTEGRESQL_TEST_DB_GET_TIMEOUT_MS", 1*60*1000 /*1 min, timeout hardcoded also in GET request handler*/)), + + // typically these timeouts should be the same as INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS + // see internal/api/server_config.go + TemplateFinalizeTimeout: time.Millisecond * time.Duration(util.GetEnvAsInt("INTEGRESQL_TEMPLATE_FINALIZE_TIMEOUT_MS", util.GetEnvAsInt("INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS", 60*1000 /*1 min*/))), + TestDatabaseGetTimeout: time.Millisecond * time.Duration(util.GetEnvAsInt("INTEGRESQL_TEST_DB_GET_TIMEOUT_MS", util.GetEnvAsInt("INTEGRESQL_ECHO_REQUEST_TIMEOUT_MS", 60*1000 /*1 min*/))), PoolConfig: pool.PoolConfig{ InitialPoolSize: util.GetEnvAsInt("INTEGRESQL_TEST_INITIAL_POOL_SIZE", runtime.NumCPU()), // previously default 10