From d00d13dabb3ef0729ed1bfe0ebef1bd5b565b6f8 Mon Sep 17 00:00:00 2001 From: Arvindh <30824765+arvindh123@users.noreply.github.com> Date: Fri, 3 Feb 2023 22:06:18 +0530 Subject: [PATCH] MF-1556 - Move the most used functions in main.go to internal package (#1601) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * MF-1525 - Add graceful stop for HTTP and GRPC servers (#1548) * Add : errgroup to cmd/auth Signed-off-by: Arvindh * Add : Handle graceful stop for auth service Remove : errgroups from auth service Signed-off-by: Arvindh * Add : Wait till server shutdown Signed-off-by: Arvindh * Change : instead of waitgroup changed to errgroups Signed-off-by: Arvindh * change : KillSignalHandler return type to error Signed-off-by: Arvindh * Empty Commit Signed-off-by: Arvindh * Add : Context to http server shutdown Rename : varaible from proto to protocol Signed-off-by: Arvindh * change : to default log level Signed-off-by: Arvindh * Add : Sign-off Signed-off-by: Arvindh * Add: graceful stop of http and grpc server Signed-off-by: Arvindh * Fix: typos and caps Signed-off-by: Arvindh * Add: Signed-off Signed-off-by: Arvindh * Rename: Func KillSignalHandler to SignalHandler Add: SIGABRT Signed-off-by: Arvindh * Fix: auth service Signed-off-by: Arvindh * Add: timeout for grpc gracefulstop Fix: typos Signed-off-by: Arvindh * Add: .vscode folder to git ignore Signed-off-by: Arvindh * change: variable name to stopWaitTime Signed-off-by: Arvindh * remove: .vscode folder Signed-off-by: Arvindh * remove: .vscode from .gitignore Signed-off-by: Arvindh * Add : logger to handlers Signed-off-by: Arvindh * Add : New line at end of .gitignore file Signed-off-by: Arvindh * Fix : variable naming Add : graceful stop for timescale Signed-off-by: Arvindh * Remove : unsued NATS library from import Signed-off-by: Arvindh * Move: "https" and "https" to moved to const var Signed-off-by: Arvindh * Move: "http" and "https" to moved to const var Signed-off-by: Arvindh * update: branch with master Signed-off-by: Arvindh Co-authored-by: Dušan Borovčanin Co-authored-by: Drasko DRASKOVIC Signed-off-by: Arvindh * Add: httpserver and grpcsever Signed-off-by: Arvindh * MF-1588 - Update Subscriber interface (#1598) * Initial commit Signed-off-by: 0x6f736f646f * Update subscriber interface Signed-off-by: dusanb94 * Add tests Signed-off-by: 0x6f736f646f * Add tests Signed-off-by: 0x6f736f646f * check subscription map Signed-off-by: 0x6f736f646f * Check topic id after topic Signed-off-by: 0x6f736f646f * reword description Signed-off-by: 0x6f736f646f * Setup empty queue Signed-off-by: 0x6f736f646f * Change mqtt implementation Signed-off-by: 0x6f736f646f * Switch statements Signed-off-by: 0x6f736f646f * Simplify Signed-off-by: 0x6f736f646f * Change mqtt subscriber Signed-off-by: 0x6f736f646f * Protect subscription map Signed-off-by: 0x6f736f646f * Fix subscription Signed-off-by: 0x6f736f646f * Set client id Signed-off-by: 0x6f736f646f * Format Signed-off-by: 0x6f736f646f * Change delete method Signed-off-by: 0x6f736f646f Co-authored-by: Dušan Borovčanin Signed-off-by: Arvindh * move http and grpc server functions Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * Move Keto and Jaeger Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * Add metrics and auth Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * MF - 1590 - Fix fetching list of users with a zero limit (#1594) * Add max and min limit size Signed-off-by: 0x6f736f646f * Format Signed-off-by: 0x6f736f646f * Format Signed-off-by: 0x6f736f646f Co-authored-by: Dušan Borovčanin Signed-off-by: Arvindh * NOISSUE - Retrieve client key on cert issuing (#1607) Signed-off-by: Manuel Imperiale Signed-off-by: Arvindh * fix bug (#1604) Signed-off-by: zhangchuanfeng <654300242@qq.com> Signed-off-by: Arvindh * Rename service name Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * Change metrics method Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * Rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * Rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * Rename package name Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :truck: Rename Keto and Jaeger functions Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * unify grpc service Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :truck: rename apiutil to initutil Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :sparkles: coap server Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :truck: rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :truck: Rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :recycle: rename packages Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :recycle: remove mf prefix Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * :truck: rename server error Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * remove dead code Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * MF - 1416 - Queue Abstraction for Mainflux & RabbitMQ Support (#1562) * MF-1263 - Move repeating errors to the separate package (#1540) * MF-1263 - Mv duplicated errors to pkg/errors Signed-off-by: Manuel Imperiale * Revert test build flags Signed-off-by: Manuel Imperiale * Fix merge Signed-off-by: Manuel Imperiale * Fix comment Signed-off-by: Manuel Imperiale Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * NOISSUE - Fix auth members list response (#1555) * NOISSUE - Fix auth members list response Signed-off-by: Manuel Imperiale * Move group type next to page details Signed-off-by: Manuel Imperiale * Rm membersRes Signed-off-by: Manuel Imperiale * Fix typo Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * MF-1261 - Use StatusUnauthorized for authn and StatusForbidden for authz (#1538) * MF-1261 - Use StatusUnauthorized for authn and StatusForbidden for authz Signed-off-by: Manuel Imperiale * ErrExternalKey typo Signed-off-by: Manuel Imperiale * Rename ErrUnauthorizedAcces -> ErrAuthentication Signed-off-by: Manuel Imperiale * Fix bootstrap error Signed-off-by: Manuel Imperiale * Fix status code in openapi Signed-off-by: Manuel Imperiale * Fix test description Signed-off-by: Manuel Imperiale * Fix test description Signed-off-by: Manuel Imperiale * Fix test description Signed-off-by: Manuel Imperiale * Add errors cases Signed-off-by: Manuel Imperiale * Fix status codes Signed-off-by: Manuel Imperiale * Add gRPC stutus code Signed-off-by: Manuel Imperiale * Fix tests description Signed-off-by: Manuel Imperiale * Fix openapi and encodeError Signed-off-by: Manuel Imperiale * Fix grpc message Signed-off-by: Manuel Imperiale * Fix test descriptions Signed-off-by: Manuel Imperiale * Revert sdk error Signed-off-by: Manuel Imperiale * Fix typo Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * MF-1059 - Add TLS support for email (#1560) * Use gomail package for sending emails Signed-off-by: Ivan Milosevic * remove print err Signed-off-by: Ivan Milosevic * Add vendor Signed-off-by: Ivan Milosevic * Rename email structure remove logger Signed-off-by: Ivan Milosevic * typo in var name Signed-off-by: Ivan Milosevic * rename var Signed-off-by: Ivan Milosevic * remove MF_EMAIL_SECRET Signed-off-by: Ivan Milosevic Signed-off-by: 0x6f736f646f * NOISSUE - Refactor MQTT subscriber (#1561) * correct suscriber interface validator + refactore token error handling Signed-off-by: tzzed * apply review suggestion Signed-off-by: tzzed Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * MF-1257 - Access messages from readers endpoint with user access token (#1470) * remove owner id Signed-off-by: Mirko Teodorovic * add user auth for db reader Signed-off-by: mteodor * add user auth for db reader Signed-off-by: mteodor * enable mongodb reader for user token reading Signed-off-by: mteodor * use uuid check for auth switch between thing key and user tok Signed-off-by: mteodor * enable user token reading Signed-off-by: mteodor * revert to correct version Signed-off-by: mteodor * fix endpoint test, add additional tests Signed-off-by: mteodor * remove logs,dead code Signed-off-by: mteodor * fix logging messages Signed-off-by: mteodor * remove auth interface, add authorization header type Signed-off-by: mteodor * update api doc Signed-off-by: mteodor * remove unused package Signed-off-by: mteodor * some refactor of cases for authorization switch Signed-off-by: mteodor * correct description in openapi Signed-off-by: mteodor * fix endpoint test to match auth service change Signed-off-by: mteodor * some rename Signed-off-by: mteodor * initialize auth url Signed-off-by: mteodor * add env variables for auth service Signed-off-by: mteodor * fix spelling Signed-off-by: mteodor * Things prefix and no prefix for Thing authorization, Bearer for user Signed-off-by: mteodor * update readme file Signed-off-by: mteodor * fix default things grpc port Signed-off-by: mteodor * enable user reading for timescaledb Signed-off-by: mteodor * remove not used error Signed-off-by: mteodor * improve errors Signed-off-by: mteodor * refactor authorize Signed-off-by: mteodor * add chanID check Signed-off-by: mteodor * inline some error checking Signed-off-by: mteodor * fixing errors Signed-off-by: mteodor * fixing errors Signed-off-by: mteodor * improve test case description Signed-off-by: mteodor * remove test code Signed-off-by: mteodor * dont inline Signed-off-by: mteodor * refactor a bit encodeError Signed-off-by: mteodor * remove unused error Signed-off-by: mteodor * remove unused error Signed-off-by: mteodor * fix things auth grpc url Signed-off-by: mteodor * rename variables for header prefix Signed-off-by: mteodor Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * Initial commit of adding rabbitmq broker Signed-off-by: 0x6f736f646f * Initial commit of adding rabbitmq broker Signed-off-by: 0x6f736f646f * Initial commit for tests Signed-off-by: 0x6f736f646f * Bump up tests Signed-off-by: 0x6f736f646f * Add more tests Signed-off-by: 0x6f736f646f * Add go routines Signed-off-by: 0x6f736f646f * Initial commit of adding rabbitmq broker Signed-off-by: 0x6f736f646f * Initial commit for tests Signed-off-by: 0x6f736f646f * Bump up tests Signed-off-by: 0x6f736f646f * Add more tests Signed-off-by: 0x6f736f646f * Add go routines Signed-off-by: 0x6f736f646f * Fix tests Signed-off-by: 0x6f736f646f * Fix with wait groups Signed-off-by: 0x6f736f646f * unsubscribe to stop delivering messages Signed-off-by: 0x6f736f646f * Remove exclusivity Signed-off-by: 0x6f736f646f * MF-1551 - Fix Cobra usage commands and clean unnecessary struct types (#1558) * MF-1551 - Fix Cobra usage commands and clean unnecessary struct types Signed-off-by: Manuel Imperiale * Use linux syntax for cmd usage description Signed-off-by: Manuel Imperiale * Fix typo Signed-off-by: Manuel Imperiale * Fix cmd.Use Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * NOISSUE - Separate Keto hosts for read and write (#1563) * Separate keto hosts for read and write Signed-off-by: Ivan Milosevic * update readme with new envars Signed-off-by: Ivan Milosevic * rename read connection name Signed-off-by: Ivan Milosevic Co-authored-by: Dušan Borovčanin Co-authored-by: Drasko DRASKOVIC Signed-off-by: 0x6f736f646f * Update dependencies (#1564) Signed-off-by: dusanb94 Signed-off-by: 0x6f736f646f * MF-1240 - Return to service transport layer only service errors (#1559) * MF-1240 - Return to service transport layer only service errors Signed-off-by: Manuel Imperiale * Remove unecessary errors Signed-off-by: Manuel Imperiale * Rm duplicated errors and fix transport Signed-off-by: Manuel Imperiale * Revert http endpoint_test Signed-off-by: Manuel Imperiale * Fix conflict Signed-off-by: Manuel Imperiale Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * Implement cancel mechanisms Signed-off-by: 0x6f736f646f * Queuename as parameter Signed-off-by: 0x6f736f646f * Queuename as parameter Signed-off-by: 0x6f736f646f * MF-1469 - Indicate proper authentication scheme in Authorization header (#1523) * MF-1469 - Indicate proper authentication scheme in Authorization header Signed-off-by: Stefan Kovacevic * Fixing the remarks on the last push Signed-off-by: Stefan Kovacevic * Remove Bearer prefix in all services and fix tests Signed-off-by: Manuel Imperiale * Fix remarks Signed-off-by: Manuel Imperiale Co-authored-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * NOISSUE - Add nats wrapper for COAP (#1569) * Add nats wrapper for COAP Signed-off-by: 0x6f736f646f * Pass pubsub as argument Signed-off-by: 0x6f736f646f * Defer close connection Signed-off-by: 0x6f736f646f * Defer close connection Signed-off-by: 0x6f736f646f * Rename endpoint to topic Signed-off-by: 0x6f736f646f * MF-1348 - Add transport errors logging (#1544) * MF-1348 - Add go-kit transport level logging Signed-off-by: Manuel Imperiale * Fix reviews Signed-off-by: Manuel Imperiale * Fix reviews Signed-off-by: Manuel Imperiale * Fix merge Signed-off-by: Manuel Imperiale * Fix remark Signed-off-by: Manuel Imperiale * Fix go test flags Signed-off-by: Manuel Imperiale * Use httputil errors in things and http service Signed-off-by: Manuel Imperiale * Fix SDK tests Signed-off-by: Manuel Imperiale * Use httputil errors in certs and provision service Signed-off-by: Manuel Imperiale * Use httputil errors in consumers service Signed-off-by: Manuel Imperiale * General renaming and add ErrMissingToken Signed-off-by: Manuel Imperiale * Rename httputil -> apiutil and use errors in users servive Signed-off-by: Manuel Imperiale * Use apiutil errors in auth, bootstrap, readers, things and twins Signed-off-by: Manuel Imperiale * Replace errors.Contain by comparison Signed-off-by: Manuel Imperiale * Fix remarks Signed-off-by: Manuel Imperiale * Simplify validateID Signed-off-by: Manuel Imperiale * Simplify validateID Signed-off-by: Manuel Imperiale * Simplify and rename ExtractAuthToken -> ExtractBearerToken Signed-off-by: Manuel Imperiale * Fix readers Signed-off-by: Manuel Imperiale * Fix auth key test and remarks Signed-off-by: Manuel Imperiale * Improve comment Signed-off-by: Manuel Imperiale * Simplify validateUUID check Signed-off-by: Manuel Imperiale * Fix typo Signed-off-by: Manuel Imperiale Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * MF-1567 - Use Bearer, Thing or Basic scheme in Authorization header (#1568) Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * MF-1565 - Document Bearer, Thing and Basic Authorization header (#1566) * MF-1565 - Document Bearer Authorization header Signed-off-by: Manuel Imperiale * Fix auth, bootstrap, http and readers openapi Signed-off-by: Manuel Imperiale * Fix openapi Signed-off-by: Manuel Imperiale * Add enc key for bootstrap Signed-off-by: Manuel Imperiale * Fix typo Signed-off-by: Manuel Imperiale * Use global security Signed-off-by: Manuel Imperiale * Fix bearer formats Signed-off-by: Manuel Imperiale * Polish descriptions Signed-off-by: Manuel Imperiale * Fix boostrap and typo Signed-off-by: Manuel Imperiale Co-authored-by: Drasko DRASKOVIC Signed-off-by: 0x6f736f646f * MF-1575 Add 'Name' field to ListMembers response in things svc (#1576) Signed-off-by: Ivan Balboteo Co-authored-by: Ivan Balboteo Signed-off-by: 0x6f736f646f * MF-1580 - Influxdb Writer changes format of update-time to string (#1581) * - MF-1580 - Modified consumers/writers/influxdb/fields.go - influxdb-writer used to update data type of update-time to string - Commented line 12 of consumers/writers/influxdb/fields.go to resolve uneccessary data type conversion issue Signed-off-by: Hasan Tariq * - MF-1580 - Removed strconv package from consumers/writers/influxdb/fields.go since it is no longer needed - Removed line 12 from consumers/writers/influxdb/fields.go - Replaced retrun value of updateTime with msg.UpdateTime (line 16 in fields.go) Signed-off-by: Hasan Tariq * Fix InflxuDB readers Signed-off-by: dusanb94 Co-authored-by: Hasan Tariq Co-authored-by: dusanb94 Signed-off-by: 0x6f736f646f * NOISSUE - Unify MF_INFLUX_READER_DB_HOST and MF_INFLUX_WRITER_DB_HOST envars (#1585) Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * NOISSUE - Fix CoAP adapter (#1572) * Revert "NOISSUE - Add nats wrapper for COAP (#1569)" This reverts commit cc5d5195ab27fa94270ada616487b7053fd9c7bd. Signed-off-by: dusanb94 * Fix CoAP adapter Signed-off-by: dusanb94 * Update CoAP observation cancel Signed-off-by: dusanb94 * Fix observe Signed-off-by: dusanb94 * Fix GET handling Signed-off-by: dusanb94 * Revert authorization Signed-off-by: dusanb94 * Use constants instead of magic numbers Signed-off-by: dusanb94 * Remove an empty line Signed-off-by: dusanb94 * Extract special observe value to constant Signed-off-by: dusanb94 Signed-off-by: 0x6f736f646f * MF-1582 - Fix lora-adapter MQTT client (#1583) * MF-1582 - Fix lora-adapter MQTT clien Signed-off-by: Manuel Imperiale * Add timeout config to the mqtt subscriber Signed-off-by: Manuel Imperiale * Rm comment Signed-off-by: Manuel Imperiale * Add sub timeout Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * NOISSUE - Update changelog and readme for release 0.13.0 (#1592) * Update release example Signed-off-by: dusanb94 * Update changelog and examples for 0.13.0 release Signed-off-by: dusanb94 Signed-off-by: 0x6f736f646f * Update VerneMQ release (#1593) Signed-off-by: dusanb94 Signed-off-by: 0x6f736f646f * NOISSUE - Update changelog for release 0.13.0 (#1595) Signed-off-by: dusanb94 Signed-off-by: 0x6f736f646f * unexport constants Signed-off-by: 0x6f736f646f * Change routingkey Signed-off-by: 0x6f736f646f * Remove wait groups Signed-off-by: 0x6f736f646f * protecting map Signed-off-by: 0x6f736f646f * Add publisher to pubsub Signed-off-by: 0x6f736f646f * Change proto library Signed-off-by: 0x6f736f646f * Fix typos Signed-off-by: 0x6f736f646f * Reduce pubsub tests based on implementation Signed-off-by: 0x6f736f646f * Remove channel cancel Signed-off-by: 0x6f736f646f * Export constant Signed-off-by: 0x6f736f646f * NOISSUE - Move invariant statements out of loop for cassandra-writer (#1596) Signed-off-by: fuzhy Signed-off-by: 0x6f736f646f * Embedding publisher into pubsub Signed-off-by: 0x6f736f646f * Naming publisher Signed-off-by: 0x6f736f646f * NOISSUE - Fix Nginx entrypoint script (#1597) * Fix Nginx entrypoint script Signed-off-by: dusanb94 * Update dependencies Signed-off-by: dusanb94 * Fix NginX entrypoint Signed-off-by: dusanb94 * Revert Makefile changes Signed-off-by: dusanb94 Signed-off-by: 0x6f736f646f * MF-1525 - Add graceful stop for HTTP and GRPC servers (#1548) * Add : errgroup to cmd/auth Signed-off-by: Arvindh * Add : Handle graceful stop for auth service Remove : errgroups from auth service Signed-off-by: Arvindh * Add : Wait till server shutdown Signed-off-by: Arvindh * Change : instead of waitgroup changed to errgroups Signed-off-by: Arvindh * change : KillSignalHandler return type to error Signed-off-by: Arvindh * Empty Commit Signed-off-by: Arvindh * Add : Context to http server shutdown Rename : varaible from proto to protocol Signed-off-by: Arvindh * change : to default log level Signed-off-by: Arvindh * Add : Sign-off Signed-off-by: Arvindh * Add: graceful stop of http and grpc server Signed-off-by: Arvindh * Fix: typos and caps Signed-off-by: Arvindh * Add: Signed-off Signed-off-by: Arvindh * Rename: Func KillSignalHandler to SignalHandler Add: SIGABRT Signed-off-by: Arvindh * Fix: auth service Signed-off-by: Arvindh * Add: timeout for grpc gracefulstop Fix: typos Signed-off-by: Arvindh * Add: .vscode folder to git ignore Signed-off-by: Arvindh * change: variable name to stopWaitTime Signed-off-by: Arvindh * remove: .vscode folder Signed-off-by: Arvindh * remove: .vscode from .gitignore Signed-off-by: Arvindh * Add : logger to handlers Signed-off-by: Arvindh * Add : New line at end of .gitignore file Signed-off-by: Arvindh * Fix : variable naming Add : graceful stop for timescale Signed-off-by: Arvindh * Remove : unsued NATS library from import Signed-off-by: Arvindh * Move: "https" and "https" to moved to const var Signed-off-by: Arvindh * Move: "http" and "https" to moved to const var Signed-off-by: Arvindh * update: branch with master Signed-off-by: Arvindh Co-authored-by: Dušan Borovčanin Co-authored-by: Drasko DRASKOVIC Signed-off-by: 0x6f736f646f * MF-1588 - Update Subscriber interface (#1598) * Initial commit Signed-off-by: 0x6f736f646f * Update subscriber interface Signed-off-by: dusanb94 * Add tests Signed-off-by: 0x6f736f646f * Add tests Signed-off-by: 0x6f736f646f * check subscription map Signed-off-by: 0x6f736f646f * Check topic id after topic Signed-off-by: 0x6f736f646f * reword description Signed-off-by: 0x6f736f646f * Setup empty queue Signed-off-by: 0x6f736f646f * Change mqtt implementation Signed-off-by: 0x6f736f646f * Switch statements Signed-off-by: 0x6f736f646f * Simplify Signed-off-by: 0x6f736f646f * Change mqtt subscriber Signed-off-by: 0x6f736f646f * Protect subscription map Signed-off-by: 0x6f736f646f * Fix subscription Signed-off-by: 0x6f736f646f * Set client id Signed-off-by: 0x6f736f646f * Format Signed-off-by: 0x6f736f646f * Change delete method Signed-off-by: 0x6f736f646f Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * Update rabbitmq subscriber interface Signed-off-by: 0x6f736f646f * using publisher composition Signed-off-by: 0x6f736f646f * Change contenttype Signed-off-by: 0x6f736f646f * rename topic for publish and subscribe Signed-off-by: 0x6f736f646f * Change errors to lower case Signed-off-by: 0x6f736f646f * Change errors to lower case Signed-off-by: 0x6f736f646f * export errors Signed-off-by: 0x6f736f646f * MF - 1590 - Fix fetching list of users with a zero limit (#1594) * Add max and min limit size Signed-off-by: 0x6f736f646f * Format Signed-off-by: 0x6f736f646f * Format Signed-off-by: 0x6f736f646f Co-authored-by: Dušan Borovčanin Signed-off-by: 0x6f736f646f * NOISSUE - Retrieve client key on cert issuing (#1607) Signed-off-by: Manuel Imperiale Signed-off-by: 0x6f736f646f * fix bug (#1604) Signed-off-by: zhangchuanfeng <654300242@qq.com> Signed-off-by: 0x6f736f646f * queue per subscription Signed-off-by: 0x6f736f646f * queue per subscription Signed-off-by: 0x6f736f646f * Change routing method Signed-off-by: 0x6f736f646f * Direct method with one exchange to many queues, one consumer per queue Signed-off-by: 0x6f736f646f * :recycle: Not casting data Signed-off-by: 0x6f736f646f * :pencil2: Fix typo Signed-off-by: 0x6f736f646f * :recycle: remove passed queue name Signed-off-by: 0x6f736f646f * :fire: removing echange kind Signed-off-by: 0x6f736f646f * Combine tests Signed-off-by: 0x6f736f646f * Refactor unsubscribe method Signed-off-by: 0x6f736f646f * Fix merge conflict Signed-off-by: 0x6f736f646f * :white_check_mark: sub and unsub to dummy topic Signed-off-by: 0x6f736f646f * generate client id from topic and ID Signed-off-by: 0x6f736f646f * Rename topicID to clientID Signed-off-by: 0x6f736f646f * update tests Signed-off-by: 0x6f736f646f * Reuse clientID Signed-off-by: 0x6f736f646f * Fix typos Signed-off-by: 0x6f736f646f * Seperate testpublish and testpubsub Signed-off-by: 0x6f736f646f Co-authored-by: Manuel Imperiale Co-authored-by: Dušan Borovčanin Co-authored-by: Ivan Milošević Co-authored-by: __touk__ Co-authored-by: Mirko Teodorovic Co-authored-by: Drasko DRASKOVIC Co-authored-by: stefankovacevic123 Co-authored-by: ibalboteo Co-authored-by: Ivan Balboteo Co-authored-by: Hasan98-git <67228396+Hasan98-git@users.noreply.github.com> Co-authored-by: Hasan Tariq Co-authored-by: fuzhy Co-authored-by: Arvindh <30824765+arvindh123@users.noreply.github.com> Co-authored-by: 张传峰 <59160162+zhang-chuanfeng@users.noreply.github.com> Signed-off-by: Arvindh * NOISSUE - Fix Groups SDK (#1609) * Fix Groups SDK Signed-off-by: dusanb94 * Fix CLI Signed-off-by: dusanb94 Signed-off-by: Arvindh * NOISSUE - Fix CI script (#1613) * Fix CI script Signed-off-by: dusanb94 * Fix linter errors Signed-off-by: dusanb94 * Add timeout to linter Signed-off-by: dusanb94 Signed-off-by: Arvindh * NOISSUE - Make application/json content-type valid in http-adapter (#1606) * NOISSUE - Make application/json content-type valid in http-adapter Signed-off-by: Manuel Imperiale * Add test Signed-off-by: Manuel Imperiale * Add CBOR content-type Signed-off-by: Manuel Imperiale * Fix naming Signed-off-by: Manuel Imperiale * Fix naming Signed-off-by: Manuel Imperiale * Fix CI Signed-off-by: Manuel Imperiale * Fix CI flag Signed-off-by: Manuel Imperiale * Fix CI install Signed-off-by: Manuel Imperiale * Upgrade grpc version Signed-off-by: Manuel Imperiale * Fix typo Signed-off-by: Manuel Imperiale * rm cli Signed-off-by: Manuel Imperiale Co-authored-by: Dušan Borovčanin Signed-off-by: Arvindh * Specify size of channel Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh * NOISSUE - fix pull request template typo (#1616) * Fix typo Signed-off-by: Filip Bugarski * Change link Signed-off-by: fbugarski Signed-off-by: Arvindh * Add: load configuration function Signed-off-by: Arvindh * change: load config from env with pkg caarlos0/env Signed-off-by: Arvindh * change: mfdatabase to internaldb Signed-off-by: Arvindh * Add: httpserver and grpcsever Signed-off-by: Arvindh move http and grpc server functions Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Move Keto and Jaeger Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Add metrics and auth Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Rename service name Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Change metrics method Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Rename package name Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :truck: Rename Keto and Jaeger functions Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh unify grpc service Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :truck: rename apiutil to initutil Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :sparkles: coap server Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :truck: rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :truck: Rename Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :recycle: rename packages Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :recycle: remove mf prefix Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh :truck: rename server error Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh remove dead code Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh NOISSUE - Fix CI script (#1613) * Fix CI script Signed-off-by: dusanb94 * Fix linter errors Signed-off-by: dusanb94 * Add timeout to linter Signed-off-by: dusanb94 Signed-off-by: Arvindh Specify size of channel Signed-off-by: 0x6f736f646f Signed-off-by: Arvindh Add: load configuration function Signed-off-by: Arvindh change: load config from env with pkg caarlos0/env Signed-off-by: Arvindh * change: mfdatabase to internaldb Signed-off-by: Arvindh * fix: merge resolve error Signed-off-by: Arvindh * fix: merge resolve error Signed-off-by: Arvindh * fix: merge resolve error Signed-off-by: Arvindh * remove: unused variables Signed-off-by: Arvindh * add: address variable in servers Signed-off-by: Arvindh * move: postgres connect to internal Signed-off-by: Arvindh * add: client wrapper for most used Signed-off-by: Arvindh * add: client wrapper for env Signed-off-by: Arvindh * unify : auth, bootstrap, cassandra reader Signed-off-by: Arvindh * unify : bootstrap code Signed-off-by: Arvindh * unify : cassandra writer Signed-off-by: Arvindh * fix: struct tag to envDefault Signed-off-by: Arvindh * fix: grpc prefix Signed-off-by: Arvindh * fix: env parser Signed-off-by: Arvindh * fix: alt prefix Signed-off-by: Arvindh * fix: env default tag Signed-off-by: Arvindh * fix: auth grpc config Signed-off-by: Arvindh * changes: internal approch and service start Signed-off-by: Arvindh * unify: http adapter service Signed-off-by: Arvindh * remove: unused code in http adapter" Signed-off-by: Arvindh * fix: config environment variable tags Signed-off-by: Arvindh * unify: timescale writer Signed-off-by: Arvindh * unify: timescale reader Signed-off-by: Arvindh * unify: thing main.go Signed-off-by: Arvindh * unify: smtp-notifier Signed-off-by: Arvindh * unify: smpp-notifier Signed-off-by: Arvindh * unify: postgres reader and writer Signed-off-by: Arvindh * unify: twins Signed-off-by: Arvindh * unify Signed-off-by: Arvindh * unify certs main.go Signed-off-by: Arvindh * unify certs main.go Signed-off-by: Arvindh * unify coap main.go Signed-off-by: Arvindh * unify lora main.go Signed-off-by: Arvindh * fix fatalf Signed-off-by: Arvindh * unify mqtt main.go Signed-off-by: Arvindh * unify mqtt main.go Signed-off-by: Arvindh * unify ocpua adapter main.go Signed-off-by: Arvindh * fix case Signed-off-by: Arvindh * unify ws_adapter Signed-off-by: Arvindh * unify ws_adapter Signed-off-by: Arvindh * unify ws_adapter Signed-off-by: Arvindh * add : comment and spacing Signed-off-by: Arvindh * fix: lint errors Signed-off-by: Arvindh * fix: lint errors Signed-off-by: Arvindh * fix: main.go config load Signed-off-by: Arvindh * fix: main.go config load Signed-off-by: Arvindh * fix: auth main.go keto config Signed-off-by: Arvindh * remove: package internal/sqlxt Signed-off-by: Arvindh * code format : internal/client/grpc/connect.go Signed-off-by: Arvindh * fix: inline code Signed-off-by: Arvindh * fix: code format Signed-off-by: Arvindh * fix: inline and code format Signed-off-by: Arvindh * fix: moved to single block Signed-off-by: Arvindh * fix: moved to single block Signed-off-by: Arvindh * fix: export function comments Signed-off-by: Arvindh * fix: export function comments Signed-off-by: Arvindh * fix: export function comments Signed-off-by: Arvindh * fix: export function comments Signed-off-by: Arvindh * fix: export function comments Signed-off-by: Arvindh * remane: newtracer.go to tracer.go Signed-off-by: Arvindh * renamee: authClient.go and thingsClient.go to client.go Signed-off-by: Arvindh * remove space Signed-off-by: Arvindh * add: jaeger default value Signed-off-by: Arvindh * fix: cassander config default values Signed-off-by: Arvindh * rename file Signed-off-by: Arvindh * fix: postgres client config default values Signed-off-by: Arvindh * add setup with default config Signed-off-by: Arvindh * fix: mongo client config default values Signed-off-by: Arvindh * add: postgres default db name in services Signed-off-by: Arvindh * fix: environment variable default for auth Signed-off-by: Arvindh * fix: environment variable default for bootstrap Signed-off-by: Arvindh * fix: environment variable default for cassandra-reader Signed-off-by: Arvindh * fix: environment variable default for cassandra-writer Signed-off-by: Arvindh * fix: environment variable default for certs Signed-off-by: Arvindh * fix: environment variable default for coap Signed-off-by: Arvindh * fix: environment variable default for http-adapter Signed-off-by: Arvindh * fix: environment variable default for influx-reader Signed-off-by: Arvindh * fix: environment variable default for influx-writer Signed-off-by: Arvindh * fix: environment variable default for lora Signed-off-by: Arvindh * fix: environment variable default for mongodb-reader Signed-off-by: Arvindh * fix: environment variable default for mongodb-writer Signed-off-by: Arvindh * fix: environment variable default for mqtt Signed-off-by: Arvindh * fix: environment variable default for opcua Signed-off-by: Arvindh * fix: environment variable default for postgres-reader Signed-off-by: Arvindh * fix: environment variable default for postgres-writer Signed-off-by: Arvindh * fix: environment variable default for smpp-notifier Signed-off-by: Arvindh * fix: environment variable default for smtp-notifier Signed-off-by: Arvindh * fix: environment variable default for things Signed-off-by: Arvindh * fix: environment variable default for timescale-reader Signed-off-by: Arvindh * fix: environment variable default for timescale-writer Signed-off-by: Arvindh * fix: environment variable default for twins Signed-off-by: Arvindh * fix: environment variable default for users Signed-off-by: Arvindh * fix: environment variable default for ws Signed-off-by: Arvindh * fix: unused variables Signed-off-by: Arvindh * empty commit Signed-off-by: Arvindh * add comments Signed-off-by: Arvindh * fix: redis env variables Signed-off-by: Arvindh * fix: adapter ports and postgres db name Signed-off-by: Arvindh * fix: adapter ports Signed-off-by: Arvindh * comments aligned Signed-off-by: Arvindh * rename cassandra session variable Signed-off-by: Arvindh * rename influxdb and influx to influxDB Signed-off-by: Arvindh * rename EsConsumername to ESConsumerName Signed-off-by: Arvindh * made comments consistant Signed-off-by: Arvindh * made comments consistant & remove empty lines Signed-off-by: Arvindh * made comments consistant & renmae function Signed-off-by: Arvindh * made comments Signed-off-by: Arvindh * comments added Signed-off-by: Arvindh * fix bootstrap Signed-off-by: Arvindh * fix empty env var Signed-off-by: Arvindh * remove : unused variable Signed-off-by: Arvindh * update: env parser library Signed-off-by: Arvindh * fix: mongodb reader and writer Signed-off-by: Arvindh * fix: cassandra reader and writer Signed-off-by: Arvindh * rename: directory Signed-off-by: Arvindh * rename: variable Signed-off-by: Arvindh * remove: unused librar Signed-off-by: Arvindh * Format code and remove unused comments Signed-off-by: dusanb94 * Fix tests Signed-off-by: dusanb94 * Move test URL construction out of the loop Signed-off-by: dusanb94 * remove end dot in single line comments Signed-off-by: Arvindh * empty Signed-off-by: Arvindh --------- Signed-off-by: Arvindh Signed-off-by: 0x6f736f646f Signed-off-by: Manuel Imperiale Signed-off-by: zhangchuanfeng <654300242@qq.com> Signed-off-by: dusanb94 Signed-off-by: fbugarski Co-authored-by: Dušan Borovčanin Co-authored-by: Drasko DRASKOVIC Co-authored-by: b1ackd0t Co-authored-by: Manuel Imperiale Co-authored-by: 张传峰 <59160162+zhang-chuanfeng@users.noreply.github.com> Co-authored-by: Ivan Milošević Co-authored-by: __touk__ Co-authored-by: Mirko Teodorovic Co-authored-by: stefankovacevic123 Co-authored-by: ibalboteo Co-authored-by: Ivan Balboteo Co-authored-by: Hasan98-git <67228396+Hasan98-git@users.noreply.github.com> Co-authored-by: Hasan Tariq Co-authored-by: fuzhy Co-authored-by: Filip Bugarski --- auth/README.md | 2 +- auth/postgres/init.go | 51 +- auth/postgres/setup_test.go | 8 +- bootstrap/README.md | 5 +- bootstrap/postgres/init.go | 48 +- bootstrap/postgres/setup_test.go | 8 +- certs/postgres/certs.go | 2 +- certs/postgres/init.go | 57 +- certs/postgres/setup_test.go | 5 +- certs/service.go | 30 +- certs/service_test.go | 27 +- cmd/auth/main.go | 352 +++--------- cmd/bootstrap/main.go | 380 +++---------- cmd/cassandra-reader/main.go | 313 ++--------- cmd/cassandra-writer/main.go | 174 ++---- cmd/certs/main.go | 375 +++---------- cmd/coap/main.go | 236 ++------ cmd/http/main.go | 221 ++------ cmd/influxdb-reader/main.go | 306 ++--------- cmd/influxdb-writer/main.go | 165 ++---- cmd/lora/main.go | 222 +++----- cmd/mongodb-reader/main.go | 294 ++-------- cmd/mongodb-writer/main.go | 150 ++---- cmd/mqtt/main.go | 288 +++------- cmd/opcua/main.go | 200 ++----- cmd/postgres-reader/main.go | 300 ++--------- cmd/postgres-writer/main.go | 170 ++---- cmd/provision/main.go | 51 +- cmd/smpp-notifier/main.go | 365 +++---------- cmd/smtp-notifier/main.go | 347 +++--------- cmd/things/main.go | 449 ++++------------ cmd/timescale-reader/main.go | 289 ++-------- cmd/timescale-writer/main.go | 172 ++---- cmd/twins/main.go | 324 +++-------- cmd/users/main.go | 381 +++---------- cmd/ws/main.go | 211 ++------ coap/README.md | 2 +- consumers/notifiers/postgres/init.go | 46 +- consumers/notifiers/postgres/setup_test.go | 6 +- consumers/notifiers/smpp/README.md | 2 +- consumers/notifiers/smpp/config.go | 16 +- consumers/notifiers/smtp/README.md | 2 +- consumers/writers/cassandra/README.md | 2 +- consumers/writers/cassandra/consumer_test.go | 7 +- consumers/writers/cassandra/init.go | 37 +- consumers/writers/cassandra/setup_test.go | 4 +- consumers/writers/influxdb/README.md | 2 +- consumers/writers/mongodb/README.md | 2 +- consumers/writers/postgres/README.md | 4 +- consumers/writers/postgres/init.go | 47 +- consumers/writers/postgres/setup_test.go | 6 +- consumers/writers/timescale/README.md | 4 +- consumers/writers/timescale/init.go | 47 +- consumers/writers/timescale/setup_test.go | 6 +- docker/.env | 2 +- go.mod | 1 + go.sum | 2 + http/README.md | 2 +- internal/clients/cassandra/cassandra.go | 72 +++ internal/clients/grpc/auth/client.go | 28 + internal/clients/grpc/connect.go | 119 ++++ internal/clients/grpc/things/client.go | 28 + internal/clients/influxdb/influxdb.go | 53 ++ internal/clients/jaeger/tracer.go | 41 ++ internal/clients/mongo/mongo.go | 51 ++ internal/clients/postgres/postgres.go | 86 +++ internal/clients/redis/redis.go | 51 ++ internal/close.go | 12 + internal/email/email.go | 16 +- internal/env/load.go | 10 + internal/env/parser.go | 117 ++++ internal/metrics.go | 26 + internal/server/coap/coap.go | 66 +++ internal/server/grpc/grpc.go | 94 ++++ internal/server/http/http.go | 76 +++ internal/server/server.go | 66 +++ lora/README.md | 2 +- mqtt/README.md | 2 +- opcua/README.md | 2 +- opcua/adapter.go | 10 +- opcua/db/subs.go | 2 +- readers/cassandra/README.md | 1 + readers/cassandra/init.go | 31 -- readers/cassandra/messages_test.go | 8 +- readers/cassandra/setup_test.go | 4 +- readers/influxdb/README.md | 1 + readers/mongodb/README.md | 1 + readers/postgres/README.md | 2 +- readers/postgres/init.go | 1 - readers/postgres/messages.go | 2 +- readers/postgres/setup_test.go | 2 +- readers/timescale/README.md | 4 +- readers/timescale/init.go | 1 - readers/timescale/setup_test.go | 2 +- things/README.md | 2 +- things/postgres/init.go | 46 +- things/postgres/setup_test.go | 6 +- twins/README.md | 4 +- users/README.md | 2 +- users/postgres/init.go | 48 +- users/postgres/setup_test.go | 6 +- vendor/github.com/caarlos0/env/v7/.gitignore | 4 + .../github.com/caarlos0/env/v7/.golangci.yml | 8 + .../caarlos0/env/v7/.goreleaser.yml | 3 + vendor/github.com/caarlos0/env/v7/LICENSE.md | 21 + vendor/github.com/caarlos0/env/v7/Makefile | 37 ++ vendor/github.com/caarlos0/env/v7/README.md | 454 ++++++++++++++++ vendor/github.com/caarlos0/env/v7/env.go | 506 ++++++++++++++++++ vendor/github.com/caarlos0/env/v7/env_unix.go | 15 + .../github.com/caarlos0/env/v7/env_windows.go | 25 + vendor/modules.txt | 3 + ws/README.md | 2 +- 112 files changed, 3604 insertions(+), 5905 deletions(-) create mode 100644 internal/clients/cassandra/cassandra.go create mode 100644 internal/clients/grpc/auth/client.go create mode 100644 internal/clients/grpc/connect.go create mode 100644 internal/clients/grpc/things/client.go create mode 100644 internal/clients/influxdb/influxdb.go create mode 100644 internal/clients/jaeger/tracer.go create mode 100644 internal/clients/mongo/mongo.go create mode 100644 internal/clients/postgres/postgres.go create mode 100644 internal/clients/redis/redis.go create mode 100644 internal/close.go create mode 100644 internal/env/load.go create mode 100644 internal/env/parser.go create mode 100644 internal/metrics.go create mode 100644 internal/server/coap/coap.go create mode 100644 internal/server/grpc/grpc.go create mode 100644 internal/server/http/http.go create mode 100644 internal/server/server.go delete mode 100644 readers/cassandra/init.go create mode 100644 vendor/github.com/caarlos0/env/v7/.gitignore create mode 100644 vendor/github.com/caarlos0/env/v7/.golangci.yml create mode 100644 vendor/github.com/caarlos0/env/v7/.goreleaser.yml create mode 100644 vendor/github.com/caarlos0/env/v7/LICENSE.md create mode 100644 vendor/github.com/caarlos0/env/v7/Makefile create mode 100644 vendor/github.com/caarlos0/env/v7/README.md create mode 100644 vendor/github.com/caarlos0/env/v7/env.go create mode 100644 vendor/github.com/caarlos0/env/v7/env_unix.go create mode 100644 vendor/github.com/caarlos0/env/v7/env_windows.go diff --git a/auth/README.md b/auth/README.md index e7d70ab2f7..61bfe52ce6 100644 --- a/auth/README.md +++ b/auth/README.md @@ -57,7 +57,7 @@ default values. | Variable | Description | Default | |-------------------------------|--------------------------------------------------------------------------|----------------| -| MF_AUTH_LOG_LEVEL | Service level (debug, info, warn, error) | error | +| MF_AUTH_LOG_LEVEL | Service level (debug, info, warn, error) | info | | MF_AUTH_DB_HOST | Database host address | localhost | | MF_AUTH_DB_PORT | Database host port | 5432 | | MF_AUTH_DB_USER | Database user | mainflux | diff --git a/auth/postgres/init.go b/auth/postgres/init.go index 3afd0c8fe9..fc6992f536 100644 --- a/auth/postgres/init.go +++ b/auth/postgres/init.go @@ -3,45 +3,11 @@ package postgres -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +// Migration of Auth service +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "auth_1", @@ -56,9 +22,9 @@ func migrateDB(db *sqlx.DB) error { PRIMARY KEY (id, issuer_id) )`, `CREATE EXTENSION IF NOT EXISTS LTREE`, - `CREATE TABLE IF NOT EXISTS groups ( + `CREATE TABLE IF NOT EXISTS groups ( id VARCHAR(254) UNIQUE NOT NULL, - parent_id VARCHAR(254), + parent_id VARCHAR(254), owner_id VARCHAR(254), name VARCHAR(254) NOT NULL, description VARCHAR(1024), @@ -80,7 +46,7 @@ func migrateDB(db *sqlx.DB) error { )`, `CREATE INDEX path_gist_idx ON groups USING GIST (path);`, `CREATE OR REPLACE FUNCTION inherit_group() - RETURNS trigger + RETURNS trigger LANGUAGE PLPGSQL AS $$ @@ -112,7 +78,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/auth/postgres/setup_test.go b/auth/postgres/setup_test.go index f7b60dcfce..d76bc174b7 100644 --- a/auth/postgres/setup_test.go +++ b/auth/postgres/setup_test.go @@ -12,9 +12,9 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" - "github.com/mainflux/mainflux/auth/postgres" + authRepo "github.com/mainflux/mainflux/auth/postgres" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" dockertest "github.com/ory/dockertest/v3" ) @@ -49,7 +49,7 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to docker: %s", err) } - dbConfig := postgres.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -61,7 +61,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - if db, err = postgres.Connect(dbConfig); err != nil { + if db, err = pgClient.SetupDB(dbConfig, *authRepo.Migration()); err != nil { log.Fatalf("Could not setup test DB connection: %s", err) } diff --git a/bootstrap/README.md b/bootstrap/README.md index e5ca3a84b8..495551ca62 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -37,7 +37,7 @@ The service is configured using the environment variables presented in the follo | Variable | Description | Default | |-------------------------------|-------------------------------------------------------------------------|----------------------------------| -| MF_BOOTSTRAP_LOG_LEVEL | Log level for Bootstrap (debug, info, warn, error) | error | +| MF_BOOTSTRAP_LOG_LEVEL | Log level for Bootstrap (debug, info, warn, error) | info | | MF_BOOTSTRAP_DB_HOST | Database host address | localhost | | MF_BOOTSTRAP_DB_PORT | Database host port | 5432 | | MF_BOOTSTRAP_DB_USER | Database user | mainflux | @@ -53,8 +53,7 @@ The service is configured using the environment variables presented in the follo | MF_BOOTSTRAP_PORT | Bootstrap service HTTP port | 8180 | | MF_BOOTSTRAP_SERVER_CERT | Path to server certificate in pem format | | | MF_BOOTSTRAP_SERVER_KEY | Path to server key in pem format | | -| MF_SDK_BASE_URL | Base url for Mainflux SDK | http://localhost | -| MF_SDK_THINGS_PREFIX | SDK prefix for Things service | | +| MF_THINGS_URL | Base url for Mainflux Things | http://localhost | | MF_THINGS_ES_URL | Things service event source URL | localhost:6379 | | MF_THINGS_ES_PASS | Things service event source password | | | MF_THINGS_ES_DB | Things service event source database | 0 | diff --git a/bootstrap/postgres/init.go b/bootstrap/postgres/init.go index 3e1647f4ae..aa02b619e2 100644 --- a/bootstrap/postgres/init.go +++ b/bootstrap/postgres/init.go @@ -3,47 +3,11 @@ package postgres -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +// Migration of bootstrap service +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "configs_1", @@ -102,8 +66,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - - return err } diff --git a/bootstrap/postgres/setup_test.go b/bootstrap/postgres/setup_test.go index 7bf55524e2..44be5c4d16 100644 --- a/bootstrap/postgres/setup_test.go +++ b/bootstrap/postgres/setup_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" - "github.com/mainflux/mainflux/bootstrap/postgres" + bootstrapRepo "github.com/mainflux/mainflux/bootstrap/postgres" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/logger" dockertest "github.com/ory/dockertest/v3" @@ -50,7 +50,7 @@ func TestMain(m *testing.M) { testLog.Error(fmt.Sprintf("Could not connect to docker: %s", err)) } - dbConfig := postgres.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -62,7 +62,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - if db, err = postgres.Connect(dbConfig); err != nil { + if db, err = pgClient.SetupDB(dbConfig, *bootstrapRepo.Migration()); err != nil { testLog.Error(fmt.Sprintf("Could not setup test DB connection: %s", err)) } diff --git a/certs/postgres/certs.go b/certs/postgres/certs.go index c85093261e..d91bfd6532 100644 --- a/certs/postgres/certs.go +++ b/certs/postgres/certs.go @@ -11,7 +11,7 @@ import ( "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5/pgconn" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access + "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux/certs" "github.com/mainflux/mainflux/logger" diff --git a/certs/postgres/init.go b/certs/postgres/init.go index fbd728a2bd..04aa8c0306 100644 --- a/certs/postgres/init.go +++ b/certs/postgres/init.go @@ -5,57 +5,11 @@ package postgres -import ( - "errors" - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -const primaryKey = "primary_key" - -// ErrMigrate indicates error during database migrations. -var ErrMigrate = errors.New("error executing database migrations") - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - mErr, ok := err.(*migrate.TxError) - if ok && mErr.Migration.Id == primaryKey { - return db, ErrMigrate - } - return nil, err - } - - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +// Migration of Certs service +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "certs_1", @@ -74,7 +28,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/certs/postgres/setup_test.go b/certs/postgres/setup_test.go index 242072aa46..475d90e167 100644 --- a/certs/postgres/setup_test.go +++ b/certs/postgres/setup_test.go @@ -10,6 +10,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux/certs/postgres" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/logger" dockertest "github.com/ory/dockertest/v3" ) @@ -49,7 +50,7 @@ func TestMain(m *testing.M) { testLog.Error(fmt.Sprintf("Could not connect to docker: %s", err)) } - dbConfig := postgres.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -61,7 +62,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - if db, err = postgres.Connect(dbConfig); err != nil { + if db, err = pgClient.SetupDB(dbConfig, *postgres.Migration()); err != nil { testLog.Error(fmt.Sprintf("Could not setup test DB connection: %s", err)) } diff --git a/certs/service.go b/certs/service.go index 42ed5f7c76..deabf727fc 100644 --- a/certs/service.go +++ b/certs/service.go @@ -5,8 +5,6 @@ package certs import ( "context" - "crypto/tls" - "crypto/x509" "time" "github.com/mainflux/mainflux" @@ -46,43 +44,19 @@ type Service interface { RevokeCert(ctx context.Context, token, serialID string) (Revoke, error) } -// Config defines the service parameters -type Config struct { - LogLevel string - ClientTLS bool - CaCerts string - HTTPPort string - ServerCert string - ServerKey string - CertsURL string - JaegerURL string - AuthURL string - AuthTimeout time.Duration - SignTLSCert tls.Certificate - SignX509Cert *x509.Certificate - SignRSABits int - SignHoursValid string - PKIHost string - PKIPath string - PKIRole string - PKIToken string -} - type certsService struct { auth mainflux.AuthServiceClient certsRepo Repository sdk mfsdk.SDK - conf Config pki pki.Agent } -// New returns new Certs service. -func New(auth mainflux.AuthServiceClient, certs Repository, sdk mfsdk.SDK, config Config, pki pki.Agent) Service { +// New returns new Certs service +func New(auth mainflux.AuthServiceClient, certs Repository, sdk mfsdk.SDK, pki pki.Agent) Service { return &certsService{ certsRepo: certs, sdk: sdk, auth: auth, - conf: config, pki: pki, } } diff --git a/certs/service_test.go b/certs/service_test.go index a2dfb7cdf9..ab564729c8 100644 --- a/certs/service_test.go +++ b/certs/service_test.go @@ -9,7 +9,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io/ioutil" "net/http/httptest" "os" "strconv" @@ -42,19 +41,11 @@ const ( ttl = "1h" certNum = 10 - cfgLogLevel = "error" - cfgClientTLS = false - cfgServerCert = "" - cfgServerKey = "" - cfgCertsURL = "http://localhost" - cfgJaegerURL = "" - cfgAuthURL = "localhost:8181" cfgAuthTimeout = "1s" caPath = "../docker/ssl/certs/ca.crt" caKeyPath = "../docker/ssl/certs/ca.key" cfgSignHoursValid = "24h" - cfgSignRSABits = 2048 ) func newService(tokens map[string]string) (certs.Service, error) { @@ -80,23 +71,9 @@ func newService(tokens map[string]string) (certs.Service, error) { return nil, err } - c := certs.Config{ - LogLevel: cfgLogLevel, - ClientTLS: cfgClientTLS, - ServerCert: cfgServerCert, - ServerKey: cfgServerKey, - CertsURL: cfgCertsURL, - JaegerURL: cfgJaegerURL, - AuthURL: cfgAuthURL, - SignTLSCert: tlsCert, - SignX509Cert: caCert, - SignHoursValid: cfgSignHoursValid, - SignRSABits: cfgSignRSABits, - } - pki := mocks.NewPkiAgent(tlsCert, caCert, cfgSignHoursValid, authTimeout) - return certs.New(auth, repo, sdk, c, pki), nil + return certs.New(auth, repo, sdk, pki), nil } func newThingsService(auth mainflux.AuthServiceClient) things.Service { @@ -413,7 +390,7 @@ func loadCertificates(caPath, caKeyPath string) (tls.Certificate, *x509.Certific return tlsCert, caCert, errors.Wrap(err, err) } - b, err := ioutil.ReadFile(caPath) + b, err := os.ReadFile(caPath) if err != nil { return tlsCert, caCert, err } diff --git a/cmd/auth/main.go b/cmd/auth/main.go index 9c1d64ea5e..7b3cf177a4 100644 --- a/cmd/auth/main.go +++ b/cmd/auth/main.go @@ -3,15 +3,10 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net" - "net/http" "os" "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/auth" @@ -20,217 +15,137 @@ import ( httpapi "github.com/mainflux/mainflux/auth/api/http" "github.com/mainflux/mainflux/auth/jwt" "github.com/mainflux/mainflux/auth/keto" - "github.com/mainflux/mainflux/auth/postgres" + authPg "github.com/mainflux/mainflux/auth/postgres" "github.com/mainflux/mainflux/auth/tracing" + "github.com/mainflux/mainflux/internal" + grpcClient "github.com/mainflux/mainflux/internal/clients/grpc" + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + grpcserver "github.com/mainflux/mainflux/internal/server/grpc" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/uuid" "github.com/opentracing/opentracing-go" acl "github.com/ory/keto/proto/ory/keto/acl/v1alpha1" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - stopWaitTime = 5 * time.Second - httpProtocol = "http" - httpsProtocol = "https" - - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "auth" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defHTTPPort = "8180" - defGRPCPort = "8181" - defSecret = "auth" - defServerCert = "" - defServerKey = "" - defJaegerURL = "" - defKetoReadHost = "mainflux-keto" - defKetoWriteHost = "mainflux-keto" - defKetoReadPort = "4466" - defKetoWritePort = "4467" - defLoginDuration = "10h" - - envLogLevel = "MF_AUTH_LOG_LEVEL" - envDBHost = "MF_AUTH_DB_HOST" - envDBPort = "MF_AUTH_DB_PORT" - envDBUser = "MF_AUTH_DB_USER" - envDBPass = "MF_AUTH_DB_PASS" - envDB = "MF_AUTH_DB" - envDBSSLMode = "MF_AUTH_DB_SSL_MODE" - envDBSSLCert = "MF_AUTH_DB_SSL_CERT" - envDBSSLKey = "MF_AUTH_DB_SSL_KEY" - envDBSSLRootCert = "MF_AUTH_DB_SSL_ROOT_CERT" - envHTTPPort = "MF_AUTH_HTTP_PORT" - envGRPCPort = "MF_AUTH_GRPC_PORT" - envSecret = "MF_AUTH_SECRET" - envServerCert = "MF_AUTH_SERVER_CERT" - envServerKey = "MF_AUTH_SERVER_KEY" - envJaegerURL = "MF_JAEGER_URL" - envKetoReadHost = "MF_KETO_READ_REMOTE_HOST" - envKetoWriteHost = "MF_KETO_WRITE_REMOTE_HOST" - envKetoReadPort = "MF_KETO_READ_REMOTE_PORT" - envKetoWritePort = "MF_KETO_WRITE_REMOTE_PORT" - envLoginDuration = "MF_AUTH_LOGIN_TOKEN_DURATION" + svcName = "auth" + envPrefix = "MF_AUTH_" + envPrefixHttp = "MF_AUTH_HTTP_" + envPrefixGrpc = "MF_AUTH_GRPC_" + defDB = "auth" + defSvcHttpPort = "8180" + defSvcGrpcPort = "8181" ) type config struct { - logLevel string - dbConfig postgres.Config - httpPort string - grpcPort string - secret string - serverCert string - serverKey string - jaegerURL string - ketoReadHost string - ketoWriteHost string - ketoWritePort string - ketoReadPort string - loginDuration time.Duration + LogLevel string `env:"MF_AUTH_LOG_LEVEL" envDefault:"info"` + Secret string `env:"MF_AUTH_SECRET" envDefault:"auth"` + KetoReadHost string `env:"MF_KETO_READ_REMOTE_HOST" envDefault:"mainflux-keto"` + KetoReadPort string `env:"MF_KETO_READ_REMOTE_PORT" envDefault:"4466"` + KetoWriteHost string `env:"MF_KETO_WRITE_REMOTE_HOST" envDefault:"mainflux-keto"` + KetoWritePort string `env:"MF_KETO_WRITE_REMOTE_PORT" envDefault:"4467"` + LoginDuration time.Duration `env:"MF_AUTH_LOGIN_TOKEN_DURATION" envDefault:"10h"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + + // Create auth service configurations + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - db := connectToDB(cfg.dbConfig, logger) + // Create new postgres client + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *authPg.Migration(), dbConfig) + if err != nil { + log.Fatalf("failed to setup postgres database : %s", err.Error()) + } defer db.Close() - tracer, closer := initJaeger("auth", cfg.jaegerURL, logger) - defer closer.Close() - - dbTracer, dbCloser := initJaeger("auth_db", cfg.jaegerURL, logger) - defer dbCloser.Close() - - readerConn, writerConn := initKeto(cfg.ketoReadHost, cfg.ketoReadPort, cfg.ketoWriteHost, cfg.ketoWritePort, logger) - - svc := newService(db, dbTracer, cfg.secret, logger, readerConn, writerConn, cfg.loginDuration) - - g.Go(func() error { - return startHTTPServer(ctx, tracer, svc, cfg.httpPort, cfg.serverCert, cfg.serverKey, logger) - }) - g.Go(func() error { - return startGRPCServer(ctx, tracer, svc, cfg.grpcPort, cfg.serverCert, cfg.serverKey, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Authentication service shutdown by signal: %s", sig)) - } - return nil - }) - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Authentication service terminated: %s", err)) + // Create new tracer for database + dbTracer, dbCloser, err := jaegerClient.NewTracer("auth_db", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } -} + defer dbCloser.Close() -func loadConfig() config { - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), + // Create new keto reader grpc client + readerConn, _, err := grpcClient.Connect(grpcClient.Config{ClientTLS: false, URL: fmt.Sprintf("%s:%s", cfg.KetoReadHost, cfg.KetoReadPort)}) + if err != nil { + log.Fatalf("failed to connect to keto gRPC: %s", err.Error()) } - loginDuration, err := time.ParseDuration(mainflux.Env(envLoginDuration, defLoginDuration)) + // Create new keto writer grpc client + writerConn, _, err := grpcClient.Connect(grpcClient.Config{ClientTLS: false, URL: fmt.Sprintf("%s:%s", cfg.KetoWriteHost, cfg.KetoWritePort)}) if err != nil { - log.Fatal(err) + log.Fatalf("failed to connect to keto gRPC: %s", err.Error()) } - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - dbConfig: dbConfig, - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - grpcPort: mainflux.Env(envGRPCPort, defGRPCPort), - secret: mainflux.Env(envSecret, defSecret), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - ketoReadHost: mainflux.Env(envKetoReadHost, defKetoReadHost), - ketoWriteHost: mainflux.Env(envKetoWriteHost, defKetoWriteHost), - ketoReadPort: mainflux.Env(envKetoReadPort, defKetoReadPort), - ketoWritePort: mainflux.Env(envKetoWritePort, defKetoWritePort), - loginDuration: loginDuration, + svc := newService(db, dbTracer, cfg.Secret, logger, readerConn, writerConn, cfg.LoginDuration) + + // Create new HTTP Server + tracer, closer, err := jaegerClient.NewTracer("auth", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer closer.Close() -} + httpServerConfig := server.Config{Port: defSvcHttpPort} -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err)) - os.Exit(1) - } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, httpapi.MakeHandler(svc, tracer, logger), logger) - return tracer, closer -} + // Create new grpc server + grpcServerConfig := server.Config{Port: defSvcGrpcPort} -func initKeto(hostReadAddress, readPort, hostWriteAddress, writePort string, logger logger.Logger) (readerConnection, writerConnection *grpc.ClientConn) { - readConn, err := grpc.Dial(fmt.Sprintf("%s:%s", hostReadAddress, readPort), grpc.WithInsecure()) - if err != nil { - logger.Error(fmt.Sprintf("Failed to dial %s:%s for Keto Read Service: %s", hostReadAddress, readPort, err)) - os.Exit(1) + if err := env.Parse(&grpcServerConfig, env.Options{Prefix: envPrefixGrpc, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s gRPC server configuration : %s", svcName, err.Error()) } - - writeConn, err := grpc.Dial(fmt.Sprintf("%s:%s", hostWriteAddress, writePort), grpc.WithInsecure()) - if err != nil { - logger.Error(fmt.Sprintf("Failed to dial %s:%s for Keto Write Service: %s", hostWriteAddress, writePort, err)) - os.Exit(1) + registerAuthServiceServer := func(srv *grpc.Server) { + mainflux.RegisterAuthServiceServer(srv, grpcapi.NewServer(tracer, svc)) } - return readConn, writeConn -} + gs := grpcserver.New(ctx, cancel, svcName, grpcServerConfig, registerAuthServiceServer, logger) -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) + // Start servers + g.Go(func() error { + return hs.Start() + }) + g.Go(func() error { + return gs.Start() + }) + + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs, gs) + }) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Authentication service terminated: %s", err)) } - return db } func newService(db *sqlx.DB, tracer opentracing.Tracer, secret string, logger logger.Logger, readerConn, writerConn *grpc.ClientConn, duration time.Duration) auth.Service { - database := postgres.NewDatabase(db) - keysRepo := tracing.New(postgres.New(database), tracer) + database := authPg.NewDatabase(db) + keysRepo := tracing.New(authPg.New(database), tracer) - groupsRepo := postgres.NewGroupRepo(database) + groupsRepo := authPg.NewGroupRepo(database) groupsRepo = tracing.GroupRepositoryMiddleware(tracer, groupsRepo) pa := keto.NewPolicyAgent(acl.NewCheckServiceClient(readerConn), acl.NewWriteServiceClient(writerConn), acl.NewReadServiceClient(readerConn)) @@ -240,102 +155,9 @@ func newService(db *sqlx.DB, tracer opentracing.Tracer, secret string, logger lo svc := auth.New(keysRepo, groupsRepo, idProvider, t, pa, duration) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "auth", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "auth", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - return svc -} - -func startHTTPServer(ctx context.Context, tracer opentracing.Tracer, svc auth.Service, port string, certFile string, keyFile string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - server := &http.Server{Addr: p, Handler: httpapi.MakeHandler(svc, tracer, logger)} - errCh := make(chan error) - protocol := httpProtocol - switch { - case certFile != "" || keyFile != "": - logger.Info(fmt.Sprintf("Authentication service started using https, cert %s key %s, exposed port %s", certFile, keyFile, port)) - go func() { - errCh <- server.ListenAndServeTLS(certFile, keyFile) - }() - protocol = httpsProtocol - default: - logger.Info(fmt.Sprintf("Authentication service started using http, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Authentication %s service error occurred during shutdown at %s: %s", protocol, p, err)) - return fmt.Errorf("Authentication %s service error occurred during shutdown at %s: %w", protocol, p, err) - } - logger.Info(fmt.Sprintf("Authentication %s service shutdown of http at %s", protocol, p)) - return nil - case err := <-errCh: - return err - } -} + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) -func startGRPCServer(ctx context.Context, tracer opentracing.Tracer, svc auth.Service, port string, certFile string, keyFile string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - - listener, err := net.Listen("tcp", p) - if err != nil { - return fmt.Errorf("failed to listen on port %s: %w", port, err) - } - - var server *grpc.Server - switch { - case certFile != "" || keyFile != "": - creds, err := credentials.NewServerTLSFromFile(certFile, keyFile) - if err != nil { - return fmt.Errorf("failed to load auth certificates: %w", err) - } - logger.Info(fmt.Sprintf("Authentication gRPC service started using https on port %s with cert %s key %s", port, certFile, keyFile)) - server = grpc.NewServer(grpc.Creds(creds)) - default: - logger.Info(fmt.Sprintf("Authentication gRPC service started using http on port %s", port)) - server = grpc.NewServer() - } - - mainflux.RegisterAuthServiceServer(server, grpcapi.NewServer(tracer, svc)) - logger.Info(fmt.Sprintf("Authentication gRPC service started, exposed port %s", port)) - go func() { - errCh <- server.Serve(listener) - }() - - select { - case <-ctx.Done(): - c := make(chan bool) - go func() { - defer close(c) - server.GracefulStop() - }() - select { - case <-c: - case <-time.After(stopWaitTime): - } - logger.Info(fmt.Sprintf("Authentication gRPC service shutdown at %s", p)) - return nil - case err := <-errCh: - return err - } + return svc } diff --git a/cmd/bootstrap/main.go b/cmd/bootstrap/main.go index 397c38d1d3..c957ef1bef 100644 --- a/cmd/bootstrap/main.go +++ b/cmd/bootstrap/main.go @@ -4,366 +4,136 @@ package main import ( "context" - "crypto/aes" - "encoding/hex" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - authapi "github.com/mainflux/mainflux/auth/api/grpc" - rediscons "github.com/mainflux/mainflux/bootstrap/redis/consumer" - redisprod "github.com/mainflux/mainflux/bootstrap/redis/producer" - "github.com/mainflux/mainflux/logger" - opentracing "github.com/opentracing/opentracing-go" - "golang.org/x/sync/errgroup" - - kitprometheus "github.com/go-kit/kit/metrics/prometheus" r "github.com/go-redis/redis/v8" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/bootstrap" api "github.com/mainflux/mainflux/bootstrap/api" - "github.com/mainflux/mainflux/bootstrap/postgres" - mflog "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" + bootstrapPg "github.com/mainflux/mainflux/bootstrap/postgres" + rediscons "github.com/mainflux/mainflux/bootstrap/redis/consumer" + redisprod "github.com/mainflux/mainflux/bootstrap/redis/producer" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + redisClient "github.com/mainflux/mainflux/internal/clients/redis" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" + "github.com/mainflux/mainflux/logger" mfsdk "github.com/mainflux/mainflux/pkg/sdk/go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" + "golang.org/x/sync/errgroup" ) const ( - stopWaitTime = 5 * time.Second - httpProtocol = "http" - httpsProtocol = "https" - - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "bootstrap" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defEncryptKey = "12345678910111213141516171819202" - defClientTLS = "false" - defCACerts = "" - defPort = "8180" - defServerCert = "" - defServerKey = "" - defThingsURL = "http://localhost" - defThingsESURL = "localhost:6379" - defThingsESPass = "" - defThingsESDB = "0" - defESURL = "localhost:6379" - defESPass = "" - defESDB = "0" - defESConsumerName = "bootstrap" - defJaegerURL = "" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - envLogLevel = "MF_BOOTSTRAP_LOG_LEVEL" - envDBHost = "MF_BOOTSTRAP_DB_HOST" - envDBPort = "MF_BOOTSTRAP_DB_PORT" - envDBUser = "MF_BOOTSTRAP_DB_USER" - envDBPass = "MF_BOOTSTRAP_DB_PASS" - envDB = "MF_BOOTSTRAP_DB" - envDBSSLMode = "MF_BOOTSTRAP_DB_SSL_MODE" - envDBSSLCert = "MF_BOOTSTRAP_DB_SSL_CERT" - envDBSSLKey = "MF_BOOTSTRAP_DB_SSL_KEY" - envDBSSLRootCert = "MF_BOOTSTRAP_DB_SSL_ROOT_CERT" - envEncryptKey = "MF_BOOTSTRAP_ENCRYPT_KEY" - envClientTLS = "MF_BOOTSTRAP_CLIENT_TLS" - envCACerts = "MF_BOOTSTRAP_CA_CERTS" - envPort = "MF_BOOTSTRAP_PORT" - envServerCert = "MF_BOOTSTRAP_SERVER_CERT" - envServerKey = "MF_BOOTSTRAP_SERVER_KEY" - envThingsURL = "MF_THINGS_URL" - envThingsESURL = "MF_THINGS_ES_URL" - envThingsESPass = "MF_THINGS_ES_PASS" - envThingsESDB = "MF_THINGS_ES_DB" - envESURL = "MF_BOOTSTRAP_ES_URL" - envESPass = "MF_BOOTSTRAP_ES_PASS" - envESDB = "MF_BOOTSTRAP_ES_DB" - envESConsumerName = "MF_BOOTSTRAP_EVENT_CONSUMER" - envJaegerURL = "MF_JAEGER_URL" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "bootstrap" + envPrefix = "MF_BOOTSTRAP_" + envPrefixES = "MF_BOOTSTRAP_ES_" + envPrefixHttp = "MF_BOOTSTRAP_HTTP_" + defDB = "bootstrap" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - dbConfig postgres.Config - clientTLS bool - encKey []byte - caCerts string - httpPort string - serverCert string - serverKey string - thingsURL string - esThingsURL string - esThingsPass string - esThingsDB string - esURL string - esPass string - esDB string - esConsumerName string - jaegerURL string - authURL string - authTimeout time.Duration + LogLevel string `env:"MF_BOOTSTRAP_LOG_LEVEL" envDefault:"info"` + EncKey string `env:"MF_BOOTSTRAP_ENCRYPT_KEY" envDefault:"12345678910111213141516171819202"` + ESConsumerName string `env:"MF_BOOTSTRAP_EVENT_CONSUMER" envDefault:"bootstrap"` + ThingsURL string `env:"MF_THINGS_URL" envDefault:"http://localhost"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := mflog.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) } - db := connectToDB(cfg.dbConfig, logger) - defer db.Close() - - thingsESConn := connectToRedis(cfg.esThingsURL, cfg.esThingsPass, cfg.esThingsDB, logger) - defer thingsESConn.Close() - - esClient := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) - defer esClient.Close() - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - - auth := authapi.NewClient(authTracer, authConn, cfg.authTimeout) - - svc := newService(auth, db, logger, esClient, cfg) - - g.Go(func() error { - return startHTTPServer(ctx, svc, cfg, logger) - }) - - go subscribeToThingsES(svc, thingsESConn, cfg.esConsumerName, logger) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Bootstrap service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Bootstrap service terminated: %s", err)) - } -} - -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - tls = false - } - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), + log.Fatal(err.Error()) } - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) - } - encKey, err := hex.DecodeString(mainflux.Env(envEncryptKey, defEncryptKey)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envEncryptKey, err.Error()) - } - if err := os.Unsetenv(envEncryptKey); err != nil { - log.Fatalf("Unable to unset %s value: %s", envEncryptKey, err.Error()) - } - if _, err := aes.NewCipher(encKey); err != nil { - log.Fatalf("Invalid %s value: %s", envEncryptKey, err.Error()) - } + // Create new postgres client + dbConfig := pgClient.Config{Name: defDB} - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - dbConfig: dbConfig, - clientTLS: tls, - encKey: encKey, - caCerts: mainflux.Env(envCACerts, defCACerts), - httpPort: mainflux.Env(envPort, defPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - thingsURL: mainflux.Env(envThingsURL, defThingsURL), - esThingsURL: mainflux.Env(envThingsESURL, defThingsESURL), - esThingsPass: mainflux.Env(envThingsESPass, defThingsESPass), - esThingsDB: mainflux.Env(envThingsESDB, defThingsESDB), - esURL: mainflux.Env(envESURL, defESURL), - esPass: mainflux.Env(envESPass, defESPass), - esDB: mainflux.Env(envESDB, defESDB), - esConsumerName: mainflux.Env(envESConsumerName, defESConsumerName), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, + db, err := pgClient.SetupWithConfig(envPrefix, *bootstrapPg.Migration(), dbConfig) + if err != nil { + log.Fatal(err.Error()) } -} + defer db.Close() -func connectToDB(cfg postgres.Config, logger mflog.Logger) *sqlx.DB { - db, err := postgres.Connect(cfg) + // Create new redis client for bootstrap event store + esClient, err := redisClient.Setup(envPrefixES) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) + log.Fatalf("failed to setup %s bootstrap event store redis client : %s", svcName, err.Error()) } - return db -} + defer esClient.Close() -func connectToRedis(redisURL, redisPass, redisDB string, logger mflog.Logger) *r.Client { - db, err := strconv.Atoi(redisDB) + // Create new auth grpc client api + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to redis: %s", err)) - os.Exit(1) + log.Fatal(err) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - return r.NewClient(&r.Options{ - Addr: redisURL, - Password: redisPass, - DB: db, - }) -} + // Create new service + svc := newService(auth, db, logger, esClient, cfg) -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + // Create an new HTTP server + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, bootstrap.NewConfigReader([]byte(cfg.EncKey)), logger), logger) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() + // Start servers + g.Go(func() error { + return hs.Start() + }) + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) + + // Subscribe to things event store + thingsESClient, err := redisClient.Setup(envPrefixES) if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + log.Fatalf(err.Error()) } + defer thingsESClient.Close() - return tracer, closer + go subscribeToThingsES(svc, thingsESClient, cfg.ESConsumerName, logger) + + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Bootstrap service terminated: %s", err)) + } } -func newService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger mflog.Logger, esClient *r.Client, cfg config) bootstrap.Service { - thingsRepo := postgres.NewConfigRepository(db, logger) +func newService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger logger.Logger, esClient *r.Client, cfg config) bootstrap.Service { + repoConfig := bootstrapPg.NewConfigRepository(db, logger) config := mfsdk.Config{ - ThingsURL: cfg.thingsURL, + ThingsURL: cfg.ThingsURL, } sdk := mfsdk.NewSDK(config) - svc := bootstrap.New(auth, thingsRepo, sdk, cfg.encKey) + svc := bootstrap.New(auth, repoConfig, sdk, []byte(cfg.EncKey)) svc = redisprod.NewEventStoreMiddleware(svc, esClient) svc = api.NewLoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "bootstrap", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "bootstrap", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - return svc -} - -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } - - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) - } - - return conn -} - -func startHTTPServer(ctx context.Context, svc bootstrap.Service, cfg config, logger mflog.Logger) error { - p := fmt.Sprintf(":%s", cfg.httpPort) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, bootstrap.NewConfigReader(cfg.encKey), logger)} - errCh := make(chan error) - protocol := httpProtocol - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("Bootstrap service started using https on port %s with cert %s key %s", - cfg.httpPort, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - protocol = httpsProtocol + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) - default: - logger.Info(fmt.Sprintf("Bootstrap service started using http on port %s", cfg.httpPort)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Bootstrap %s service error occurred during shutdown at %s: %s", protocol, p, err)) - return fmt.Errorf("bootstrap %s service error occurred during shutdown at %s: %w", protocol, p, err) - } - logger.Info(fmt.Sprintf("Bootstrap %s service shutdown of http at %s", protocol, p)) - return nil - case err := <-errCh: - return err - } + return svc } -func subscribeToThingsES(svc bootstrap.Service, client *r.Client, consumer string, logger mflog.Logger) { +func subscribeToThingsES(svc bootstrap.Service, client *r.Client, consumer string, logger logger.Logger) { eventStore := rediscons.NewEventStore(svc, client, consumer, logger) logger.Info("Subscribed to Redis Event Store") if err := eventStore.Subscribe(context.Background(), "mainflux.things"); err != nil { diff --git a/cmd/cassandra-reader/main.go b/cmd/cassandra-reader/main.go index bc5f8e3441..29aca24d4d 100644 --- a/cmd/cassandra-reader/main.go +++ b/cmd/cassandra-reader/main.go @@ -6,312 +6,105 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "strings" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/gocql/gocql" - "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + cassandraClient "github.com/mainflux/mainflux/internal/clients/cassandra" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/readers" "github.com/mainflux/mainflux/readers/api" "github.com/mainflux/mainflux/readers/cassandra" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - stopWaitTime = 5 * time.Second - - sep = "," - defLogLevel = "error" - defPort = "8180" - defCluster = "127.0.0.1" - defKeyspace = "mainflux" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDBPort = "9042" - defClientTLS = "false" - defCACerts = "" - defServerCert = "" - defServerKey = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - defUsersAuthURL = "localhost:8181" - defUsersAuthTimeout = "1s" - - envLogLevel = "MF_CASSANDRA_READER_LOG_LEVEL" - envPort = "MF_CASSANDRA_READER_PORT" - envCluster = "MF_CASSANDRA_READER_DB_CLUSTER" - envKeyspace = "MF_CASSANDRA_READER_DB_KEYSPACE" - envDBUser = "MF_CASSANDRA_READER_DB_USER" - envDBPass = "MF_CASSANDRA_READER_DB_PASS" - envDBPort = "MF_CASSANDRA_READER_DB_PORT" - envClientTLS = "MF_CASSANDRA_READER_CLIENT_TLS" - envCACerts = "MF_CASSANDRA_READER_CA_CERTS" - envServerCert = "MF_CASSANDRA_READER_SERVER_CERT" - envServerKey = "MF_CASSANDRA_READER_SERVER_KEY" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" - envUsersAuthURL = "MF_AUTH_GRPC_URL" - envUsersAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "cassandra-reader" + envPrefix = "MF_CASSANDRA_READER_" + envPrefixHttp = "MF_CASSANDRA_READER_HTTP_" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - port string - dbCfg cassandra.DBConfig - clientTLS bool - caCerts string - serverCert string - serverKey string - jaegerURL string - thingsAuthURL string - usersAuthURL string - thingsAuthTimeout time.Duration - usersAuthTimeout time.Duration + LogLevel string `env:"MF_CASSANDRA_READER_LOG_LEVEL" envDefault:"info"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) - } - - session := connectToCassandra(cfg.dbCfg, logger) - defer session.Close() - - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - - auth := authapi.NewClient(authTracer, authConn, cfg.usersAuthTimeout) - - repo := newService(session, logger) - - g.Go(func() error { - return startHTTPServer(ctx, repo, tc, auth, cfg, logger) - }) + // Create cassandra reader service configurations + cfg := config{} - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Cassandra reader service shutdown by signal: %s", sig)) - } - return nil - }) - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Cassandra reader service terminated: %s", err)) + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) } -} -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - logger.Info("Connecting to auth via gRPC") - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } - - conn, err := grpc.Dial(cfg.usersAuthURL, opts...) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) + log.Fatalf(err.Error()) } - logger.Info(fmt.Sprintf("Established gRPC connection to things via gRPC: %s", cfg.usersAuthURL)) - return conn -} -func loadConfig() config { - dbPort, err := strconv.Atoi(mainflux.Env(envDBPort, defDBPort)) + // Create new thing grpc client + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatal(err) - } - - dbCfg := cassandra.DBConfig{ - Hosts: strings.Split(mainflux.Env(envCluster, defCluster), sep), - Keyspace: mainflux.Env(envKeyspace, defKeyspace), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Port: dbPort, + log.Fatal(err.Error()) } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + // Create new auth grpc client + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatal(err.Error()) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) + // Create new cassandra client + csdSession, err := cassandraClient.Setup(envPrefix) if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + log.Fatal(err.Error()) } + defer csdSession.Close() - usersAuthTimeout, err := time.ParseDuration(mainflux.Env(envUsersAuthTimeout, defUsersAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) - } + // Create new service + repo := newService(csdSession, logger) - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - dbCfg: dbCfg, - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - usersAuthURL: mainflux.Env(envUsersAuthURL, defUsersAuthURL), - usersAuthTimeout: usersAuthTimeout, - thingsAuthTimeout: authTimeout, - } -} + // Create new http server + httpServerConfig := server.Config{Port: defSvcHttpPort} -func connectToCassandra(dbCfg cassandra.DBConfig, logger logger.Logger) *gocql.Session { - session, err := cassandra.Connect(dbCfg) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to Cassandra cluster: %s", err)) - os.Exit(1) + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } - return session -} + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(repo, tc, auth, svcName, logger), logger) -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } - - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) - } - logger.Info(fmt.Sprintf("Established gRPC connection to things via gRPC: %s", cfg.thingsAuthURL)) - return conn -} + // Start servers + g.Go(func() error { + return hs.Start() + }) -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Cassandra reader service terminated: %s", err)) } - - return tracer, closer } -func newService(session *gocql.Session, logger logger.Logger) readers.MessageRepository { - repo := cassandra.New(session) +func newService(csdSession *gocql.Session, logger logger.Logger) readers.MessageRepository { + repo := cassandra.New(csdSession) repo = api.LoggingMiddleware(repo, logger) - repo = api.MetricsMiddleware( - repo, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "cassandra", - Subsystem: "message_reader", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "cassandra", - Subsystem: "message_reader", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - + counter, latency := internal.MakeMetrics("cassandra", "message_reader") + repo = api.MetricsMiddleware(repo, counter, latency) return repo } - -func startHTTPServer(ctx context.Context, repo readers.MessageRepository, tc mainflux.ThingsServiceClient, ac mainflux.AuthServiceClient, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(repo, tc, ac, "cassandra-reader", logger)} - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("Cassandra reader service started using https on port %s with cert %s key %s", cfg.port, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - default: - logger.Info(fmt.Sprintf("Cassandra reader service started, exposed port %s", cfg.port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Cassandra reader service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("cassandra reader service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Cassandra reader service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 3fff698691..974fdbff66 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -7,175 +7,101 @@ import ( "context" "fmt" "log" - "net/http" "os" - "strconv" - "strings" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/gocql/gocql" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/writers/api" "github.com/mainflux/mainflux/consumers/writers/cassandra" + "github.com/mainflux/mainflux/internal" + cassandraClient "github.com/mainflux/mainflux/internal/clients/cassandra" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" - stdprometheus "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) const ( - svcName = "cassandra-writer" - sep = "," - stopWaitTime = 5 * time.Second - - defBrokerURL = "nats://localhost:4222" - defLogLevel = "error" - defPort = "8180" - defCluster = "127.0.0.1" - defKeyspace = "mainflux" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDBPort = "9042" - defConfigPath = "/config.toml" - - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_CASSANDRA_WRITER_LOG_LEVEL" - envPort = "MF_CASSANDRA_WRITER_PORT" - envCluster = "MF_CASSANDRA_WRITER_DB_CLUSTER" - envKeyspace = "MF_CASSANDRA_WRITER_DB_KEYSPACE" - envDBUser = "MF_CASSANDRA_WRITER_DB_USER" - envDBPass = "MF_CASSANDRA_WRITER_DB_PASS" - envDBPort = "MF_CASSANDRA_WRITER_DB_PORT" - envConfigPath = "MF_CASSANDRA_WRITER_CONFIG_PATH" + svcName = "cassandra-writer" + envPrefix = "MF_CASSANDRA_WRITER_" + envPrefixHttp = "MF_CASSANDRA_WRITER_HTTP_" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - logLevel string - port string - configPath string - dbCfg cassandra.DBConfig + LogLevel string `env:"MF_CASSANDRA_WRITER_LOG_LEVEL" envDefault:"info"` + ConfigPath string `env:"MF_CASSANDRA_WRITER_CONFIG_PATH" envDefault:"/config.toml"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + // Create new cassandra writer service configurations + cfg := config{} + + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + // Create new to cassandra client + csdSession, err := cassandraClient.SetupDB(envPrefix, cassandra.Table) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatal(err.Error()) } - defer pubSub.Close() + defer csdSession.Close() - session := connectToCassandra(cfg.dbCfg, logger) - defer session.Close() + // Create new cassandra-writer repo + repo := newService(csdSession, logger) - repo := newService(session, logger) + // Create new pub sub broker + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) + if err != nil { + log.Fatalf("failed to connect to message broker: %s", err.Error()) + } + defer pubSub.Close() - if err := consumers.Start(svcName, pubSub, repo, cfg.configPath, logger); err != nil { + // Start new consumer + if err := consumers.Start(svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Cassandra writer: %s", err)) } - go startHTTPServer(ctx, cfg.port, logger) + // Create new http server + httpServerConfig := server.Config{Port: defSvcHttpPort} - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Cassandra writer service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Cassandra writer service terminated: %s", err)) + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefix, AltPrefix: envPrefixHttp}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } -} -func loadConfig() config { - dbPort, err := strconv.Atoi(mainflux.Env(envDBPort, defDBPort)) - if err != nil { - log.Fatal(err) - } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svcName), logger) - dbCfg := cassandra.DBConfig{ - Hosts: strings.Split(mainflux.Env(envCluster, defCluster), sep), - Keyspace: mainflux.Env(envKeyspace, defKeyspace), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Port: dbPort, - } + // Start servers + g.Go(func() error { + return hs.Start() + }) - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - configPath: mainflux.Env(envConfigPath, defConfigPath), - dbCfg: dbCfg, - } -} + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) -func connectToCassandra(dbCfg cassandra.DBConfig, logger logger.Logger) *gocql.Session { - session, err := cassandra.Connect(dbCfg) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to Cassandra cluster: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Cassandra writer service terminated: %s", err)) } - return session } func newService(session *gocql.Session, logger logger.Logger) consumers.Consumer { repo := cassandra.New(session) repo = api.LoggingMiddleware(repo, logger) - repo = api.MetricsMiddleware( - repo, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "cassandra", - Subsystem: "message_writer", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "cassandra", - Subsystem: "message_writer", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - + counter, latency := internal.MakeMetrics("cassandra", "message_writer") + repo = api.MetricsMiddleware(repo, counter, latency) return repo } - -func startHTTPServer(ctx context.Context, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svcName)} - logger.Info(fmt.Sprintf("Cassandra writer service started, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Cassandra writer service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("cassandra writer service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Cassandra writer service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/certs/main.go b/cmd/certs/main.go index 37a2805dd8..32b9e9685e 100644 --- a/cmd/certs/main.go +++ b/cmd/certs/main.go @@ -7,99 +7,35 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/go-redis/redis/v8" "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" "github.com/mainflux/mainflux/certs" "github.com/mainflux/mainflux/certs/api" vault "github.com/mainflux/mainflux/certs/pki" - "github.com/mainflux/mainflux/certs/postgres" + certsPg "github.com/mainflux/mainflux/certs/postgres" + "github.com/mainflux/mainflux/internal" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "github.com/jmoiron/sqlx" - mflog "github.com/mainflux/mainflux/logger" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/pkg/errors" mfsdk "github.com/mainflux/mainflux/pkg/sdk/go" - jconfig "github.com/uber/jaeger-client-go/config" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "certs" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defClientTLS = "false" - defCACerts = "" - defPort = "8204" - defServerCert = "" - defServerKey = "" - defCertsURL = "http://localhost" - defThingsURL = "http://things:8182" - defJaegerURL = "" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - defSignCAPath = "ca.crt" - defSignCAKeyPath = "ca.key" - defSignHoursValid = "2048h" - defSignRSABits = "" - - defVaultHost = "" - defVaultRole = "mainflux" - defVaultToken = "" - defVaultPKIIntPath = "pki_int" - - envPort = "MF_CERTS_HTTP_PORT" - envLogLevel = "MF_CERTS_LOG_LEVEL" - envDBHost = "MF_CERTS_DB_HOST" - envDBPort = "MF_CERTS_DB_PORT" - envDBUser = "MF_CERTS_DB_USER" - envDBPass = "MF_CERTS_DB_PASS" - envDB = "MF_CERTS_DB" - envDBSSLMode = "MF_CERTS_DB_SSL_MODE" - envDBSSLCert = "MF_CERTS_DB_SSL_CERT" - envDBSSLKey = "MF_CERTS_DB_SSL_KEY" - envDBSSLRootCert = "MF_CERTS_DB_SSL_ROOT_CERT" - envClientTLS = "MF_CERTS_CLIENT_TLS" - envCACerts = "MF_CERTS_CA_CERTS" - envServerCert = "MF_CERTS_SERVER_CERT" - envServerKey = "MF_CERTS_SERVER_KEY" - envCertsURL = "MF_SDK_CERTS_URL" - envJaegerURL = "MF_JAEGER_URL" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" - envThingsURL = "MF_THINGS_URL" - envSignCAPath = "MF_CERTS_SIGN_CA_PATH" - envSignCAKey = "MF_CERTS_SIGN_CA_KEY_PATH" - envSignHoursValid = "MF_CERTS_SIGN_HOURS_VALID" - envSignRSABits = "MF_CERTS_SIGN_RSA_BITS" - - envVaultHost = "MF_CERTS_VAULT_HOST" - envVaultPKIIntPath = "MF_VAULT_PKI_INT_PATH" - envVaultRole = "MF_VAULT_CA_ROLE_NAME" - envVaultToken = "MF_VAULT_TOKEN" + svcName = "certs" + envPrefix = "MF_CERTS_" + envPrefixHttp = "MF_CERTS_HTTP_" + defDB = "certs" + defSvcHttpPort = "8204" ) var ( @@ -110,36 +46,35 @@ var ( ) type config struct { - logLevel string - dbConfig postgres.Config - clientTLS bool - caCerts string - httpPort string - serverCert string - serverKey string - certsURL string - thingsURL string - jaegerURL string - authURL string - authTimeout time.Duration + LogLevel string `env:"MF_CERTS_LOG_LEVEL" envDefault:"info"` + CertsURL string `env:"MF_SDK_CERTS_URL" envDefault:"http://localhost"` + ThingsURL string `env:"MF_THINGS_URL" envDefault:"http://things:8182"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` + // Sign and issue certificates without 3rd party PKI - signCAPath string - signCAKeyPath string - signRSABits int - signHoursValid string + SignCAPath string `env:"MF_CERTS_SIGN_CA_PATH" envDefault:"ca.crt"` + SignCAKeyPath string `env:"MF_CERTS_SIGN_CA_KEY_PATH" envDefault:"ca.key"` + // used in pki mock , need to clean up certs in separate PR + SignRSABits int `env:"MF_CERTS_SIGN_RSA_BITS," envDefault:""` + SignHoursValid string `env:"MF_CERTS_SIGN_HOURS_VALID" envDefault:"2048h"` + // 3rd party PKI API access settings - pkiPath string - pkiToken string - pkiHost string - pkiRole string + PkiHost string `env:"MF_CERTS_VAULT_HOST" envDefault:""` + PkiPath string `env:"MF_VAULT_PKI_INT_PATH" envDefault:"pki_int"` + PkiRole string `env:"MF_VAULT_CA_ROLE_NAME" envDefault:"mainflux"` + PkiToken string `env:"MF_VAULT_TOKEN" envDefault:""` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := mflog.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } @@ -149,38 +84,43 @@ func main() { logger.Error("Failed to load CA certificates for issuing client certs") } - if cfg.pkiHost == "" { + if cfg.PkiHost == "" { log.Fatalf("No host specified for PKI engine") } - pkiClient, err := vault.NewVaultClient(cfg.pkiToken, cfg.pkiHost, cfg.pkiPath, cfg.pkiRole) + pkiClient, err := vault.NewVaultClient(cfg.PkiToken, cfg.PkiHost, cfg.PkiPath, cfg.PkiRole) if err != nil { - log.Fatalf("Failed to configure client for PKI engine") + log.Fatalf("failed to configure client for PKI engine") } - db := connectToDB(cfg.dbConfig, logger) + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *certsPg.Migration(), dbConfig) + if err != nil { + log.Fatal(err) + } defer db.Close() - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - - auth := authapi.NewClient(authTracer, authConn, cfg.authTimeout) + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err.Error()) + } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) svc := newService(auth, db, logger, nil, tlsCert, caCert, cfg, pkiClient) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) + } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger), logger) + g.Go(func() error { - return startHTTPServer(ctx, svc, cfg, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Certs service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -188,223 +128,42 @@ func main() { } } -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) - if err != nil { - tls = false - } - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), - } - - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) - } - - signRSABits, err := strconv.Atoi(mainflux.Env(envSignRSABits, defSignRSABits)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envSignRSABits, err.Error()) - } - - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - dbConfig: dbConfig, - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - httpPort: mainflux.Env(envPort, defPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - certsURL: mainflux.Env(envCertsURL, defCertsURL), - thingsURL: mainflux.Env(envThingsURL, defThingsURL), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, - - signCAKeyPath: mainflux.Env(envSignCAKey, defSignCAKeyPath), - signCAPath: mainflux.Env(envSignCAPath, defSignCAPath), - signHoursValid: mainflux.Env(envSignHoursValid, defSignHoursValid), - signRSABits: signRSABits, - - pkiToken: mainflux.Env(envVaultToken, defVaultToken), - pkiPath: mainflux.Env(envVaultPKIIntPath, defVaultPKIIntPath), - pkiRole: mainflux.Env(envVaultRole, defVaultRole), - pkiHost: mainflux.Env(envVaultHost, defVaultHost), - } - -} - -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) - } - return db -} - -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } - - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) - } - - return conn -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } - - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) - } - - return tracer, closer -} - -func newService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger mflog.Logger, esClient *redis.Client, tlsCert tls.Certificate, x509Cert *x509.Certificate, cfg config, pkiAgent vault.Agent) certs.Service { - certsRepo := postgres.NewRepository(db, logger) - - certsConfig := certs.Config{ - LogLevel: cfg.logLevel, - ClientTLS: cfg.clientTLS, - CaCerts: cfg.caCerts, - HTTPPort: cfg.httpPort, - ServerCert: cfg.serverCert, - ServerKey: cfg.serverKey, - CertsURL: cfg.certsURL, - JaegerURL: cfg.jaegerURL, - AuthURL: cfg.authURL, - AuthTimeout: cfg.authTimeout, - SignTLSCert: tlsCert, - SignX509Cert: x509Cert, - SignHoursValid: cfg.signHoursValid, - SignRSABits: cfg.signRSABits, - PKIToken: cfg.pkiToken, - PKIHost: cfg.pkiHost, - PKIPath: cfg.pkiPath, - PKIRole: cfg.pkiRole, - } - +func newService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger logger.Logger, esClient *redis.Client, tlsCert tls.Certificate, x509Cert *x509.Certificate, cfg config, pkiAgent vault.Agent) certs.Service { + certsRepo := certsPg.NewRepository(db, logger) config := mfsdk.Config{ - CertsURL: cfg.certsURL, - ThingsURL: cfg.thingsURL, + CertsURL: cfg.CertsURL, + ThingsURL: cfg.ThingsURL, } - sdk := mfsdk.NewSDK(config) - - svc := certs.New(auth, certsRepo, sdk, certsConfig, pkiAgent) + svc := certs.New(auth, certsRepo, sdk, pkiAgent) svc = api.NewLoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "certs", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "certs", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) return svc } -func startHTTPServer(ctx context.Context, svc certs.Service, cfg config, logger mflog.Logger) error { - p := fmt.Sprintf(":%s", cfg.httpPort) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, logger)} - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("Certs service started using https on port %s with cert %s key %s", cfg.httpPort, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - - default: - logger.Info(fmt.Sprintf("Certs service started using http on port %s", cfg.httpPort)) - go func() { - errCh <- http.ListenAndServe(p, api.MakeHandler(svc, logger)) - }() - } - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Certs service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("certs service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Certs service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} - func loadCertificates(conf config) (tls.Certificate, *x509.Certificate, error) { var tlsCert tls.Certificate var caCert *x509.Certificate - if conf.signCAPath == "" || conf.signCAKeyPath == "" { + if conf.SignCAPath == "" || conf.SignCAKeyPath == "" { return tlsCert, caCert, nil } - if _, err := os.Stat(conf.signCAPath); os.IsNotExist(err) { + if _, err := os.Stat(conf.SignCAPath); os.IsNotExist(err) { return tlsCert, caCert, errCACertificateNotExist } - if _, err := os.Stat(conf.signCAKeyPath); os.IsNotExist(err) { + if _, err := os.Stat(conf.SignCAKeyPath); os.IsNotExist(err) { return tlsCert, caCert, errCAKeyNotExist } - tlsCert, err := tls.LoadX509KeyPair(conf.signCAPath, conf.signCAKeyPath) + tlsCert, err := tls.LoadX509KeyPair(conf.SignCAPath, conf.SignCAKeyPath) if err != nil { return tlsCert, caCert, errors.Wrap(errFailedCertLoading, err) } - b, err := ioutil.ReadFile(conf.signCAPath) + b, err := os.ReadFile(conf.SignCAPath) if err != nil { return tlsCert, caCert, errors.Wrap(errFailedCertLoading, err) } diff --git a/cmd/coap/main.go b/cmd/coap/main.go index ced471d95b..ceb1088eab 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -6,86 +6,61 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/coap" "github.com/mainflux/mainflux/coap/api" + "github.com/mainflux/mainflux/internal" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + coapserver "github.com/mainflux/mainflux/internal/server/coap" + httpserver "github.com/mainflux/mainflux/internal/server/http" logger "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - opentracing "github.com/opentracing/opentracing-go" - gocoap "github.com/plgd-dev/go-coap/v2" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - stopWaitTime = 5 * time.Second - - defPort = "5683" - defBrokerURL = "nats://localhost:4222" - defLogLevel = "error" - defClientTLS = "false" - defCACerts = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - - envPort = "MF_COAP_ADAPTER_PORT" - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_COAP_ADAPTER_LOG_LEVEL" - envClientTLS = "MF_COAP_ADAPTER_CLIENT_TLS" - envCACerts = "MF_COAP_ADAPTER_CA_CERTS" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" + svcName = "coap_adapter" + envPrefix = "MF_COAP_ADAPTER_" + envPrefixHttp = "MF_COAP_ADAPTER_HTTP_" + envPrefixCoap = "MF_COAP_ADAPTER_COAP_" + defSvcHttpPort = "5683" + defSvcCoapPort = "5683" ) type config struct { - port string - brokerURL string - logLevel string - clientTLS bool - caCerts string - jaegerURL string - thingsAuthURL string - thingsAuthTimeout time.Duration + LogLevel string `env:"MF_INFLUX_READER_LOG_LEVEL" envDefault:"info"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err.Error()) + } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - nps, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + nps, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer nps.Close() @@ -93,151 +68,32 @@ func main() { svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "coap_adapter", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "coap_adapter", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) + + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) + } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHTTPHandler(), logger) + + coapServerConfig := server.Config{Port: defSvcCoapPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixCoap, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s CoAP server configuration : %s", svcName, err.Error()) + } + cs := coapserver.New(ctx, cancel, svcName, coapServerConfig, api.MakeCoAPHandler(svc, logger), logger) g.Go(func() error { - return startHTTPServer(ctx, cfg.port, logger) + return hs.Start() }) - g.Go(func() error { - return startCOAPServer(ctx, cfg, svc, nil, logger) + return cs.Start() }) - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("CoAP adapter service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs, cs) }) if err := g.Wait(); err != nil { logger.Error(fmt.Sprintf("CoAP adapter service terminated: %s", err)) } } - -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) - if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) - } - - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) - } - - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - port: mainflux.Env(envPort, defPort), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - thingsAuthTimeout: authTimeout, - } -} - -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } - - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) - } - return conn -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } - - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) - } - - return tracer, closer -} - -func startHTTPServer(ctx context.Context, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHTTPHandler()} - logger.Info(fmt.Sprintf("CoAP service started, exposed port %s", port)) - - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("CoAP adapter service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("CoAP adapter service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("CoAP adapter service shutdown at %s", p)) - return nil - case err := <-errCh: - return err - } -} - -func startCOAPServer(ctx context.Context, cfg config, svc coap.Service, auth mainflux.ThingsServiceClient, l logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.port) - errCh := make(chan error) - l.Info(fmt.Sprintf("CoAP adapter service started, exposed port %s", cfg.port)) - go func() { - errCh <- gocoap.ListenAndServe("udp", p, api.MakeCoAPHandler(svc, l)) - }() - select { - case <-ctx.Done(): - l.Info(fmt.Sprintf("CoAP adapter service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/http/main.go b/cmd/http/main.go index d058718d26..463cf9324f 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -6,218 +6,95 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - "google.golang.org/grpc/credentials" - - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/http/api" + "github.com/mainflux/mainflux/internal" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" + "github.com/mainflux/mainflux/pkg/messaging" "github.com/mainflux/mainflux/pkg/messaging/brokers" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defClientTLS = "false" - defCACerts = "" - defPort = "8180" - defBrokerURL = "nats://localhost:4222" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - - envLogLevel = "MF_HTTP_ADAPTER_LOG_LEVEL" - envClientTLS = "MF_HTTP_ADAPTER_CLIENT_TLS" - envCACerts = "MF_HTTP_ADAPTER_CA_CERTS" - envPort = "MF_HTTP_ADAPTER_PORT" - envBrokerURL = "MF_BROKER_URL" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" + svcName = "http_adapter" + envPrefix = "MF_HTTP_ADAPTER_" + envPrefixHttp = "MF_HTTP_ADAPTER_HTTP_" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - logLevel string - port string - clientTLS bool - caCerts string - jaegerURL string - thingsAuthURL string - thingsAuthTimeout time.Duration + LogLevel string `env:"MF_HTTP_ADAPTER_LOG_LEVEL" envDefault:"info"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) } - conn := connectToThings(cfg, logger) - defer conn.Close() - - tracer, closer := initJaeger("http_adapter", cfg.jaegerURL, logger) - defer closer.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - pub, err := brokers.NewPublisher(cfg.brokerURL) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) - } - defer pub.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - svc := adapter.New(pub, tc) - - svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "http_adapter", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "http_adapter", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - - g.Go(func() error { - return startHTTPServer(ctx, svc, cfg, logger, tracer) - }) - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("HTTP adapter service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("HTTP adapter service terminated: %s", err)) + log.Fatalf(err.Error()) } -} - -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatal(err.Error()) } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) + pub, err := brokers.NewPublisher(cfg.BrokerURL) if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } + defer pub.Close() - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - thingsAuthTimeout: authTimeout, - } -} + svc := newService(pub, tc, logger) -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + tracer, closer, err := jaegerClient.NewTracer("http_adapter", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer closer.Close() - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, tracer, logger), logger) - return tracer, closer -} + g.Go(func() error { + return hs.Start() + }) -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("HTTP adapter service terminated: %s", err)) } - return conn } -func startHTTPServer(ctx context.Context, svc adapter.Service, cfg config, logger logger.Logger, tracer opentracing.Tracer) error { - p := fmt.Sprintf(":%s", cfg.port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, tracer, logger)} - logger.Info(fmt.Sprintf("HTTP adapter service started on port %s", cfg.port)) - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("HTTP adapter service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("http adapter service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("HTTP adapter service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } +func newService(pub messaging.Publisher, tc mainflux.ThingsServiceClient, logger logger.Logger) adapter.Service { + svc := adapter.New(pub, tc) + svc = api.LoggingMiddleware(svc, logger) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) + return svc } diff --git a/cmd/influxdb-reader/main.go b/cmd/influxdb-reader/main.go index 41153ec6f1..b85b9cbd87 100644 --- a/cmd/influxdb-reader/main.go +++ b/cmd/influxdb-reader/main.go @@ -3,311 +3,101 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" influxdata "github.com/influxdata/influxdb/client/v2" - "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + influxDBClient "github.com/mainflux/mainflux/internal/clients/influxdb" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/readers" "github.com/mainflux/mainflux/readers/api" "github.com/mainflux/mainflux/readers/influxdb" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defPort = "8180" - defDB = "mainflux" - defDBHost = "localhost" - defDBPort = "8086" - defDBUser = "mainflux" - defDBPass = "mainflux" - defClientTLS = "false" - defCACerts = "" - defServerCert = "" - defServerKey = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - defUsersAuthURL = "localhost:8181" - defUsersAuthTimeout = "1s" - - envLogLevel = "MF_INFLUX_READER_LOG_LEVEL" - envPort = "MF_INFLUX_READER_PORT" - envDB = "MF_INFLUXDB_DB" - envDBHost = "MF_INFLUXDB_HOST" - envDBPort = "MF_INFLUXDB_PORT" - envDBUser = "MF_INFLUXDB_ADMIN_USER" - envDBPass = "MF_INFLUXDB_ADMIN_PASSWORD" - envClientTLS = "MF_INFLUX_READER_CLIENT_TLS" - envCACerts = "MF_INFLUX_READER_CA_CERTS" - envServerCert = "MF_INFLUX_READER_SERVER_CERT" - envServerKey = "MF_INFLUX_READER_SERVER_KEY" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" - envAuthURL = "MF_AUTH_GRPC_URL" - envUsersAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "influxdb-reader" + envPrefix = "MF_INFLUX_READER_" + envPrefixHttp = "MF_INFLUX_READER_HTTP_" + envPrefixInfluxdb = "MF_INFLUXDB_" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - port string - dbName string - dbHost string - dbPort string - dbUser string - dbPass string - clientTLS bool - caCerts string - serverCert string - serverKey string - jaegerURL string - thingsAuthURL string - usersAuthURL string - thingsAuthTimeout time.Duration - usersAuthTimeout time.Duration + LogLevel string `env:"MF_INFLUX_READER_LOG_LEVEL" envDefault:"info"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg, clientCfg := loadConfigs() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) } - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - - auth := authapi.NewClient(authTracer, authConn, cfg.usersAuthTimeout) - - client, err := influxdata.NewHTTPClient(clientCfg) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to create InfluxDB client: %s", err)) - os.Exit(1) - } - defer client.Close() - - repo := newService(client, cfg.dbName, logger) - - g.Go(func() error { - return startHTTPServer(ctx, repo, tc, auth, cfg, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("InfluxDB reader service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("InfluxDB reader service terminated: %s", err)) - } -} - -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - logger.Info("Connecting to auth via gRPC") - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") + log.Fatalf(err.Error()) } - conn, err := grpc.Dial(cfg.usersAuthURL, opts...) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) + log.Fatal(err.Error()) } - logger.Info("Established gRPC connection to auth via gRPC") - return conn -} + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) -func loadConfigs() (config, influxdata.HTTPConfig) { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatal(err.Error()) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + influxDBConfig := influxDBClient.Config{} + if err := env.Parse(&influxDBConfig, env.Options{Prefix: envPrefixInfluxdb}); err != nil { + log.Fatalf("failed to load InfluxDB client configuration from environment variable : %s", err.Error()) } - - userAuthTimeout, err := time.ParseDuration(mainflux.Env(envUsersAuthTimeout, defUsersAuthTimeout)) + client, err := influxDBClient.Connect(influxDBConfig) if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + log.Fatalf("failed to connect to InfluxDB : %s", err.Error()) } + defer client.Close() - cfg := config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - dbName: mainflux.Env(envDB, defDB), - dbHost: mainflux.Env(envDBHost, defDBHost), - dbPort: mainflux.Env(envDBPort, defDBPort), - dbUser: mainflux.Env(envDBUser, defDBUser), - dbPass: mainflux.Env(envDBPass, defDBPass), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - thingsAuthTimeout: authTimeout, - usersAuthURL: mainflux.Env(envAuthURL, defUsersAuthURL), - usersAuthTimeout: userAuthTimeout, - } + repo := newService(client, influxDBConfig.DbName, logger) - clientCfg := influxdata.HTTPConfig{ - Addr: fmt.Sprintf("http://%s:%s", cfg.dbHost, cfg.dbPort), - Username: cfg.dbUser, - Password: cfg.dbPass, + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(repo, tc, auth, svcName, logger), logger) - return cfg, clientCfg -} - -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - logger.Info("connecting to things via gRPC") - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } - - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) - } - logger.Info(fmt.Sprintf("Established gRPC connection to things via gRPC: %s", cfg.thingsAuthURL)) - return conn -} + g.Go(func() error { + return hs.Start() + }) -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("InfluxDB reader service terminated: %s", err)) } - - return tracer, closer } func newService(client influxdata.Client, dbName string, logger logger.Logger) readers.MessageRepository { repo := influxdb.New(client, dbName) repo = api.LoggingMiddleware(repo, logger) - repo = api.MetricsMiddleware( - repo, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "influxdb", - Subsystem: "message_reader", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "influxdb", - Subsystem: "message_reader", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics("influxdb", "message_reader") + repo = api.MetricsMiddleware(repo, counter, latency) return repo } - -func startHTTPServer(ctx context.Context, repo readers.MessageRepository, tc mainflux.ThingsServiceClient, ac mainflux.AuthServiceClient, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(repo, tc, ac, "influxdb-reader", logger)} - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("InfluxDB reader service started using https on port %s with cert %s key %s", - cfg.port, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - default: - logger.Info(fmt.Sprintf("InfluxDB reader service started, exposed port %s", cfg.port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("InfluxDB reader service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("influxDB reader service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("InfluxDB reader service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index 3a31fabcfa..e50eb03e60 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -7,105 +7,84 @@ import ( "context" "fmt" "log" - "net/http" "os" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" influxdata "github.com/influxdata/influxdb/client/v2" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/writers/api" "github.com/mainflux/mainflux/consumers/writers/influxdb" + "github.com/mainflux/mainflux/internal" + influxDBClient "github.com/mainflux/mainflux/internal/clients/influxdb" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" - stdprometheus "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) const ( - svcName = "influxdb-writer" - stopWaitTime = 5 * time.Second - - defBrokerURL = "nats://localhost:4222" - defLogLevel = "error" - defPort = "8180" - defDB = "mainflux" - defDBHost = "localhost" - defDBPort = "8086" - defDBUser = "mainflux" - defDBPass = "mainflux" - defConfigPath = "/config.toml" - - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_INFLUX_WRITER_LOG_LEVEL" - envPort = "MF_INFLUX_WRITER_PORT" - envDB = "MF_INFLUXDB_DB" - envDBHost = "MF_INFLUXDB_HOST" - envDBPort = "MF_INFLUXDB_PORT" - envDBUser = "MF_INFLUXDB_ADMIN_USER" - envDBPass = "MF_INFLUXDB_ADMIN_PASSWORD" - envConfigPath = "MF_INFLUX_WRITER_CONFIG_PATH" + svcName = "influxdb-writer" + envPrefix = "MF_INFLUX_WRITER_" + envPrefixHttp = "MF_INFLUX_WRITER_HTTP_" + envPrefixInfluxdb = "MF_INFLUXDB_" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - logLevel string - port string - dbName string - dbHost string - dbPort string - dbUser string - dbPass string - configPath string + LogLevel string `env:"MF_INFLUX_WRITER_LOG_LEVEL" envDefault:"info"` + ConfigPath string `env:"MF_INFLUX_WRITER_CONFIG_PATH" envDefault:"/config.toml"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg, clientCfg := loadConfigs() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer pubSub.Close() - client, err := influxdata.NewHTTPClient(clientCfg) + influxDBConfig := influxDBClient.Config{} + if err := env.Parse(&influxDBConfig, env.Options{Prefix: envPrefixInfluxdb}); err != nil { + log.Fatalf("failed to load InfluxDB client configuration from environment variable : %s", err.Error()) + } + client, err := influxDBClient.Connect(influxDBConfig) if err != nil { - logger.Error(fmt.Sprintf("Failed to create InfluxDB client: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to InfluxDB : %s", err.Error()) } defer client.Close() - repo := influxdb.New(client, cfg.dbName) + repo := newService(client, influxDBConfig.DbName, logger) - counter, latency := makeMetrics() - repo = api.LoggingMiddleware(repo, logger) - repo = api.MetricsMiddleware(repo, counter, latency) + if err := consumers.Start(svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil { + log.Fatalf("failed to start InfluxDB writer: %s", err.Error()) + } - if err := consumers.Start(svcName, pubSub, repo, cfg.configPath, logger); err != nil { - logger.Error(fmt.Sprintf("Failed to start InfluxDB writer: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svcName), logger) g.Go(func() error { - return startHTTPService(ctx, cfg.port, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("InfluxDB reader service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -113,68 +92,10 @@ func main() { } } -func loadConfigs() (config, influxdata.HTTPConfig) { - cfg := config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - dbName: mainflux.Env(envDB, defDB), - dbHost: mainflux.Env(envDBHost, defDBHost), - dbPort: mainflux.Env(envDBPort, defDBPort), - dbUser: mainflux.Env(envDBUser, defDBUser), - dbPass: mainflux.Env(envDBPass, defDBPass), - configPath: mainflux.Env(envConfigPath, defConfigPath), - } - - clientCfg := influxdata.HTTPConfig{ - Addr: fmt.Sprintf("http://%s:%s", cfg.dbHost, cfg.dbPort), - Username: cfg.dbUser, - Password: cfg.dbPass, - } - - return cfg, clientCfg -} - -func makeMetrics() (*kitprometheus.Counter, *kitprometheus.Summary) { - counter := kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "influxdb", - Subsystem: "message_writer", - Name: "request_count", - Help: "Number of database inserts.", - }, []string{"method"}) - - latency := kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "influxdb", - Subsystem: "message_writer", - Name: "request_latency_microseconds", - Help: "Total duration of inserts in microseconds.", - }, []string{"method"}) - - return counter, latency -} - -func startHTTPService(ctx context.Context, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svcName)} - - logger.Info(fmt.Sprintf("InfluxDB writer service started, exposed port %s", p)) - - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("InfluxDB writer service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("influxDB writer service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("InfluxDB writer service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } +func newService(client influxdata.Client, dbName string, logger logger.Logger) consumers.Consumer { + repo := influxdb.New(client, dbName) + repo = api.LoggingMiddleware(repo, logger) + counter, latency := internal.MakeMetrics("influxdb", "message_writer") + repo = api.MetricsMiddleware(repo, counter, latency) + return repo } diff --git a/cmd/lora/main.go b/cmd/lora/main.go index c01644efe5..c0c9007505 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -7,61 +7,34 @@ import ( "context" "fmt" "log" - "net/http" "os" - "strconv" "time" mqttPaho "github.com/eclipse/paho.mqtt.golang" r "github.com/go-redis/redis/v8" - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/internal" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/lora" "github.com/mainflux/mainflux/lora/api" "github.com/mainflux/mainflux/lora/mqtt" - "github.com/mainflux/mainflux/pkg/errors" + "github.com/mainflux/mainflux/pkg/messaging" "github.com/mainflux/mainflux/pkg/messaging/brokers" "golang.org/x/sync/errgroup" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" + redisClient "github.com/mainflux/mainflux/internal/clients/redis" "github.com/mainflux/mainflux/lora/redis" - stdprometheus "github.com/prometheus/client_golang/prometheus" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defHTTPPort = "8180" - defLoraMsgURL = "tcp://localhost:1883" - defBrokerURL = "nats://localhost:4222" - defLoraMsgTopic = "application/+/device/+/event/up" - defLoraMsgUser = "" - defLoraMsgPass = "" - defLoraMsgTimeout = "30s" - defESURL = "localhost:6379" - defESPass = "" - defESDB = "0" - defESConsumerName = "lora" - defRouteMapURL = "localhost:6379" - defRouteMapPass = "" - defRouteMapDB = "0" - - envHTTPPort = "MF_LORA_ADAPTER_HTTP_PORT" - envLoraMsgURL = "MF_LORA_ADAPTER_MESSAGES_URL" - envBrokerURL = "MF_BROKER_URL" - envLoraMsgTopic = "MF_LORA_ADAPTER_MESSAGES_TOPIC" - envLoraMsgUser = "MF_LORA_ADAPTER_MESSAGES_USER" - envLoraMsgPass = "MF_LORA_ADAPTER_MESSAGES_PASS" - envLoraMsgTimeout = "MF_LORA_ADAPTER_MESSAGES_TIMEOUT" - envLogLevel = "MF_LORA_ADAPTER_LOG_LEVEL" - envESURL = "MF_THINGS_ES_URL" - envESPass = "MF_THINGS_ES_PASS" - envESDB = "MF_THINGS_ES_DB" - envESConsumerName = "MF_LORA_ADAPTER_EVENT_CONSUMER" - envRouteMapURL = "MF_LORA_ADAPTER_ROUTE_MAP_URL" - envRouteMapPass = "MF_LORA_ADAPTER_ROUTE_MAP_PASS" - envRouteMapDB = "MF_LORA_ADAPTER_ROUTE_MAP_DB" + svcName = "lora-adapter" + envPrefix = "MF_LORA_ADAPTER_" + envPrefixHttp = "MF_LORA_ADAPTER_HTTP_" + envPrefixRouteMap = "MF_LORA_ADAPTER_ROUTE_MAP_" + envPrefixThingsES = "MF_THINGS_ES_" + defSvcHttpPort = "8180" thingsRMPrefix = "thing" channelsRMPrefix = "channel" @@ -69,114 +42,72 @@ const ( ) type config struct { - httpPort string - loraMsgURL string - brokerURL string - loraMsgUser string - loraMsgPass string - loraMsgTopic string - loraMsgTimeout time.Duration - logLevel string - esURL string - esPass string - esDB string - esConsumerName string - routeMapURL string - routeMapPass string - routeMapDB string + LogLevel string `env:"MF_BOOTSTRAP_LOG_LEVEL" envDefault:"info"` + LoraMsgURL string `env:"MF_LORA_ADAPTER_MESSAGES_URL" envDefault:"tcp://localhost:1883"` + LoraMsgUser string `env:"MF_LORA_ADAPTER_MESSAGES_USER" envDefault:""` + LoraMsgPass string `env:"MF_LORA_ADAPTER_MESSAGES_PASS" envDefault:""` + LoraMsgTopic string `env:"MF_LORA_ADAPTER_MESSAGES_TOPIC" envDefault:"application/+/device/+/event/up"` + LoraMsgTimeout time.Duration `env:"MF_LORA_ADAPTER_MESSAGES_TIMEOUT" envDefault:"30s"` + ESConsumerName string `env:"MF_LORA_ADAPTER_EVENT_CONSUMER" envDefault:"lora"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - rmConn := connectToRedis(cfg.routeMapURL, cfg.routeMapPass, cfg.routeMapDB, logger) + rmConn, err := redisClient.Setup(envPrefixRouteMap) + if err != nil { + log.Fatalf("failed to setup route map redis client : %s", err.Error()) + } defer rmConn.Close() - esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) - defer esConn.Close() - - pub, err := brokers.NewPublisher(cfg.brokerURL) + pub, err := brokers.NewPublisher(cfg.BrokerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer pub.Close() - thingsRM := newRouteMapRepository(rmConn, thingsRMPrefix, logger) - chansRM := newRouteMapRepository(rmConn, channelsRMPrefix, logger) - connsRM := newRouteMapRepository(rmConn, connsRMPrefix, logger) + svc := newService(pub, rmConn, thingsRMPrefix, channelsRMPrefix, connsRMPrefix, logger) - svc := lora.New(pub, thingsRM, chansRM, connsRM) - svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "lora_adapter", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "lora_adapter", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + esConn, err := redisClient.Setup(envPrefixThingsES) + if err != nil { + log.Fatalf("failed to setup things event store redis client : %s", err.Error()) + } + defer esConn.Close() + + mqttConn := connectToMQTTBroker(cfg.LoraMsgURL, cfg.LoraMsgUser, cfg.LoraMsgPass, cfg.LoraMsgTimeout, logger) - mqttConn := connectToMQTTBroker(cfg.loraMsgURL, cfg.loraMsgUser, cfg.loraMsgPass, cfg.loraMsgTimeout, logger) + go subscribeToLoRaBroker(svc, mqttConn, cfg.LoraMsgTimeout, cfg.LoraMsgTopic, logger) + go subscribeToThingsES(svc, esConn, cfg.ESConsumerName, logger) - go subscribeToLoRaBroker(svc, mqttConn, cfg.loraMsgTimeout, cfg.loraMsgTopic, logger) - go subscribeToThingsES(svc, esConn, cfg.esConsumerName, logger) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) + } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(), logger) g.Go(func() error { - return startHTTPServer(ctx, cfg, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("LoRa adapter shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { logger.Error(fmt.Sprintf("LoRa adapter terminated: %s", err)) } - -} - -func loadConfig() config { - mqttTimeout, err := time.ParseDuration(mainflux.Env(envLoraMsgTimeout, defLoraMsgTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envLoraMsgTimeout, err.Error()) - } - - return config{ - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - loraMsgURL: mainflux.Env(envLoraMsgURL, defLoraMsgURL), - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - loraMsgTopic: mainflux.Env(envLoraMsgTopic, defLoraMsgTopic), - loraMsgUser: mainflux.Env(envLoraMsgUser, defLoraMsgUser), - loraMsgPass: mainflux.Env(envLoraMsgPass, defLoraMsgPass), - loraMsgTimeout: mqttTimeout, - logLevel: mainflux.Env(envLogLevel, defLogLevel), - esURL: mainflux.Env(envESURL, defESURL), - esPass: mainflux.Env(envESPass, defESPass), - esDB: mainflux.Env(envESDB, defESDB), - esConsumerName: mainflux.Env(envESConsumerName, defESConsumerName), - routeMapURL: mainflux.Env(envRouteMapURL, defRouteMapURL), - routeMapPass: mainflux.Env(envRouteMapPass, defRouteMapPass), - routeMapDB: mainflux.Env(envRouteMapDB, defRouteMapDB), - } } func connectToMQTTBroker(url, user, password string, timeout time.Duration, logger logger.Logger) mqttPaho.Client { @@ -188,40 +119,23 @@ func connectToMQTTBroker(url, user, password string, timeout time.Duration, logg logger.Info("Connected to Lora MQTT broker") }) opts.SetConnectionLostHandler(func(c mqttPaho.Client, err error) { - logger.Error(fmt.Sprintf("MQTT connection lost: %s", err.Error())) - os.Exit(1) + log.Fatalf("MQTT connection lost: %s", err.Error()) }) client := mqttPaho.NewClient(opts) if token := client.Connect(); token.WaitTimeout(timeout) && token.Error() != nil { - logger.Error(fmt.Sprintf("Failed to connect to Lora MQTT broker: %s", token.Error())) - os.Exit(1) + log.Fatalf("failed to connect to Lora MQTT broker: %s", token.Error()) } return client } -func connectToRedis(redisURL, redisPass, redisDB string, logger logger.Logger) *r.Client { - db, err := strconv.Atoi(redisDB) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to redis: %s", err)) - os.Exit(1) - } - - return r.NewClient(&r.Options{ - Addr: redisURL, - Password: redisPass, - DB: db, - }) -} - func subscribeToLoRaBroker(svc lora.Service, mc mqttPaho.Client, timeout time.Duration, topic string, logger logger.Logger) { mqtt := mqtt.NewBroker(svc, mc, timeout, logger) logger.Info("Subscribed to Lora MQTT broker") if err := mqtt.Subscribe(topic); err != nil { - logger.Error(fmt.Sprintf("Failed to subscribe to Lora MQTT broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to subscribe to Lora MQTT broker: %s", err.Error()) } } @@ -238,29 +152,15 @@ func newRouteMapRepository(client *r.Client, prefix string, logger logger.Logger return redis.NewRouteMapRepository(client, prefix) } -func startHTTPServer(ctx context.Context, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.httpPort) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler()} - - logger.Info(fmt.Sprintf("LoRa-adapter service started, exposed port %s", cfg.httpPort)) - - go func() { - errCh <- http.ListenAndServe(p, api.MakeHandler()) - }() +func newService(pub messaging.Publisher, rmConn *r.Client, thingsRMPrefix, channelsRMPrefix, connsRMPrefix string, logger logger.Logger) lora.Service { + thingsRM := newRouteMapRepository(rmConn, thingsRMPrefix, logger) + chansRM := newRouteMapRepository(rmConn, channelsRMPrefix, logger) + connsRM := newRouteMapRepository(rmConn, connsRMPrefix, logger) - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("LoRa-adapter service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("LoRa-adapter service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("LoRa-adapter service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } + svc := lora.New(pub, thingsRM, chansRM, connsRM) + svc = api.LoggingMiddleware(svc, logger) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) + return svc } diff --git a/cmd/mongodb-reader/main.go b/cmd/mongodb-reader/main.go index 41fd671cda..269e03e910 100644 --- a/cmd/mongodb-reader/main.go +++ b/cmd/mongodb-reader/main.go @@ -6,303 +6,97 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" - "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + mongoClient "github.com/mainflux/mainflux/internal/clients/mongo" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/readers" "github.com/mainflux/mainflux/readers/api" "github.com/mainflux/mainflux/readers/mongodb" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defPort = "8180" - defDB = "mainflux" - defDBHost = "localhost" - defDBPort = "27017" - defClientTLS = "false" - defCACerts = "" - defServerCert = "" - defServerKey = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - defUsersAuthURL = "localhost:8181" - defUsersAuthTimeout = "1s" - - envLogLevel = "MF_MONGO_READER_LOG_LEVEL" - envPort = "MF_MONGO_READER_PORT" - envDB = "MF_MONGO_READER_DB" - envDBHost = "MF_MONGO_READER_DB_HOST" - envDBPort = "MF_MONGO_READER_DB_PORT" - envClientTLS = "MF_MONGO_READER_CLIENT_TLS" - envCACerts = "MF_MONGO_READER_CA_CERTS" - envServerCert = "MF_MONGO_READER_SERVER_CERT" - envServerKey = "MF_MONGO_READER_SERVER_KEY" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" - envUsersAuthURL = "MF_AUTH_GRPC_URL" - envUsersAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "mongodb-reader" + envPrefix = "MF_MONGO_READER_" + envPrefixDB = "MF_MONGO_READER_DB_" + envPrefixHttp = "MF_MONGO_READER_HTTP_" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - port string - dbName string - dbHost string - dbPort string - clientTLS bool - caCerts string - serverCert string - serverKey string - jaegerURL string - thingsAuthURL string - usersAuthURL string - thingsAuthTimeout time.Duration - usersAuthTimeout time.Duration + LogLevel string `env:"MF_MONGO_READER_LOG_LEVEL" envDefault:"info"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfigs() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) - } - - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - - auth := authapi.NewClient(authTracer, authConn, cfg.usersAuthTimeout) - - db := connectToMongoDB(cfg.dbHost, cfg.dbPort, cfg.dbName, logger) - - repo := newService(db, logger) - g.Go(func() error { - return startHTTPServer(ctx, repo, tc, auth, cfg, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("MongoDB reader service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("MongoDB reader service terminated: %s", err)) - } - -} - -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - logger.Info("Connecting to auth via gRPC") - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) } - conn, err := grpc.Dial(cfg.usersAuthURL, opts...) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) + log.Fatalf(err.Error()) } - logger.Info(fmt.Sprintf("Established gRPC connection to auth via gRPC: %s", cfg.usersAuthURL)) - return conn -} -func loadConfigs() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + db, err := mongoClient.Setup(envPrefixDB) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatalf("failed to setup mongo database : %s", err.Error()) } - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) - } + repo := newService(db, logger) - usersAuthTimeout, err := time.ParseDuration(mainflux.Env(envUsersAuthTimeout, defUsersAuthTimeout)) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + log.Fatal(err.Error()) } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - dbName: mainflux.Env(envDB, defDB), - dbHost: mainflux.Env(envDBHost, defDBHost), - dbPort: mainflux.Env(envDBPort, defDBPort), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - usersAuthURL: mainflux.Env(envUsersAuthURL, defUsersAuthURL), - thingsAuthTimeout: authTimeout, - usersAuthTimeout: usersAuthTimeout, - } -} - -func connectToMongoDB(host, port, name string, logger logger.Logger) *mongo.Database { - addr := fmt.Sprintf("mongodb://%s:%s", host, port) - client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to database: %s", err)) - os.Exit(1) + log.Fatal(err.Error()) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - return client.Database(name) -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(repo, tc, auth, svcName, logger), logger) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) - } + g.Go(func() error { + return hs.Start() + }) - return tracer, closer -} + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("MongoDB reader service terminated: %s", err)) } - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) - } - logger.Info(fmt.Sprintf("Established gRPC connection to things via gRPC: %s", cfg.thingsAuthURL)) - return conn } func newService(db *mongo.Database, logger logger.Logger) readers.MessageRepository { repo := mongodb.New(db) repo = api.LoggingMiddleware(repo, logger) - repo = api.MetricsMiddleware( - repo, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "mongodb", - Subsystem: "message_reader", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "mongodb", - Subsystem: "message_reader", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics("mongodb", "message_reader") + repo = api.MetricsMiddleware(repo, counter, latency) return repo } - -func startHTTPServer(ctx context.Context, repo readers.MessageRepository, tc mainflux.ThingsServiceClient, ac mainflux.AuthServiceClient, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(repo, tc, ac, "mongodb-reader", logger)} - - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("Mongo reader service started using https on port %s with cert %s key %s", - cfg.port, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - default: - logger.Info(fmt.Sprintf("Mongo reader service started, exposed port %s", cfg.port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("MongoDB reader service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("mongodb reader service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("MongoDB reader service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index a1f1210140..5f3d592933 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -7,162 +7,90 @@ import ( "context" "fmt" "log" - "net/http" "os" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/writers/api" "github.com/mainflux/mainflux/consumers/writers/mongodb" + "github.com/mainflux/mainflux/internal" + mongoClient "github.com/mainflux/mainflux/internal/clients/mongo" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" - stdprometheus "github.com/prometheus/client_golang/prometheus" "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "golang.org/x/sync/errgroup" ) const ( - svcName = "mongodb-writer" - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defBrokerURL = "nats://localhost:4222" - defPort = "8180" - defDB = "mainflux" - defDBHost = "localhost" - defDBPort = "27017" - defConfigPath = "/config.toml" - - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_MONGO_WRITER_LOG_LEVEL" - envPort = "MF_MONGO_WRITER_PORT" - envDB = "MF_MONGO_WRITER_DB" - envDBHost = "MF_MONGO_WRITER_DB_HOST" - envDBPort = "MF_MONGO_WRITER_DB_PORT" - envConfigPath = "MF_MONGO_WRITER_CONFIG_PATH" + svcName = "mongodb-writer" + envPrefix = "MF_MONGO_WRITER_" + envPrefixDB = "MF_MONGO_WRITER_DB_" + envPrefixHttp = "MF_MONGO_WRITER_HTTP_" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - logLevel string - port string - dbName string - dbHost string - dbPort string - configPath string + LogLevel string `env:"MF_MONGO_WRITER_LOG_LEVEL" envDefault:"info"` + ConfigPath string `env:"MF_MONGO_WRITER_CONFIG_PATH" envDefault:"/config.toml"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfigs() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatal(err) } - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer pubSub.Close() - addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) - client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) + db, err := mongoClient.Setup(envPrefixDB) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to database: %s", err)) - os.Exit(1) + log.Fatalf("failed to setup mongo database : %s", err.Error()) } - db := client.Database(cfg.dbName) - repo := mongodb.New(db) + repo := newService(db, logger) - counter, latency := makeMetrics() - repo = api.LoggingMiddleware(repo, logger) - repo = api.MetricsMiddleware(repo, counter, latency) + if err := consumers.Start(svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil { + log.Fatalf("failed to start MongoDB writer: %s", err.Error()) + } - if err := consumers.Start(svcName, pubSub, repo, cfg.configPath, logger); err != nil { - logger.Error(fmt.Sprintf("Failed to start MongoDB writer: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svcName), logger) g.Go(func() error { - return startHTTPService(ctx, cfg.port, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("MongoDB reader service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { logger.Error(fmt.Sprintf("MongoDB writer service terminated: %s", err)) } - } -func loadConfigs() config { - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - dbName: mainflux.Env(envDB, defDB), - dbHost: mainflux.Env(envDBHost, defDBHost), - dbPort: mainflux.Env(envDBPort, defDBPort), - configPath: mainflux.Env(envConfigPath, defConfigPath), - } -} - -func makeMetrics() (*kitprometheus.Counter, *kitprometheus.Summary) { - counter := kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "mongodb", - Subsystem: "message_writer", - Name: "request_count", - Help: "Number of database inserts.", - }, []string{"method"}) - - latency := kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "mongodb", - Subsystem: "message_writer", - Name: "request_latency_microseconds", - Help: "Total duration of inserts in microseconds.", - }, []string{"method"}) - - return counter, latency -} - -func startHTTPService(ctx context.Context, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svcName)} - - logger.Info(fmt.Sprintf("MongoDB writer service started, exposed port %s", p)) - - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("MongoDB writer service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("mongodb writer service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("MongoDB writer service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } - +func newService(db *mongo.Database, logger logger.Logger) consumers.Consumer { + repo := mongodb.New(db) + repo = api.LoggingMiddleware(repo, logger) + counter, latency := internal.MakeMetrics("mongodb", "message_writer") + repo = api.MetricsMiddleware(repo, counter, latency) + return repo } diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index d479a51aaa..6aa18d3b60 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -4,16 +4,15 @@ import ( "context" "fmt" "io" - "io/ioutil" "log" "net/http" "os" - "strconv" "time" "github.com/cenkalti/backoff/v4" - "github.com/go-redis/redis/v8" - "github.com/mainflux/mainflux" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + redisClient "github.com/mainflux/mainflux/internal/clients/redis" + "github.com/mainflux/mainflux/internal/env" mflog "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/mqtt" mqttredis "github.com/mainflux/mainflux/mqtt/redis" @@ -22,170 +21,113 @@ import ( "github.com/mainflux/mainflux/pkg/messaging" "github.com/mainflux/mainflux/pkg/messaging/brokers" mqttpub "github.com/mainflux/mainflux/pkg/messaging/mqtt" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" mp "github.com/mainflux/mproxy/pkg/mqtt" "github.com/mainflux/mproxy/pkg/session" ws "github.com/mainflux/mproxy/pkg/websocket" - opentracing "github.com/opentracing/opentracing-go" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - svcName = "mqtt" - - defLogLevel = "error" - defMQTTPort = "1883" - defMQTTTargetHost = "0.0.0.0" - defMQTTTargetPort = "1883" - defMQTTForwarderTimeout = "30s" // 30 seconds - defMQTTTargetHealthCheck = "" - defHTTPPort = "8080" - defHTTPTargetHost = "localhost" - defHTTPTargetPort = "8080" - defHTTPTargetPath = "/mqtt" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - defBrokerURL = "nats://localhost:4222" - defJaegerURL = "" - defClientTLS = "false" - defCACerts = "" - defInstance = "" - defESURL = "localhost:6379" - defESPass = "" - defESDB = "0" - defAuthcacheURL = "localhost:6379" - defAuthCachePass = "" - defAuthCacheDB = "0" - - envLogLevel = "MF_MQTT_ADAPTER_LOG_LEVEL" - envMQTTPort = "MF_MQTT_ADAPTER_MQTT_PORT" - envMQTTTargetHost = "MF_MQTT_ADAPTER_MQTT_TARGET_HOST" - envMQTTTargetPort = "MF_MQTT_ADAPTER_MQTT_TARGET_PORT" - envMQTTTargetHealthCheck = "MF_MQTT_ADAPTER_MQTT_TARGET_HEALTH_CHECK" - envMQTTForwarderTimeout = "MF_MQTT_ADAPTER_FORWARDER_TIMEOUT" - envHTTPPort = "MF_MQTT_ADAPTER_WS_PORT" - envHTTPTargetHost = "MF_MQTT_ADAPTER_WS_TARGET_HOST" - envHTTPTargetPort = "MF_MQTT_ADAPTER_WS_TARGET_PORT" - envHTTPTargetPath = "MF_MQTT_ADAPTER_WS_TARGET_PATH" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" - envBrokerURL = "MF_BROKER_URL" - envJaegerURL = "MF_JAEGER_URL" - envClientTLS = "MF_MQTT_ADAPTER_CLIENT_TLS" - envCACerts = "MF_MQTT_ADAPTER_CA_CERTS" - envInstance = "MF_MQTT_ADAPTER_INSTANCE" - envESURL = "MF_MQTT_ADAPTER_ES_URL" - envESPass = "MF_MQTT_ADAPTER_ES_PASS" - envESDB = "MF_MQTT_ADAPTER_ES_DB" - envAuthCacheURL = "MF_AUTH_CACHE_URL" - envAuthCachePass = "MF_AUTH_CACHE_PASS" - envAuthCacheDB = "MF_AUTH_CACHE_DB" + svcName = "mqtt" + envPrefix = "MF_MQTT_ADAPTER_" + envPrefixES = "MF_MQTT_ADAPTER_ES_" + envPrefixAuthCache = "MF_AUTH_CACHE_" ) type config struct { - mqttPort string - mqttTargetHost string - mqttTargetPort string - mqttForwarderTimeout time.Duration - mqttTargetHealthCheck string - httpPort string - httpTargetHost string - httpTargetPort string - httpTargetPath string - jaegerURL string - logLevel string - thingsURL string - thingsAuthURL string - thingsAuthTimeout time.Duration - brokerURL string - clientTLS bool - caCerts string - instance string - esURL string - esPass string - esDB string - authURL string - authPass string - authDB string + LogLevel string `env:"MF_MQTT_ADAPTER_LOG_LEVEL" envDefault:"info"` + MqttPort string `env:"MF_MQTT_ADAPTER_MQTT_PORT" envDefault:"1883"` + MqttTargetHost string `env:"MF_MQTT_ADAPTER_MQTT_TARGET_HOST" envDefault:"localhost"` + MqttTargetPort string `env:"MF_MQTT_ADAPTER_MQTT_TARGET_PORT" envDefault:"1883"` + MqttForwarderTimeout time.Duration `env:"MF_MQTT_ADAPTER_FORWARDER_TIMEOUT" envDefault:"30s"` + MqttTargetHealthCheck string `env:"MF_MQTT_ADAPTER_MQTT_TARGET_HEALTH_CHECK" envDefault:""` + HttpPort string `env:"MF_MQTT_ADAPTER_WS_PORT" envDefault:"8080"` + HttpTargetHost string `env:"MF_MQTT_ADAPTER_WS_TARGET_HOST" envDefault:"localhost"` + HttpTargetPort string `env:"MF_MQTT_ADAPTER_WS_TARGET_PORT" envDefault:"8080"` + HttpTargetPath string `env:"MF_MQTT_ADAPTER_WS_TARGET_PATH" envDefault:"/mqtt"` + Instance string `env:"MF_MQTT_ADAPTER_INSTANCE" envDefault:""` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := mflog.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := mflog.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - if cfg.mqttTargetHealthCheck != "" { + if cfg.MqttTargetHealthCheck != "" { notify := func(e error, next time.Duration) { logger.Info(fmt.Sprintf("Broker not ready: %s, next try in %s", e.Error(), next)) } err := backoff.RetryNotify(healthcheck(cfg), backoff.NewExponentialBackOff(), notify) if err != nil { - logger.Info(fmt.Sprintf("MQTT healthcheck limit exceeded, exiting. %s ", err.Error())) - os.Exit(1) + log.Fatalf("MQTT healthcheck limit exceeded, exiting. %s ", err.Error()) } } - conn := connectToThings(cfg, logger) - defer conn.Close() - - ec := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) - defer ec.Close() - - nps, err := brokers.NewPubSub(cfg.brokerURL, "mqtt", logger) + nps, err := brokers.NewPubSub(cfg.BrokerURL, "mqtt", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer nps.Close() - mpub, err := mqttpub.NewPublisher(fmt.Sprintf("%s:%s", cfg.mqttTargetHost, cfg.mqttTargetPort), cfg.mqttForwarderTimeout) + mpub, err := mqttpub.NewPublisher(fmt.Sprintf("%s:%s", cfg.MqttTargetHost, cfg.MqttTargetPort), cfg.MqttForwarderTimeout) if err != nil { - logger.Error(fmt.Sprintf("Failed to create MQTT publisher: %s", err)) - os.Exit(1) + log.Fatalf("failed to create MQTT publisher: %s", err.Error()) } fwd := mqtt.NewForwarder(brokers.SubjectAllChannels, logger) if err := fwd.Forward(svcName, nps, mpub); err != nil { - logger.Error(fmt.Sprintf("Failed to forward message broker messages: %s", err)) - os.Exit(1) + log.Fatalf("failed to forward message broker messages: %s", err) } - np, err := brokers.NewPublisher(cfg.brokerURL) + np, err := brokers.NewPublisher(cfg.BrokerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer np.Close() - es := mqttredis.NewEventStore(ec, cfg.instance) + ec, err := redisClient.Setup(envPrefixES) + if err != nil { + log.Fatalf("failed to setup %s event store redis client : %s", svcName, err.Error()) + } + defer ec.Close() - ac := connectToRedis(cfg.authURL, cfg.authPass, cfg.authDB, logger) + es := mqttredis.NewEventStore(ec, cfg.Instance) + + ac, err := redisClient.Setup(envPrefixAuthCache) + if err != nil { + log.Fatalf("failed to setup %s event store redis client : %s", svcName, err.Error()) + } defer ac.Close() - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err.Error()) + } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) authClient := auth.New(ac, tc) - // Event handler for MQTT hooks h := mqtt.NewHandler([]messaging.Publisher{np}, es, logger, authClient) - logger.Info(fmt.Sprintf("Starting MQTT proxy on port %s", cfg.mqttPort)) + logger.Info(fmt.Sprintf("Starting MQTT proxy on port %s", cfg.MqttPort)) g.Go(func() error { return proxyMQTT(ctx, cfg, logger, h) }) - logger.Info(fmt.Sprintf("Starting MQTT over WS proxy on port %s", cfg.httpPort)) + logger.Info(fmt.Sprintf("Starting MQTT over WS proxy on port %s", cfg.HttpPort)) g.Go(func() error { return proxyWS(ctx, cfg, logger, h) }) @@ -203,115 +145,9 @@ func main() { } } -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) - if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) - } - - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) - } - - mqttTimeout, err := time.ParseDuration(mainflux.Env(envMQTTForwarderTimeout, defMQTTForwarderTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envMQTTForwarderTimeout, err.Error()) - } - - return config{ - mqttPort: mainflux.Env(envMQTTPort, defMQTTPort), - mqttTargetHost: mainflux.Env(envMQTTTargetHost, defMQTTTargetHost), - mqttTargetPort: mainflux.Env(envMQTTTargetPort, defMQTTTargetPort), - mqttForwarderTimeout: mqttTimeout, - mqttTargetHealthCheck: mainflux.Env(envMQTTTargetHealthCheck, defMQTTTargetHealthCheck), - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - httpTargetHost: mainflux.Env(envHTTPTargetHost, defHTTPTargetHost), - httpTargetPort: mainflux.Env(envHTTPTargetPort, defHTTPTargetPort), - httpTargetPath: mainflux.Env(envHTTPTargetPath, defHTTPTargetPath), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - thingsAuthTimeout: authTimeout, - thingsURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - instance: mainflux.Env(envInstance, defInstance), - esURL: mainflux.Env(envESURL, defESURL), - esPass: mainflux.Env(envESPass, defESPass), - esDB: mainflux.Env(envESDB, defESDB), - authURL: mainflux.Env(envAuthCacheURL, defAuthcacheURL), - authPass: mainflux.Env(envAuthCachePass, defAuthCachePass), - authDB: mainflux.Env(envAuthCacheDB, defAuthCacheDB), - } -} - -func initJaeger(svcName, url string, logger mflog.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } - - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) - } - - return tracer, closer -} - -func connectToThings(cfg config, logger mflog.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } - - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) - } - return conn -} - -func connectToRedis(redisURL, redisPass, redisDB string, logger mflog.Logger) *redis.Client { - db, err := strconv.Atoi(redisDB) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to redis: %s", err)) - os.Exit(1) - } - - return redis.NewClient(&redis.Options{ - Addr: redisURL, - Password: redisPass, - DB: db, - }) -} - func proxyMQTT(ctx context.Context, cfg config, logger mflog.Logger, handler session.Handler) error { - address := fmt.Sprintf(":%s", cfg.mqttPort) - target := fmt.Sprintf("%s:%s", cfg.mqttTargetHost, cfg.mqttTargetPort) + address := fmt.Sprintf(":%s", cfg.MqttPort) + target := fmt.Sprintf("%s:%s", cfg.MqttTargetHost, cfg.MqttTargetPort) mp := mp.New(address, target, handler, logger) errCh := make(chan error) @@ -326,17 +162,17 @@ func proxyMQTT(ctx context.Context, cfg config, logger mflog.Logger, handler ses case err := <-errCh: return err } - } + func proxyWS(ctx context.Context, cfg config, logger mflog.Logger, handler session.Handler) error { - target := fmt.Sprintf("%s:%s", cfg.httpTargetHost, cfg.httpTargetPort) - wp := ws.New(target, cfg.httpTargetPath, "ws", handler, logger) + target := fmt.Sprintf("%s:%s", cfg.HttpTargetHost, cfg.HttpTargetPort) + wp := ws.New(target, cfg.HttpTargetPath, "ws", handler, logger) http.Handle("/mqtt", wp.Handler()) errCh := make(chan error) go func() { - errCh <- wp.Listen(cfg.httpPort) + errCh <- wp.Listen(cfg.HttpPort) }() select { @@ -350,12 +186,12 @@ func proxyWS(ctx context.Context, cfg config, logger mflog.Logger, handler sessi func healthcheck(cfg config) func() error { return func() error { - res, err := http.Get(cfg.mqttTargetHealthCheck) + res, err := http.Get(cfg.MqttTargetHealthCheck) if err != nil { return err } defer res.Body.Close() - body, err := ioutil.ReadAll(res.Body) + body, err := io.ReadAll(res.Body) if err != nil { return err } diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index 9999b7d3c0..7d5d80aabe 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -7,61 +7,31 @@ import ( "context" "fmt" "log" - "net/http" "os" - "strconv" - "time" r "github.com/go-redis/redis/v8" - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/internal" + redisClient "github.com/mainflux/mainflux/internal/clients/redis" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/opcua" "github.com/mainflux/mainflux/opcua/api" "github.com/mainflux/mainflux/opcua/db" "github.com/mainflux/mainflux/opcua/gopcua" "github.com/mainflux/mainflux/opcua/redis" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" "golang.org/x/sync/errgroup" - - kitprometheus "github.com/go-kit/kit/metrics/prometheus" - stdprometheus "github.com/prometheus/client_golang/prometheus" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defHTTPPort = "8180" - defOPCIntervalMs = "1000" - defOPCPolicy = "" - defOPCMode = "" - defOPCCertFile = "" - defOPCKeyFile = "" - defBrokerURL = "nats://localhost:4222" - defESURL = "localhost:6379" - defESPass = "" - defESDB = "0" - defESConsumerName = "opcua" - defRouteMapURL = "localhost:6379" - defRouteMapPass = "" - defRouteMapDB = "0" - - envLogLevel = "MF_OPCUA_ADAPTER_LOG_LEVEL" - envHTTPPort = "MF_OPCUA_ADAPTER_HTTP_PORT" - envOPCIntervalMs = "MF_OPCUA_ADAPTER_INTERVAL_MS" - envOPCPolicy = "MF_OPCUA_ADAPTER_POLICY" - envOPCMode = "MF_OPCUA_ADAPTER_MODE" - envOPCCertFile = "MF_OPCUA_ADAPTER_CERT_FILE" - envOPCKeyFile = "MF_OPCUA_ADAPTER_KEY_FILE" - envBrokerURL = "MF_BROKER_URL" - envESURL = "MF_THINGS_ES_URL" - envESPass = "MF_THINGS_ES_PASS" - envESDB = "MF_THINGS_ES_DB" - envESConsumerName = "MF_OPCUA_ADAPTER_EVENT_CONSUMER" - envRouteMapURL = "MF_OPCUA_ADAPTER_ROUTE_MAP_URL" - envRouteMapPass = "MF_OPCUA_ADAPTER_ROUTE_MAP_PASS" - envRouteMapDB = "MF_OPCUA_ADAPTER_ROUTE_MAP_DB" + svcName = "opc-ua-adapter" + envPrefix = "MF_OPCUA_ADAPTER_" + envPrefixES = "MF_OPCUA_ADAPTER_ES_" + envPrefixHttp = "MF_OPCUA_ADAPTER_HTTP_" + envPrefixRouteMap = "MF_OPCUA_ADAPTER_ROUTE_MAP_" + defSvcHttpPort = "8180" thingsRMPrefix = "thing" channelsRMPrefix = "channel" @@ -69,43 +39,49 @@ const ( ) type config struct { - httpPort string - opcuaConfig opcua.Config - brokerURL string - logLevel string - esURL string - esPass string - esDB string - esConsumerName string - routeMapURL string - routeMapPass string - routeMapDB string + LogLevel string `env:"MF_OPCUA_ADAPTER_LOG_LEVEL" envDefault:"info"` + ESConsumerName string `env:"MF_OPCUA_ADAPTER_EVENT_CONSUMER" envDefault:""` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfig() httpCtx, httpCancel := context.WithCancel(context.Background()) g, httpCtx := errgroup.WithContext(httpCtx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + opcConfig := opcua.Config{} + if err := env.Parse(&opcConfig); err != nil { + log.Fatalf("failed to load %s opcua client configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - rmConn := connectToRedis(cfg.routeMapURL, cfg.routeMapPass, cfg.routeMapDB, logger) + rmConn, err := redisClient.Setup(envPrefixRouteMap) + if err != nil { + log.Fatalf("failed to setup %s bootstrap event store redis client : %s", svcName, err.Error()) + } defer rmConn.Close() thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) chanRM := newRouteMapRepositoy(rmConn, channelsRMPrefix, logger) connRM := newRouteMapRepositoy(rmConn, connectionRMPrefix, logger) - esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) + esConn, err := redisClient.Setup(envPrefixES) + if err != nil { + log.Fatalf("failed to setup %s bootstrap event store redis client : %s", svcName, err.Error()) + } defer esConn.Close() - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer pubSub.Close() @@ -113,37 +89,23 @@ func main() { sub := gopcua.NewSubscriber(ctx, pubSub, thingRM, chanRM, connRM, logger) browser := gopcua.NewBrowser(ctx, logger) - svc := opcua.New(sub, browser, thingRM, chanRM, connRM, cfg.opcuaConfig, logger) - svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "opc_adapter", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "opc_adapter", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - - go subscribeToStoredSubs(sub, cfg.opcuaConfig, logger) - go subscribeToThingsES(svc, esConn, cfg.esConsumerName, logger) + svc := newService(sub, browser, thingRM, chanRM, connRM, opcConfig, logger) + + go subscribeToStoredSubs(sub, opcConfig, logger) + go subscribeToThingsES(svc, esConn, cfg.ESConsumerName, logger) + + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) + } + hs := httpserver.New(httpCtx, httpCancel, svcName, httpServerConfig, api.MakeHandler(svc, logger), logger) g.Go(func() error { - return startHTTPServer(httpCtx, svc, cfg, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(httpCtx); sig != nil { - httpCancel() - logger.Info(fmt.Sprintf("OPC-UA adapter service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(httpCtx, httpCancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -151,43 +113,6 @@ func main() { } } -func loadConfig() config { - oc := opcua.Config{ - Interval: mainflux.Env(envOPCIntervalMs, defOPCIntervalMs), - Policy: mainflux.Env(envOPCPolicy, defOPCPolicy), - Mode: mainflux.Env(envOPCMode, defOPCMode), - CertFile: mainflux.Env(envOPCCertFile, defOPCCertFile), - KeyFile: mainflux.Env(envOPCKeyFile, defOPCKeyFile), - } - return config{ - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - opcuaConfig: oc, - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - esURL: mainflux.Env(envESURL, defESURL), - esPass: mainflux.Env(envESPass, defESPass), - esDB: mainflux.Env(envESDB, defESDB), - esConsumerName: mainflux.Env(envESConsumerName, defESConsumerName), - routeMapURL: mainflux.Env(envRouteMapURL, defRouteMapURL), - routeMapPass: mainflux.Env(envRouteMapPass, defRouteMapPass), - routeMapDB: mainflux.Env(envRouteMapDB, defRouteMapDB), - } -} - -func connectToRedis(redisURL, redisPass, redisDB string, logger logger.Logger) *r.Client { - db, err := strconv.Atoi(redisDB) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to redis: %s", err)) - os.Exit(1) - } - - return r.NewClient(&r.Options{ - Addr: redisURL, - Password: redisPass, - DB: db, - }) -} - func subscribeToStoredSubs(sub opcua.Subscriber, cfg opcua.Config, logger logger.Logger) { // Get all stored subscriptions nodes, err := db.ReadAll() @@ -218,28 +143,11 @@ func newRouteMapRepositoy(client *r.Client, prefix string, logger logger.Logger) return redis.NewRouteMapRepository(client, prefix) } -func startHTTPServer(ctx context.Context, svc opcua.Service, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.httpPort) - - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, logger)} - logger.Info(fmt.Sprintf("OPC-UA adapter service started, exposed port %s", cfg.httpPort)) - - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("OPC-UA adapter service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("OPC-UA adapter service error occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("OPC-UA adapter service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } +func newService(sub opcua.Subscriber, browser opcua.Browser, thingRM, chanRM, connRM opcua.RouteMapRepository, opcuaConfig opcua.Config, logger logger.Logger) opcua.Service { + svc := opcua.New(sub, browser, thingRM, chanRM, connRM, opcuaConfig, logger) + svc = api.LoggingMiddleware(svc, logger) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) + + return svc } diff --git a/cmd/postgres-reader/main.go b/cmd/postgres-reader/main.go index 0385a27712..f47fcababe 100644 --- a/cmd/postgres-reader/main.go +++ b/cmd/postgres-reader/main.go @@ -6,305 +6,101 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" - "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/readers" "github.com/mainflux/mainflux/readers/api" "github.com/mainflux/mainflux/readers/postgres" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - svcName = "postgres-reader" - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defPort = "8180" - defClientTLS = "false" - defCACerts = "" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "mainflux" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - defUsersAuthURL = "localhost:8181" - defUsersAuthTimeout = "1s" - - envLogLevel = "MF_POSTGRES_READER_LOG_LEVEL" - envPort = "MF_POSTGRES_READER_PORT" - envClientTLS = "MF_POSTGRES_READER_CLIENT_TLS" - envCACerts = "MF_POSTGRES_READER_CA_CERTS" - envDBHost = "MF_POSTGRES_READER_DB_HOST" - envDBPort = "MF_POSTGRES_READER_DB_PORT" - envDBUser = "MF_POSTGRES_READER_DB_USER" - envDBPass = "MF_POSTGRES_READER_DB_PASS" - envDB = "MF_POSTGRES_READER_DB" - envDBSSLMode = "MF_POSTGRES_READER_DB_SSL_MODE" - envDBSSLCert = "MF_POSTGRES_READER_DB_SSL_CERT" - envDBSSLKey = "MF_POSTGRES_READER_DB_SSL_KEY" - envDBSSLRootCert = "MF_POSTGRES_READER_DB_SSL_ROOT_CERT" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" - envUsersAuthURL = "MF_AUTH_GRPC_URL" - envUsersAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "postgres-reader" + envPrefix = "MF_POSTGRES_READER_" + envPrefixHttp = "MF_POSTGRES_READER_HTTP_" + defDB = "messages" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - port string - clientTLS bool - caCerts string - dbConfig postgres.Config - jaegerURL string - thingsAuthURL string - usersAuthURL string - thingsAuthTimeout time.Duration - usersAuthTimeout time.Duration + LogLevel string `env:"MF_POSTGRES_READER_LOG_LEVEL" envDefault:"info"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) - } - - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - - auth := authapi.NewClient(authTracer, authConn, cfg.usersAuthTimeout) - - db := connectToDB(cfg.dbConfig, logger) - defer db.Close() - - repo := newService(db, logger) - - g.Go(func() error { - return startHTTPServer(ctx, repo, tc, auth, cfg.port, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Postgres reader service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Postgres reader service terminated: %s", err)) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) } -} -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - logger.Info("Connecting to auth via gRPC") - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } - - conn, err := grpc.Dial(cfg.usersAuthURL, opts...) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) - } - logger.Info(fmt.Sprintf("Established gRPC connection to auth via gRPC: %s", cfg.usersAuthURL)) - return conn -} - -func loadConfig() config { - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), - } - - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) - if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatalf(err.Error()) } - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + log.Fatal(err.Error()) } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - usersAuthTimeout, err := time.ParseDuration(mainflux.Env(envUsersAuthTimeout, defUsersAuthTimeout)) + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + log.Fatal(err.Error()) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - dbConfig: dbConfig, - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - usersAuthURL: mainflux.Env(envUsersAuthURL, defUsersAuthURL), - thingsAuthTimeout: authTimeout, - usersAuthTimeout: usersAuthTimeout, + dbConfig := pgClient.Config{Name: defDB} + if err := dbConfig.LoadEnv(envPrefix); err != nil { + log.Fatal(err.Error()) } -} - -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) + db, err := pgClient.Connect(dbConfig) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to Postgres: %s", err)) - os.Exit(1) + log.Fatalf("failed to setup postgres database : %s", err.Error()) } - return db -} + defer db.Close() -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } + repo := newService(db, logger) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(repo, tc, auth, svcName, logger), logger) - return tracer, closer -} + g.Go(func() error { + return hs.Start() + }) -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Postgres reader service terminated: %s", err)) } - logger.Info(fmt.Sprintf("Established gRPC connection to things via gRPC: %s", cfg.thingsAuthURL)) - return conn } func newService(db *sqlx.DB, logger logger.Logger) readers.MessageRepository { svc := postgres.New(db) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "postgres", - Subsystem: "message_reader", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "postgres", - Subsystem: "message_reader", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics("postgres", "message_reader") + svc = api.MetricsMiddleware(svc, counter, latency) return svc } - -func startHTTPServer(ctx context.Context, repo readers.MessageRepository, tc mainflux.ThingsServiceClient, ac mainflux.AuthServiceClient, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(repo, tc, ac, svcName, logger)} - - logger.Info(fmt.Sprintf("Postgres reader service started, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Postgres reader service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("postgres reader service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Postgres reader service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index b2af7f1aa4..569f98df37 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -7,182 +7,92 @@ import ( "context" "fmt" "log" - "net/http" "os" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/writers/api" - "github.com/mainflux/mainflux/consumers/writers/postgres" + writerPg "github.com/mainflux/mainflux/consumers/writers/postgres" + "github.com/mainflux/mainflux/internal" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" - stdprometheus "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) const ( - svcName = "postgres-writer" - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defBrokerURL = "nats://localhost:4222" - defPort = "8180" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "mainflux" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defConfigPath = "/config.toml" - - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_POSTGRES_WRITER_LOG_LEVEL" - envPort = "MF_POSTGRES_WRITER_PORT" - envDBHost = "MF_POSTGRES_WRITER_DB_HOST" - envDBPort = "MF_POSTGRES_WRITER_DB_PORT" - envDBUser = "MF_POSTGRES_WRITER_DB_USER" - envDBPass = "MF_POSTGRES_WRITER_DB_PASS" - envDB = "MF_POSTGRES_WRITER_DB" - envDBSSLMode = "MF_POSTGRES_WRITER_DB_SSL_MODE" - envDBSSLCert = "MF_POSTGRES_WRITER_DB_SSL_CERT" - envDBSSLKey = "MF_POSTGRES_WRITER_DB_SSL_KEY" - envDBSSLRootCert = "MF_POSTGRES_WRITER_DB_SSL_ROOT_CERT" - envConfigPath = "MF_POSTGRES_WRITER_CONFIG_PATH" + svcName = "postgres-writer" + envPrefix = "MF_POSTGRES_WRITER_" + envPrefixHttp = "MF_POSTGRES_WRITER_HTTP_" + defDB = "messages" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - logLevel string - port string - configPath string - dbConfig postgres.Config + LogLevel string `env:"MF_POSTGRES_WRITER_LOG_LEVEL" envDefault:"info"` + ConfigPath string `env:"MF_POSTGRES_WRITER_CONFIG_PATH" envDefault:"/config.toml"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } defer pubSub.Close() - db := connectToDB(cfg.dbConfig, logger) + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *writerPg.Migration(), dbConfig) + if err != nil { + log.Fatal(err) + } defer db.Close() repo := newService(db, logger) - if err = consumers.Start(svcName, pubSub, repo, cfg.configPath, logger); err != nil { - logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) + if err = consumers.Start(svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil { + log.Fatalf("failed to create Postgres writer: %s", err.Error()) + } + + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svcName), logger) g.Go(func() error { - return startHTTPServer(ctx, cfg.port, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Postgres writer service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { logger.Error(fmt.Sprintf("Postgres writer service terminated: %s", err)) } - -} - -func loadConfig() config { - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), - } - - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - configPath: mainflux.Env(envConfigPath, defConfigPath), - dbConfig: dbConfig, - } -} - -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to Postgres: %s", err)) - os.Exit(1) - } - return db } func newService(db *sqlx.DB, logger logger.Logger) consumers.Consumer { - svc := postgres.New(db) + svc := writerPg.New(db) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "postgres", - Subsystem: "message_writer", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "postgres", - Subsystem: "message_writer", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - + counter, latency := internal.MakeMetrics("postgres", "message_writer") + svc = api.MetricsMiddleware(svc, counter, latency) return svc } - -func startHTTPServer(ctx context.Context, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svcName)} - - logger.Info(fmt.Sprintf("Postgres writer service started, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Postgres writer service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("postgres writer service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Postgres writer service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/provision/main.go b/cmd/provision/main.go index eceda9c25e..4ecac8ff98 100644 --- a/cmd/provision/main.go +++ b/cmd/provision/main.go @@ -5,13 +5,13 @@ import ( "encoding/json" "fmt" "log" - "net/http" "os" "reflect" "strconv" - "time" "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/pkg/errors" mfSDK "github.com/mainflux/mainflux/pkg/sdk/go" @@ -22,7 +22,7 @@ import ( ) const ( - stopWaitTime = 5 * time.Second + svcName = "provision" defLogLevel = "error" defConfigFile = "config.toml" @@ -110,16 +110,15 @@ func main() { svc := provision.New(cfg, SDK, logger) svc = api.NewLoggingMiddleware(svc, logger) + httpServerConfig := server.Config{Host: "", Port: cfg.Server.HTTPPort, KeyFile: cfg.Server.ServerKey, CertFile: cfg.Server.ServerCert} + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger), logger) + g.Go(func() error { - return startHTTPServer(ctx, svc, cfg, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Provision service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -128,40 +127,6 @@ func main() { } -func startHTTPServer(ctx context.Context, svc provision.Service, cfg provision.Config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.Server.HTTPPort) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, logger)} - - switch { - case cfg.Server.ServerCert != "" || cfg.Server.ServerKey != "": - logger.Info(fmt.Sprintf("Provision service started using https on port %s with cert %s key %s", - cfg.Server.HTTPPort, cfg.Server.ServerCert, cfg.Server.ServerKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.Server.ServerCert, cfg.Server.ServerKey) - }() - default: - logger.Info(fmt.Sprintf("Provision service started using http on port %s", cfg.Server.HTTPPort)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Provision service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("provision service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Provision service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} - func loadConfigFromFile(file string) (provision.Config, error) { _, err := os.Stat(file) if os.IsNotExist(err) { diff --git a/cmd/smpp-notifier/main.go b/cmd/smpp-notifier/main.go index d2b2d451ac..5c8a11de75 100644 --- a/cmd/smpp-notifier/main.go +++ b/cmd/smpp-notifier/main.go @@ -6,363 +6,134 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/notifiers" "github.com/mainflux/mainflux/consumers/notifiers/api" - "github.com/mainflux/mainflux/consumers/notifiers/postgres" + notifierPg "github.com/mainflux/mainflux/consumers/notifiers/postgres" + "github.com/mainflux/mainflux/internal" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "golang.org/x/sync/errgroup" mfsmpp "github.com/mainflux/mainflux/consumers/notifiers/smpp" "github.com/mainflux/mainflux/consumers/notifiers/tracing" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" "github.com/mainflux/mainflux/pkg/ulid" opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - svcName = "smpp-notifier" - stopWaitTime = 5 * time.Second - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "subscriptions" - defConfigPath = "/config.toml" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defHTTPPort = "8907" - defServerCert = "" - defServerKey = "" - defFrom = "" - defJaegerURL = "" - defBrokerURL = "nats://localhost:4222" - - defSmppAddress = "" - defSmppUsername = "" - defSmppPassword = "" - defSmppSystemType = "" - defSmppSrcAddrTON = "0" - defSmppDstAddrTON = "0" - defSmppSrcAddrNPI = "0" - defSmppDstAddrNPI = "0" - - defAuthTLS = "false" - defAuthCACerts = "" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - envLogLevel = "MF_SMPP_NOTIFIER_LOG_LEVEL" - envDBHost = "MF_SMPP_NOTIFIER_DB_HOST" - envDBPort = "MF_SMPP_NOTIFIER_DB_PORT" - envDBUser = "MF_SMPP_NOTIFIER_DB_USER" - envDBPass = "MF_SMPP_NOTIFIER_DB_PASS" - envDB = "MF_SMPP_NOTIFIER_DB" - envConfigPath = "MF_SMPP_NOTIFIER_WRITER_CONFIG_PATH" - envDBSSLMode = "MF_SMPP_NOTIFIER_DB_SSL_MODE" - envDBSSLCert = "MF_SMPP_NOTIFIER_DB_SSL_CERT" - envDBSSLKey = "MF_SMPP_NOTIFIER_DB_SSL_KEY" - envDBSSLRootCert = "MF_SMPP_NOTIFIER_DB_SSL_ROOT_CERT" - envHTTPPort = "MF_SMPP_NOTIFIER_HTTP_PORT" - envServerCert = "MF_SMPP_NOTIFIER_SERVER_CERT" - envServerKey = "MF_SMPP_NOTIFIER_SERVER_KEY" - envFrom = "MF_SMPP_NOTIFIER_SOURCE_ADDR" - envJaegerURL = "MF_JAEGER_URL" - envBrokerURL = "MF_BROKER_URL" - - envSmppAddress = "MF_SMPP_ADDRESS" - envSmppUsername = "MF_SMPP_USERNAME" - envSmppPassword = "MF_SMPP_PASSWORD" - envSmppSystemType = "MF_SMPP_SYSTEM_TYPE" - envSmppSrcAddrTON = "MF_SMPP_SRC_ADDR_TON" - envSmppDstAddrTON = "MF_SMPP_DST_ADDR_TON" - envSmppSrcAddrNPI = "MF_SMPP_SRC_ADDR_NPI" - envSmppDstAddrNPI = "MF_SMPP_DST_ADDR_NPI" - - envAuthTLS = "MF_AUTH_CLIENT_TLS" - envAuthCACerts = "MF_AUTH_CA_CERTS" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "smpp-notifier" + envPrefix = "MF_SMPP_NOTIFIER_" + envPrefixHttp = "MF_SMPP_NOTIFIER_HTTP_" + defDB = "subscriptions" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - configPath string - logLevel string - dbConfig postgres.Config - smppConf mfsmpp.Config - from string - httpPort string - serverCert string - serverKey string - jaegerURL string - authTLS bool - authCACerts string - authURL string - authTimeout time.Duration + LogLevel string `env:"MF_SMPP_NOTIFIER_LOG_LEVEL" envDefault:"info"` + From string `env:"MF_SMPP_NOTIFIER_FROM_ADDR" envDefault:""` + ConfigPath string `env:"MF_SMPP_NOTIFIER_CONFIG_PATH" envDefault:"/config.toml"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) } - db := connectToDB(cfg.dbConfig, logger) - defer db.Close() - - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) - } - defer pubSub.Close() - - authTracer, closer := initJaeger("auth", cfg.jaegerURL, logger) - defer closer.Close() - - auth, close := connectToAuth(cfg, authTracer, logger) - if close != nil { - defer close() + log.Fatalf(err.Error()) } - tracer, closer := initJaeger("smpp-notifier", cfg.jaegerURL, logger) - defer closer.Close() - - dbTracer, dbCloser := initJaeger("smpp-notifier_db", cfg.jaegerURL, logger) - defer dbCloser.Close() - - svc := newService(db, dbTracer, auth, cfg, logger) - - if err = consumers.Start(svcName, pubSub, svc, cfg.configPath, logger); err != nil { - logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *notifierPg.Migration(), dbConfig) + if err != nil { + log.Fatal(err) } + defer db.Close() - g.Go(func() error { - return startHTTPServer(ctx, tracer, svc, cfg.httpPort, cfg.serverCert, cfg.serverKey, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("SMPP notifier service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("SMPP notifier service terminated: %s", err)) + smppConfig := mfsmpp.Config{} + if err := env.Parse(&smppConfig); err != nil { + log.Fatalf("failed to load SMPP configuration from environment : %s", err.Error()) } -} - -func loadConfig() config { - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } + defer pubSub.Close() - tls, err := strconv.ParseBool(mainflux.Env(envAuthTLS, defAuthTLS)) + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envAuthTLS) - } - - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), + log.Fatalf(err.Error()) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - saton, err := strconv.ParseUint(mainflux.Env(envSmppSrcAddrTON, defSmppSrcAddrTON), 10, 8) - if err != nil { - log.Fatalf("Invalid value passed for %s", envSmppSrcAddrTON) - } - daton, err := strconv.ParseUint(mainflux.Env(envSmppDstAddrTON, defSmppDstAddrTON), 10, 8) - if err != nil { - log.Fatalf("Invalid value passed for %s", envSmppDstAddrTON) - } - sanpi, err := strconv.ParseUint(mainflux.Env(envSmppSrcAddrNPI, defSmppSrcAddrNPI), 10, 8) + tracer, closer, err := jaegerClient.NewTracer("smpp-notifier", cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid value passed for %s", envSmppSrcAddrNPI) - } - danpi, err := strconv.ParseUint(mainflux.Env(envSmppDstAddrNPI, defSmppDstAddrNPI), 10, 8) - if err != nil { - log.Fatalf("Invalid value passed for %s", envSmppDstAddrNPI) - } - - smppConf := mfsmpp.Config{ - Address: mainflux.Env(envSmppAddress, defSmppAddress), - Username: mainflux.Env(envSmppUsername, defSmppUsername), - Password: mainflux.Env(envSmppPassword, defSmppPassword), - SystemType: mainflux.Env(envSmppSystemType, defSmppSystemType), - SourceAddrTON: uint8(saton), - DestAddrTON: uint8(daton), - SourceAddrNPI: uint8(sanpi), - DestAddrNPI: uint8(danpi), + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer closer.Close() - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - configPath: mainflux.Env(envConfigPath, defConfigPath), - dbConfig: dbConfig, - smppConf: smppConf, - from: mainflux.Env(envFrom, defFrom), - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - authTLS: tls, - authCACerts: mainflux.Env(envAuthCACerts, defAuthCACerts), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, + dbTracer, dbCloser, err := jaegerClient.NewTracer("smpp-notifier_db", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer dbCloser.Close() -} + svc := newService(db, dbTracer, auth, cfg, smppConfig, logger) -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + if err = consumers.Start(svcName, pubSub, svc, cfg.ConfigPath, logger); err != nil { + log.Fatalf("failed to create Postgres writer: %s", err.Error()) } - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, tracer, logger), logger) - return tracer, closer -} - -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) - } - return db -} + g.Go(func() error { + return hs.Start() + }) -func connectToAuth(cfg config, tracer opentracing.Tracer, logger logger.Logger) (mainflux.AuthServiceClient, func() error) { - var opts []grpc.DialOption - if cfg.authTLS { - if cfg.authCACerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.authCACerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("SMPP notifier service terminated: %s", err)) } - return authapi.NewClient(tracer, conn, cfg.authTimeout), conn.Close } -func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServiceClient, c config, logger logger.Logger) notifiers.Service { - database := postgres.NewDatabase(db) - repo := tracing.New(postgres.New(database), tracer) +func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServiceClient, c config, sc mfsmpp.Config, logger logger.Logger) notifiers.Service { + database := notifierPg.NewDatabase(db) + repo := tracing.New(notifierPg.New(database), tracer) idp := ulid.New() - notifier := mfsmpp.New(c.smppConf) - svc := notifiers.New(auth, repo, idp, notifier, c.from) + notifier := mfsmpp.New(sc) + svc := notifiers.New(auth, repo, idp, notifier, c.From) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "notifier", - Subsystem: "smpp", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "notifier", - Subsystem: "smpp", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - return svc -} - -func startHTTPServer(ctx context.Context, tracer opentracing.Tracer, svc notifiers.Service, port string, certFile string, keyFile string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, tracer, logger)} - - switch { - case certFile != "" || keyFile != "": - logger.Info(fmt.Sprintf("SMPP notifier service started using https, cert %s key %s, exposed port %s", certFile, keyFile, port)) - go func() { - errCh <- server.ListenAndServeTLS(certFile, keyFile) - }() - default: - logger.Info(fmt.Sprintf("SMPP notifier service started using http, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("SMPP notifier service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("smpp notifier service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("SMPP notifier service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } + counter, latency := internal.MakeMetrics("notifier", "smpp") + svc = api.MetricsMiddleware(svc, counter, latency) + return svc } diff --git a/cmd/smtp-notifier/main.go b/cmd/smtp-notifier/main.go index 053cafacb1..6f14d25981 100644 --- a/cmd/smtp-notifier/main.go +++ b/cmd/smtp-notifier/main.go @@ -6,170 +6,117 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/notifiers" "github.com/mainflux/mainflux/consumers/notifiers/api" - "github.com/mainflux/mainflux/consumers/notifiers/postgres" + notifierPg "github.com/mainflux/mainflux/consumers/notifiers/postgres" "github.com/mainflux/mainflux/consumers/notifiers/smtp" "github.com/mainflux/mainflux/consumers/notifiers/tracing" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + jagerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/internal/email" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" "github.com/mainflux/mainflux/pkg/ulid" opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - svcName = "smtp-notifier" - stopWaitTime = 5 * time.Second - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "subscriptions" - defConfigPath = "/config.toml" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defHTTPPort = "8906" - defServerCert = "" - defServerKey = "" - defFrom = "" - defJaegerURL = "" - defBrokerURL = "nats://localhost:4222" - - defEmailHost = "localhost" - defEmailPort = "25" - defEmailUsername = "root" - defEmailPassword = "" - defEmailFromAddress = "" - defEmailFromName = "" - defEmailTemplate = "email.tmpl" - - defAuthTLS = "false" - defAuthCACerts = "" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - envLogLevel = "MF_SMTP_NOTIFIER_LOG_LEVEL" - envDBHost = "MF_SMTP_NOTIFIER_DB_HOST" - envDBPort = "MF_SMTP_NOTIFIER_DB_PORT" - envDBUser = "MF_SMTP_NOTIFIER_DB_USER" - envDBPass = "MF_SMTP_NOTIFIER_DB_PASS" - envDB = "MF_SMTP_NOTIFIER_DB" - envConfigPath = "MF_SMTP_NOTIFIER_CONFIG_PATH" - envDBSSLMode = "MF_SMTP_NOTIFIER_DB_SSL_MODE" - envDBSSLCert = "MF_SMTP_NOTIFIER_DB_SSL_CERT" - envDBSSLKey = "MF_SMTP_NOTIFIER_DB_SSL_KEY" - envDBSSLRootCert = "MF_SMTP_NOTIFIER_DB_SSL_ROOT_CERT" - envHTTPPort = "MF_SMTP_NOTIFIER_PORT" - envServerCert = "MF_SMTP_NOTIFIER_SERVER_CERT" - envServerKey = "MF_SMTP_NOTIFIER_SERVER_KEY" - envFrom = "MF_SMTP_NOTIFIER_FROM_ADDR" - envJaegerURL = "MF_JAEGER_URL" - envBrokerURL = "MF_BROKER_URL" - - envEmailHost = "MF_EMAIL_HOST" - envEmailPort = "MF_EMAIL_PORT" - envEmailUsername = "MF_EMAIL_USERNAME" - envEmailPassword = "MF_EMAIL_PASSWORD" - envEmailFromAddress = "MF_EMAIL_FROM_ADDRESS" - envEmailFromName = "MF_EMAIL_FROM_NAME" - envEmailTemplate = "MF_SMTP_NOTIFIER_TEMPLATE" - - envAuthTLS = "MF_AUTH_CLIENT_TLS" - envAuthCACerts = "MF_AUTH_CA_CERTS" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "smtp-notifier" + envPrefix = "MF_SMTP_NOTIFIER_" + envPrefixHttp = "MF_SMTP_NOTIFIER_HTTP_" + defDB = "subscriptions" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - configPath string - logLevel string - dbConfig postgres.Config - emailConf email.Config - from string - httpPort string - serverCert string - serverKey string - jaegerURL string - authTLS bool - authCACerts string - authURL string - authTimeout time.Duration + LogLevel string `env:"MF_SMTP_NOTIFIER_LOG_LEVEL" envDefault:"info"` + ConfigPath string `env:"MF_SMTP_NOTIFIER_CONFIG_PATH" envDefault:"/config.toml"` + From string `env:"MF_SMTP_NOTIFIER_FROM_ADDR" envDefault:""` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - db := connectToDB(cfg.dbConfig, logger) + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *notifierPg.Migration(), dbConfig) + if err != nil { + log.Fatal(err) + } defer db.Close() - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + ec := email.Config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load email configuration : %s", err.Error()) + } + + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err) } defer pubSub.Close() - authTracer, closer := initJaeger("auth", cfg.jaegerURL, logger) - defer closer.Close() - - auth, close := connectToAuth(cfg, authTracer, logger) - if close != nil { - defer close() + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - tracer, closer := initJaeger("smtp-notifier", cfg.jaegerURL, logger) + tracer, closer, err := jagerClient.NewTracer("smtp-notifier", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) + } defer closer.Close() - dbTracer, dbCloser := initJaeger("smtp-notifier_db", cfg.jaegerURL, logger) + dbTracer, dbCloser, err := jagerClient.NewTracer("smtp-notifier_db", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) + } defer dbCloser.Close() - svc := newService(db, dbTracer, auth, cfg, logger) + svc := newService(db, dbTracer, auth, cfg, ec, logger) + + if err = consumers.Start(svcName, pubSub, svc, cfg.ConfigPath, logger); err != nil { + log.Fatalf("failed to create Postgres writer: %s", err) + } - if err = consumers.Start(svcName, pubSub, svc, cfg.configPath, logger); err != nil { - logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, tracer, logger), logger) g.Go(func() error { - return startHTTPServer(ctx, tracer, svc, cfg.httpPort, cfg.serverCert, cfg.serverKey, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("SMTP notifier service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -178,177 +125,21 @@ func main() { } -func loadConfig() config { - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) - } - - tls, err := strconv.ParseBool(mainflux.Env(envAuthTLS, defAuthTLS)) - if err != nil { - log.Fatalf("Invalid value passed for %s\n", envAuthTLS) - } - - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), - } - - emailConf := email.Config{ - FromAddress: mainflux.Env(envEmailFromAddress, defEmailFromAddress), - FromName: mainflux.Env(envEmailFromName, defEmailFromName), - Host: mainflux.Env(envEmailHost, defEmailHost), - Port: mainflux.Env(envEmailPort, defEmailPort), - Username: mainflux.Env(envEmailUsername, defEmailUsername), - Password: mainflux.Env(envEmailPassword, defEmailPassword), - Template: mainflux.Env(envEmailTemplate, defEmailTemplate), - } - - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - configPath: mainflux.Env(envConfigPath, defConfigPath), - dbConfig: dbConfig, - emailConf: emailConf, - from: mainflux.Env(envFrom, defFrom), - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - authTLS: tls, - authCACerts: mainflux.Env(envAuthCACerts, defAuthCACerts), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, - } - -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } - - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err)) - os.Exit(1) - } - - return tracer, closer -} - -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) - } - return db -} - -func connectToAuth(cfg config, tracer opentracing.Tracer, logger logger.Logger) (mainflux.AuthServiceClient, func() error) { - var opts []grpc.DialOption - if cfg.authTLS { - if cfg.authCACerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.authCACerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } - - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) - } - - return authapi.NewClient(tracer, conn, cfg.authTimeout), conn.Close -} - -func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServiceClient, c config, logger logger.Logger) notifiers.Service { - database := postgres.NewDatabase(db) - repo := tracing.New(postgres.New(database), tracer) +func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServiceClient, c config, ec email.Config, logger logger.Logger) notifiers.Service { + database := notifierPg.NewDatabase(db) + repo := tracing.New(notifierPg.New(database), tracer) idp := ulid.New() - agent, err := email.New(&c.emailConf) + agent, err := email.New(&ec) if err != nil { - logger.Error(fmt.Sprintf("Failed to create email agent: %s", err)) - os.Exit(1) + log.Fatalf("failed to create email agent: %s", err.Error()) } notifier := smtp.New(agent) - svc := notifiers.New(auth, repo, idp, notifier, c.from) + svc := notifiers.New(auth, repo, idp, notifier, c.From) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "notifier", - Subsystem: "smtp", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "notifier", - Subsystem: "smtp", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - return svc -} + counter, latency := internal.MakeMetrics("notifier", "smtp") + svc = api.MetricsMiddleware(svc, counter, latency) -func startHTTPServer(ctx context.Context, tracer opentracing.Tracer, svc notifiers.Service, port string, certFile string, keyFile string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, tracer, logger)} - - switch { - case certFile != "" || keyFile != "": - logger.Info(fmt.Sprintf("SMTP notifier service started using https, cert %s key %s, exposed port %s", certFile, keyFile, port)) - go func() { - errCh <- server.ListenAndServeTLS(certFile, keyFile) - }() - default: - logger.Info(fmt.Sprintf("SMTP notifier service started using http, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("SMTP notifier service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("smtp notifier service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("SMTP notifier service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } + return svc } diff --git a/cmd/things/main.go b/cmd/things/main.go index 594da2e2cb..cf1401e8d1 100644 --- a/cmd/things/main.go +++ b/cmd/things/main.go @@ -6,321 +6,178 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/go-redis/redis/v8" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + redisClient "github.com/mainflux/mainflux/internal/clients/redis" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + grpcserver "github.com/mainflux/mainflux/internal/server/grpc" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/uuid" "github.com/mainflux/mainflux/things" "github.com/mainflux/mainflux/things/api" authgrpcapi "github.com/mainflux/mainflux/things/api/auth/grpc" authhttpapi "github.com/mainflux/mainflux/things/api/auth/http" thhttpapi "github.com/mainflux/mainflux/things/api/things/http" - "github.com/mainflux/mainflux/things/postgres" + thingsPg "github.com/mainflux/mainflux/things/postgres" rediscache "github.com/mainflux/mainflux/things/redis" - localusers "github.com/mainflux/mainflux/things/standalone" "github.com/mainflux/mainflux/things/tracing" opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" + svcName = "things" + envPrefix = "MF_THINGS_" + envPrefixCache = "MF_THINGS_CACHE_" + envPrefixES = "MF_THINGS_ES_" + envPrefixHttp = "MF_THINGS_HTTP_" + envPrefixAuthHttp = "MF_THINGS_AUTH_HTTP_" + envPrefixAuthGrpc = "MF_THINGS_AUTH_GRPC_" defDB = "things" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defClientTLS = "false" - defCACerts = "" - defCacheURL = "localhost:6379" - defCachePass = "" - defCacheDB = "0" - defESURL = "localhost:6379" - defESPass = "" - defESDB = "0" - defHTTPPort = "8182" - defAuthHTTPPort = "8989" - defAuthGRPCPort = "8181" - defServerCert = "" - defServerKey = "" - defStandaloneEmail = "" - defStandaloneToken = "" - defJaegerURL = "" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - envLogLevel = "MF_THINGS_LOG_LEVEL" - envDBHost = "MF_THINGS_DB_HOST" - envDBPort = "MF_THINGS_DB_PORT" - envDBUser = "MF_THINGS_DB_USER" - envDBPass = "MF_THINGS_DB_PASS" - envDB = "MF_THINGS_DB" - envDBSSLMode = "MF_THINGS_DB_SSL_MODE" - envDBSSLCert = "MF_THINGS_DB_SSL_CERT" - envDBSSLKey = "MF_THINGS_DB_SSL_KEY" - envDBSSLRootCert = "MF_THINGS_DB_SSL_ROOT_CERT" - envClientTLS = "MF_THINGS_CLIENT_TLS" - envCACerts = "MF_THINGS_CA_CERTS" - envCacheURL = "MF_THINGS_CACHE_URL" - envCachePass = "MF_THINGS_CACHE_PASS" - envCacheDB = "MF_THINGS_CACHE_DB" - envESURL = "MF_THINGS_ES_URL" - envESPass = "MF_THINGS_ES_PASS" - envESDB = "MF_THINGS_ES_DB" - envHTTPPort = "MF_THINGS_HTTP_PORT" - envAuthHTTPPort = "MF_THINGS_AUTH_HTTP_PORT" - envAuthGRPCPort = "MF_THINGS_AUTH_GRPC_PORT" - envServerCert = "MF_THINGS_SERVER_CERT" - envServerKey = "MF_THINGS_SERVER_KEY" - envStandaloneEmail = "MF_THINGS_STANDALONE_EMAIL" - envStandaloneToken = "MF_THINGS_STANDALONE_TOKEN" - envJaegerURL = "MF_JAEGER_URL" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + defSvcHttpPort = "8182" + defSvcAuthHttpPort = "8989" + defSvcAuthGrpcPort = "8181" ) type config struct { - logLevel string - dbConfig postgres.Config - clientTLS bool - caCerts string - cacheURL string - cachePass string - cacheDB string - esURL string - esPass string - esDB string - httpPort string - authHTTPPort string - authGRPCPort string - serverCert string - serverKey string - standaloneEmail string - standaloneToken string - jaegerURL string - authURL string - authTimeout time.Duration + LogLevel string `env:"MF_THINGS_LOG_LEVEL" envDefault:"info"` + StandaloneEmail string `env:"MF_THINGS_STANDALONE_EMAIL" envDefault:""` + StandaloneToken string `env:"MF_THINGS_STANDALONE_TOKEN" envDefault:""` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + // Create new things configuration + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - cacheClient := connectToRedis(cfg.cacheURL, cfg.cachePass, cfg.cacheDB, logger) - - esClient := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) - - db := connectToDB(cfg.dbConfig, logger) - defer db.Close() - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - auth, close := createAuthClient(cfg, authTracer, logger) - if close != nil { - defer close() + // Create new database for things + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *thingsPg.Migration(), dbConfig) + if err != nil { + log.Fatal(err.Error()) } + defer db.Close() - dbTracer, dbCloser := initJaeger("things_db", cfg.jaegerURL, logger) - defer dbCloser.Close() - - cacheTracer, cacheCloser := initJaeger("things_cache", cfg.jaegerURL, logger) - defer cacheCloser.Close() - - svc := newService(auth, dbTracer, cacheTracer, db, cacheClient, esClient, logger) - - g.Go(func() error { - return startHTTPServer(ctx, "thing-http", thhttpapi.MakeHandler(thingsTracer, svc, logger), cfg.httpPort, cfg, logger) - }) - - g.Go(func() error { - return startHTTPServer(ctx, "auth-http", authhttpapi.MakeHandler(thingsTracer, svc, logger), cfg.authHTTPPort, cfg, logger) - }) - - g.Go(func() error { - return startGRPCServer(ctx, svc, thingsTracer, cfg, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Things service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Things service terminated: %s", err)) + // Setup new redis cache client + cacheClient, err := redisClient.Setup(envPrefixCache) + if err != nil { + log.Fatal(err.Error()) } -} + defer cacheClient.Close() -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + // Setup new redis event store client + esClient, err := redisClient.Setup(envPrefixES) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatal(err.Error()) } + defer esClient.Close() - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) + // Setup new auth grpc client + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) + log.Fatal(err) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), + // Create tracer for things database + dbTracer, dbCloser, err := jaegerClient.NewTracer("things_db", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer dbCloser.Close() - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - dbConfig: dbConfig, - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - cacheURL: mainflux.Env(envCacheURL, defCacheURL), - cachePass: mainflux.Env(envCachePass, defCachePass), - cacheDB: mainflux.Env(envCacheDB, defCacheDB), - esURL: mainflux.Env(envESURL, defESURL), - esPass: mainflux.Env(envESPass, defESPass), - esDB: mainflux.Env(envESDB, defESDB), - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - authHTTPPort: mainflux.Env(envAuthHTTPPort, defAuthHTTPPort), - authGRPCPort: mainflux.Env(envAuthGRPCPort, defAuthGRPCPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - standaloneEmail: mainflux.Env(envStandaloneEmail, defStandaloneEmail), - standaloneToken: mainflux.Env(envStandaloneToken, defStandaloneToken), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, + // Create tracer for things cache + cacheTracer, cacheCloser, err := jaegerClient.NewTracer("things_cache", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } -} + defer cacheCloser.Close() -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } + // Create new service + svc := newService(auth, dbTracer, cacheTracer, db, cacheClient, esClient, logger) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() + // Create tracer for HTTP handler things + thingsTracer, thingsCloser, err := jaegerClient.NewTracer("things", cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer thingsCloser.Close() - return tracer, closer -} + // Create new HTTP server + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s gRPC server configuration : %s", svcName, err.Error()) + } + hs1 := httpserver.New(ctx, cancel, "thing-http", httpServerConfig, thhttpapi.MakeHandler(thingsTracer, svc, logger), logger) -func connectToRedis(cacheURL, cachePass string, cacheDB string, logger logger.Logger) *redis.Client { - db, err := strconv.Atoi(cacheDB) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to cache: %s", err)) - os.Exit(1) + // Create new things auth http server + authHttpServerConfig := server.Config{Port: defSvcAuthHttpPort} + if err := env.Parse(&authHttpServerConfig, env.Options{Prefix: envPrefixAuthHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s gRPC server configuration : %s", svcName, err.Error()) } + hs2 := httpserver.New(ctx, cancel, "auth-http", authHttpServerConfig, authhttpapi.MakeHandler(thingsTracer, svc, logger), logger) - return redis.NewClient(&redis.Options{ - Addr: cacheURL, - Password: cachePass, - DB: db, - }) -} + // Create new grpc server + registerThingsServiceServer := func(srv *grpc.Server) { + mainflux.RegisterThingsServiceServer(srv, authgrpcapi.NewServer(thingsTracer, svc)) -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) } - return db -} - -func createAuthClient(cfg config, tracer opentracing.Tracer, logger logger.Logger) (mainflux.AuthServiceClient, func() error) { - if cfg.standaloneEmail != "" && cfg.standaloneToken != "" { - return localusers.NewAuthService(cfg.standaloneEmail, cfg.standaloneToken), nil + grpcServerConfig := server.Config{Port: defSvcAuthGrpcPort} + if err := env.Parse(&grpcServerConfig, env.Options{Prefix: envPrefixAuthGrpc, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s gRPC server configuration : %s", svcName, err.Error()) } + gs := grpcserver.New(ctx, cancel, svcName, grpcServerConfig, registerThingsServiceServer, logger) - conn := connectToAuth(cfg, logger) - return authapi.NewClient(tracer, conn, cfg.authTimeout), conn.Close -} + //Start all servers + g.Go(func() error { + return hs1.Start() + }) + g.Go(func() error { + return hs2.Start() + }) + g.Go(func() error { + return gs.Start() + }) -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs1, hs2, gs) + }) - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Things service terminated: %s", err)) } - - return conn } func newService(auth mainflux.AuthServiceClient, dbTracer opentracing.Tracer, cacheTracer opentracing.Tracer, db *sqlx.DB, cacheClient *redis.Client, esClient *redis.Client, logger logger.Logger) things.Service { - database := postgres.NewDatabase(db) + database := thingsPg.NewDatabase(db) - thingsRepo := postgres.NewThingRepository(database) + thingsRepo := thingsPg.NewThingRepository(database) thingsRepo = tracing.ThingRepositoryMiddleware(dbTracer, thingsRepo) - channelsRepo := postgres.NewChannelRepository(database) + channelsRepo := thingsPg.NewChannelRepository(database) channelsRepo = tracing.ChannelRepositoryMiddleware(dbTracer, channelsRepo) chanCache := rediscache.NewChannelCache(cacheClient) @@ -333,102 +190,8 @@ func newService(auth mainflux.AuthServiceClient, dbTracer opentracing.Tracer, ca svc := things.New(auth, thingsRepo, channelsRepo, chanCache, thingCache, idProvider) svc = rediscache.NewEventStoreMiddleware(svc, esClient) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "things", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "things", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - return svc -} - -func startHTTPServer(ctx context.Context, typ string, handler http.Handler, port string, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: handler} - - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("Things %s service started using https on port %s with cert %s key %s", - typ, port, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - default: - logger.Info(fmt.Sprintf("Things %s service started using http on port %s", typ, cfg.httpPort)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Things %s service error occurred during shutdown at %s: %s", typ, p, err)) - return fmt.Errorf("things %s service occurred during shutdown at %s: %w", typ, p, err) - } - logger.Info(fmt.Sprintf("Things %s service shutdown of http at %s", typ, p)) - return nil - case err := <-errCh: - return err - } - -} - -func startGRPCServer(ctx context.Context, svc things.Service, tracer opentracing.Tracer, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.authGRPCPort) - errCh := make(chan error) - var server *grpc.Server - - listener, err := net.Listen("tcp", p) - if err != nil { - return fmt.Errorf("failed to listen on port %s: %w", cfg.authGRPCPort, err) - } + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - creds, err := credentials.NewServerTLSFromFile(cfg.serverCert, cfg.serverKey) - if err != nil { - return fmt.Errorf("failed to load things certificates: %w", err) - } - logger.Info(fmt.Sprintf("Things gRPC service started using https on port %s with cert %s key %s", - cfg.authGRPCPort, cfg.serverCert, cfg.serverKey)) - server = grpc.NewServer(grpc.Creds(creds)) - default: - logger.Info(fmt.Sprintf("Things gRPC service started using http on port %s", cfg.authGRPCPort)) - server = grpc.NewServer() - } - - mainflux.RegisterThingsServiceServer(server, authgrpcapi.NewServer(tracer, svc)) - go func() { - errCh <- server.Serve(listener) - }() - - select { - case <-ctx.Done(): - c := make(chan bool) - go func() { - defer close(c) - server.GracefulStop() - }() - select { - case <-c: - case <-time.After(stopWaitTime): - } - logger.Info(fmt.Sprintf("Things gRPC service shutdown at %s", p)) - return nil - case err := <-errCh: - return err - } + return svc } diff --git a/cmd/timescale-reader/main.go b/cmd/timescale-reader/main.go index fb8833f32d..12af6ecd4e 100644 --- a/cmd/timescale-reader/main.go +++ b/cmd/timescale-reader/main.go @@ -6,292 +6,101 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" - "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/readers" "github.com/mainflux/mainflux/readers/api" "github.com/mainflux/mainflux/readers/timescale" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - svcName = "timescaledb-reader" - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defPort = "8911" - defClientTLS = "false" - defCACerts = "" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "mainflux" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - - envLogLevel = "MF_TIMESCALE_READER_LOG_LEVEL" - envPort = "MF_TIMESCALE_READER_PORT" - envClientTLS = "MF_TIMESCALE_READER_CLIENT_TLS" - envCACerts = "MF_TIMESCALE_READER_CA_CERTS" - envDBHost = "MF_TIMESCALE_READER_DB_HOST" - envDBPort = "MF_TIMESCALE_READER_DB_PORT" - envDBUser = "MF_TIMESCALE_READER_DB_USER" - envDBPass = "MF_TIMESCALE_READER_DB_PASS" - envDB = "MF_TIMESCALE_READER_DB" - envDBSSLMode = "MF_TIMESCALE_READER_DB_SSL_MODE" - envDBSSLCert = "MF_TIMESCALE_READER_DB_SSL_CERT" - envDBSSLKey = "MF_TIMESCALE_READER_DB_SSL_KEY" - envDBSSLRootCert = "MF_TIMESCALE_READER_DB_SSL_ROOT_CERT" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsAuthTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" + svcName = "timescaledb-reader" + envPrefix = "MF_TIMESCALE_READER_" + envPrefixHttp = "MF_TIMESCALE_READER_HTTP_" + defDB = "messages" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - port string - clientTLS bool - caCerts string - dbConfig timescale.Config - jaegerURL string - thingsAuthURL string - usersAuthURL string - thingsAuthTimeout time.Duration - usersAuthTimeout time.Duration + LogLevel string `env:"MF_TIMESCALE_READER_LOG_LEVEL" envDefault:"info"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) } - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - - authConn := connectToAuth(cfg, logger) - defer authConn.Close() - auth := authapi.NewClient(authTracer, authConn, cfg.usersAuthTimeout) - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - - db := connectToDB(cfg.dbConfig, logger) - defer db.Close() - - repo := newService(db, logger) - - g.Go(func() error { - return startHTTPServer(ctx, repo, tc, auth, cfg.port, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Timescale reader service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Timescale reader service terminated: %s", err)) - } -} - -func loadConfig() config { - dbConfig := timescale.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), - } - - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatalf(err.Error()) } - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsAuthTimeout, defThingsAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsAuthTimeout, err.Error()) + dbConfig := pgClient.Config{Name: defDB} + if err := dbConfig.LoadEnv(envPrefix); err != nil { + log.Fatal(err.Error()) } - - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - dbConfig: dbConfig, - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - thingsAuthTimeout: authTimeout, + db, err := pgClient.Connect(dbConfig) + if err != nil { + log.Fatal(err.Error()) } -} + defer db.Close() -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - logger.Info("Connecting to auth via gRPC") - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } + repo := newService(db, logger) - conn, err := grpc.Dial(cfg.usersAuthURL, opts...) + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) + log.Fatal(err) } - logger.Info(fmt.Sprintf("Established gRPC connection to auth via gRPC: %s", cfg.usersAuthURL)) - return conn -} + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) -func connectToDB(dbConfig timescale.Config, logger logger.Logger) *sqlx.DB { - db, err := timescale.Connect(dbConfig) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to Timescale: %s", err)) - os.Exit(1) - } - return db -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + log.Fatal(err.Error()) } + defer tcHandler.Close() + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(repo, tc, auth, svcName, logger), logger) - return tracer, closer -} + g.Go(func() error { + return hs.Start() + }) -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithInsecure()) - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Timescale reader service terminated: %s", err)) } - return conn } func newService(db *sqlx.DB, logger logger.Logger) readers.MessageRepository { svc := timescale.New(db) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "timescale", - Subsystem: "message_reader", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "timescale", - Subsystem: "message_reader", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics("timescale", "message_reader") + svc = api.MetricsMiddleware(svc, counter, latency) return svc } - -func startHTTPServer(ctx context.Context, repo readers.MessageRepository, tc mainflux.ThingsServiceClient, ac mainflux.AuthServiceClient, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(repo, tc, ac, svcName, logger)} - - logger.Info(fmt.Sprintf("Timescale reader service started, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Timescale reader service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("Timescale reader service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Timescale reader service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/cmd/timescale-writer/main.go b/cmd/timescale-writer/main.go index 98d9b5521b..9d639495a4 100644 --- a/cmd/timescale-writer/main.go +++ b/cmd/timescale-writer/main.go @@ -7,100 +7,81 @@ import ( "context" "fmt" "log" - "net/http" "os" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/consumers" "github.com/mainflux/mainflux/consumers/writers/api" "github.com/mainflux/mainflux/consumers/writers/timescale" + "github.com/mainflux/mainflux/internal" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging/brokers" - stdprometheus "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) const ( - svcName = "timescaledb-writer" - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defBrokerURL = "nats://localhost:4222" - defPort = "8180" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "mainflux" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defConfigPath = "/config.toml" - - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_TIMESCALE_WRITER_LOG_LEVEL" - envPort = "MF_TIMESCALE_WRITER_PORT" - envDBHost = "MF_TIMESCALE_WRITER_DB_HOST" - envDBPort = "MF_TIMESCALE_WRITER_DB_PORT" - envDBUser = "MF_TIMESCALE_WRITER_DB_USER" - envDBPass = "MF_TIMESCALE_WRITER_DB_PASS" - envDB = "MF_TIMESCALE_WRITER_DB" - envDBSSLMode = "MF_TIMESCALE_WRITER_DB_SSL_MODE" - envDBSSLCert = "MF_TIMESCALE_WRITER_DB_SSL_CERT" - envDBSSLKey = "MF_TIMESCALE_WRITER_DB_SSL_KEY" - envDBSSLRootCert = "MF_TIMESCALE_WRITER_DB_SSL_ROOT_CERT" - envConfigPath = "MF_TIMESCALE_WRITER_CONFIG_PATH" + svcName = "timescaledb-writer" + envPrefix = "MF_TIMESCALE_WRITER_" + envPrefixHttp = "MF_TIMESCALE_WRITER_HTTP_" + defDB = "messages" + defSvcHttpPort = "8180" ) type config struct { - brokerURL string - logLevel string - port string - configPath string - dbConfig timescale.Config + LogLevel string `env:"MF_TIMESCALE_WRITER_LOG_LEVEL" envDefault:"info"` + ConfigPath string `env:"MF_TIMESCALE_WRITER_CONFIG_PATH" envDefault:"/config.toml"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) } - pubSub, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatal(err.Error()) } - defer pubSub.Close() - db := connectToDB(cfg.dbConfig, logger) + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *timescale.Migration(), dbConfig) + if err != nil { + log.Fatal(err.Error()) + } defer db.Close() repo := newService(db, logger) - if err = consumers.Start(svcName, pubSub, repo, cfg.configPath, logger); err != nil { - logger.Error(fmt.Sprintf("Failed to create Timescale writer: %s", err)) + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) + if err != nil { + log.Fatalf("failed to connect to message broker: %s", err.Error()) + } + defer pubSub.Close() + + if err = consumers.Start(svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil { + log.Fatalf("failed to create Timescale writer: %s", err.Error()) + } + + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svcName), logger) g.Go(func() error { - return startHTTPServer(ctx, cfg.port, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Timescale writer service shutdown by signal: %s", sig)) - } - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -108,81 +89,10 @@ func main() { } } -func loadConfig() config { - dbConfig := timescale.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), - } - - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - port: mainflux.Env(envPort, defPort), - configPath: mainflux.Env(envConfigPath, defConfigPath), - dbConfig: dbConfig, - } -} - -func connectToDB(dbConfig timescale.Config, logger logger.Logger) *sqlx.DB { - db, err := timescale.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to Postgres: %s", err)) - os.Exit(1) - } - return db -} - func newService(db *sqlx.DB, logger logger.Logger) consumers.Consumer { svc := timescale.New(db) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "timescale", - Subsystem: "message_writer", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "timescale", - Subsystem: "message_writer", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) - + counter, latency := internal.MakeMetrics("timescale", "message_writer") + svc = api.MetricsMiddleware(svc, counter, latency) return svc } - -func startHTTPServer(ctx context.Context, port string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svcName)} - - logger.Info(fmt.Sprintf("Timescale writer service started, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("timescale writer service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("timescale writer service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Timescale writer service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } - -} diff --git a/cmd/twins/main.go b/cmd/twins/main.go index 901daae77d..c4617bbecf 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -6,20 +6,20 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/go-redis/redis/v8" "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + mongoClient "github.com/mainflux/mainflux/internal/clients/mongo" + redisClient "github.com/mainflux/mainflux/internal/clients/redis" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging" "github.com/mainflux/mainflux/pkg/messaging/brokers" "github.com/mainflux/mainflux/pkg/uuid" @@ -31,243 +31,109 @@ import ( rediscache "github.com/mainflux/mainflux/twins/redis" "github.com/mainflux/mainflux/twins/tracing" opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" "go.mongodb.org/mongo-driver/mongo" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( - svcName = "twins" - queue = "twins" - stopWaitTime = 5 * time.Second - defLogLevel = "error" - defHTTPPort = "8180" - defJaegerURL = "" - defServerCert = "" - defServerKey = "" - defDB = "mainflux-twins" - defDBHost = "localhost" - defDBPort = "27017" - defCacheURL = "localhost:6379" - defCachePass = "" - defCacheDB = "0" - defStandaloneEmail = "" - defStandaloneToken = "" - defClientTLS = "false" - defCACerts = "" - defChannelID = "" - defBrokerURL = "nats://localhost:4222" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - envLogLevel = "MF_TWINS_LOG_LEVEL" - envHTTPPort = "MF_TWINS_HTTP_PORT" - envJaegerURL = "MF_JAEGER_URL" - envServerCert = "MF_TWINS_SERVER_CERT" - envServerKey = "MF_TWINS_SERVER_KEY" - envDB = "MF_TWINS_DB" - envDBHost = "MF_TWINS_DB_HOST" - envDBPort = "MF_TWINS_DB_PORT" - envCacheURL = "MF_TWINS_CACHE_URL" - envCachePass = "MF_TWINS_CACHE_PASS" - envCacheDB = "MF_TWINS_CACHE_DB" - envStandaloneEmail = "MF_TWINS_STANDALONE_EMAIL" - envStandaloneToken = "MF_TWINS_STANDALONE_TOKEN" - envClientTLS = "MF_TWINS_CLIENT_TLS" - envCACerts = "MF_TWINS_CA_CERTS" - envChannelID = "MF_TWINS_CHANNEL_ID" - envBrokerURL = "MF_BROKER_URL" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" + svcName = "twins" + queue = "twins" + envPrefix = "MF_TWINS_" + envPrefixHttp = "MF_TWINS_HTTP_" + envPrefixCache = "MF_TWINS_CACHE_" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - httpPort string - jaegerURL string - serverCert string - serverKey string - dbCfg twmongodb.Config - cacheURL string - cachePass string - cacheDB string - standaloneEmail string - standaloneToken string - clientTLS bool - caCerts string - channelID string - brokerURL string - - authURL string - authTimeout time.Duration + LogLevel string `env:"MF_TWINS_LOG_LEVEL" envDefault:"info"` + StandaloneEmail string `env:"MF_TWINS_STANDALONE_EMAIL" envDefault:""` + StandaloneToken string `env:"MF_TWINS_STANDALONE_TOKEN" envDefault:""` + ChannelID string `env:"MF_TWINS_CHANNEL_ID" envDefault:""` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg, env.Options{Prefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) + } + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - cacheClient := connectToRedis(cfg.cacheURL, cfg.cachePass, cfg.cacheDB, logger) - cacheTracer, cacheCloser := initJaeger("twins_cache", cfg.jaegerURL, logger) - defer cacheCloser.Close() - - db, err := twmongodb.Connect(cfg.dbCfg, logger) + cacheClient, err := redisClient.Setup(envPrefixCache) if err != nil { - logger.Error(err.Error()) - os.Exit(1) + log.Fatalf(err.Error()) } - dbTracer, dbCloser := initJaeger("twins_db", cfg.jaegerURL, logger) - defer dbCloser.Close() - - authTracer, authCloser := initJaeger("auth", cfg.jaegerURL, logger) - defer authCloser.Close() - auth, _ := createAuthClient(cfg, authTracer, logger) + defer cacheClient.Close() - pubSub, err := brokers.NewPubSub(cfg.brokerURL, queue, logger) + cacheTracer, cacheCloser, err := jaegerClient.NewTracer("twins_cache", cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) - } - defer pubSub.Close() - - svc := newService(svcName, pubSub, cfg.channelID, auth, dbTracer, db, cacheTracer, cacheClient, logger) - - tracer, closer := initJaeger("twins", cfg.jaegerURL, logger) - defer closer.Close() - - g.Go(func() error { - return startHTTPServer(ctx, twapi.MakeHandler(tracer, svc, logger), cfg.httpPort, cfg, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Twins service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Twins service terminated: %s", err)) + log.Fatalf("failed to init Jaeger: %s", err.Error()) } -} + defer cacheCloser.Close() -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) + db, err := mongoClient.Setup(envPrefix) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) + log.Fatalf("failed to setup postgres database : %s", err.Error()) } - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) + dbTracer, dbCloser, err := jaegerClient.NewTracer("twins_db", cfg.JaegerURL) if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) - } - - dbCfg := twmongodb.Config{ - Name: mainflux.Env(envDB, defDB), - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer dbCloser.Close() - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - dbCfg: dbCfg, - cacheURL: mainflux.Env(envCacheURL, defCacheURL), - cachePass: mainflux.Env(envCachePass, defCachePass), - cacheDB: mainflux.Env(envCacheDB, defCacheDB), - standaloneEmail: mainflux.Env(envStandaloneEmail, defStandaloneEmail), - standaloneToken: mainflux.Env(envStandaloneToken, defStandaloneToken), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - channelID: mainflux.Env(envChannelID, defChannelID), - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, - } -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) + var auth mainflux.AuthServiceClient + switch cfg.StandaloneEmail != "" && cfg.StandaloneToken != "" { + case true: + auth = localusers.NewAuthService(cfg.StandaloneEmail, cfg.StandaloneToken) + default: + authServiceClient, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err.Error()) + } + defer authHandler.Close() + auth = authServiceClient + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) } - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() + pubSub, err := brokers.NewPubSub(cfg.BrokerURL, queue, logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) + log.Fatalf("failed to connect to message broker: %s", err.Error()) } + defer pubSub.Close() - return tracer, closer -} + svc := newService(svcName, pubSub, cfg.ChannelID, auth, dbTracer, db, cacheTracer, cacheClient, logger) -func createAuthClient(cfg config, tracer opentracing.Tracer, logger logger.Logger) (mainflux.AuthServiceClient, func() error) { - if cfg.standaloneEmail != "" && cfg.standaloneToken != "" { - return localusers.NewAuthService(cfg.standaloneEmail, cfg.standaloneToken), nil + tracer, closer, err := jaegerClient.NewTracer("twins", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer closer.Close() - conn := connectToAuth(cfg, logger) - return authapi.NewClient(tracer, conn, cfg.authTimeout), conn.Close -} - -func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, twapi.MakeHandler(tracer, svc, logger), logger) - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) - } + g.Go(func() error { + return hs.Start() + }) - return conn -} + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) -func connectToRedis(cacheURL, cachePass, cacheDB string, logger logger.Logger) *redis.Client { - db, err := strconv.Atoi(cacheDB) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to cache: %s", err)) - os.Exit(1) + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Twins service terminated: %s", err)) } - - return redis.NewClient(&redis.Options{ - Addr: cacheURL, - Password: cachePass, - DB: db, - }) } func newService(id string, ps messaging.PubSub, chanID string, users mainflux.AuthServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, cacheTracer opentracing.Tracer, cacheClient *redis.Client, logger logger.Logger) twins.Service { @@ -283,25 +149,11 @@ func newService(id string, ps messaging.PubSub, chanID string, users mainflux.Au svc := twins.New(ps, users, twinRepo, twinCache, stateRepo, idProvider, chanID, logger) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "twins", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "twins", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) err := ps.Subscribe(id, brokers.SubjectAllChannels, handle(logger, chanID, svc)) if err != nil { - logger.Error(err.Error()) - os.Exit(1) + log.Fatalf(err.Error()) } return svc } @@ -321,40 +173,6 @@ func handle(logger logger.Logger, chanID string, svc twins.Service) handlerFunc } } -func startHTTPServer(ctx context.Context, handler http.Handler, port string, cfg config, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: handler} - - switch { - case cfg.serverCert != "" || cfg.serverKey != "": - logger.Info(fmt.Sprintf("Twins service started using https on port %s with cert %s key %s", - port, cfg.serverCert, cfg.serverKey)) - go func() { - errCh <- server.ListenAndServeTLS(cfg.serverCert, cfg.serverKey) - }() - default: - logger.Info(fmt.Sprintf("Twins service started using http on port %s", cfg.httpPort)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Twins service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("twins service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Twins service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } -} - type handlerFunc func(msg *messaging.Message) error func (h handlerFunc) Handle(msg *messaging.Message) error { diff --git a/cmd/users/main.go b/cmd/users/main.go index a821e50e74..592fe6d143 100644 --- a/cmd/users/main.go +++ b/cmd/users/main.go @@ -6,16 +6,17 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" "regexp" - "strconv" - "time" + "github.com/mainflux/mainflux/internal" + authClient "github.com/mainflux/mainflux/internal/clients/grpc/auth" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/internal/email" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/uuid" "github.com/mainflux/mainflux/users" @@ -23,316 +24,128 @@ import ( "github.com/mainflux/mainflux/users/emailer" "github.com/mainflux/mainflux/users/tracing" "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" - authapi "github.com/mainflux/mainflux/auth/api/grpc" + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/users/api" - "github.com/mainflux/mainflux/users/postgres" + usersPg "github.com/mainflux/mainflux/users/postgres" opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" ) const ( - stopWaitTime = 5 * time.Second - - defLogLevel = "error" - defDBHost = "localhost" - defDBPort = "5432" - defDBUser = "mainflux" - defDBPass = "mainflux" - defDB = "users" - defDBSSLMode = "disable" - defDBSSLCert = "" - defDBSSLKey = "" - defDBSSLRootCert = "" - defHTTPPort = "8180" - defServerCert = "" - defServerKey = "" - defJaegerURL = "" - - defEmailHost = "localhost" - defEmailPort = "25" - defEmailUsername = "root" - defEmailPassword = "" - defEmailFromAddress = "" - defEmailFromName = "" - defEmailTemplate = "email.tmpl" - defAdminEmail = "" - defAdminPassword = "" - defPassRegex = "^.{8,}$" - - defTokenResetEndpoint = "/reset-request" // URL where user lands after click on the reset link from email - - defAuthTLS = "false" - defAuthCACerts = "" - defAuthURL = "localhost:8181" - defAuthTimeout = "1s" - - defSelfRegister = "true" // By default, everybody can create a user. Otherwise, only admin can create a user. - - envLogLevel = "MF_USERS_LOG_LEVEL" - envDBHost = "MF_USERS_DB_HOST" - envDBPort = "MF_USERS_DB_PORT" - envDBUser = "MF_USERS_DB_USER" - envDBPass = "MF_USERS_DB_PASS" - envDB = "MF_USERS_DB" - envDBSSLMode = "MF_USERS_DB_SSL_MODE" - envDBSSLCert = "MF_USERS_DB_SSL_CERT" - envDBSSLKey = "MF_USERS_DB_SSL_KEY" - envDBSSLRootCert = "MF_USERS_DB_SSL_ROOT_CERT" - envHTTPPort = "MF_USERS_HTTP_PORT" - envServerCert = "MF_USERS_SERVER_CERT" - envServerKey = "MF_USERS_SERVER_KEY" - envJaegerURL = "MF_JAEGER_URL" - - envAdminEmail = "MF_USERS_ADMIN_EMAIL" - envAdminPassword = "MF_USERS_ADMIN_PASSWORD" - envPassRegex = "MF_USERS_PASS_REGEX" - - envEmailHost = "MF_EMAIL_HOST" - envEmailPort = "MF_EMAIL_PORT" - envEmailUsername = "MF_EMAIL_USERNAME" - envEmailPassword = "MF_EMAIL_PASSWORD" - envEmailFromAddress = "MF_EMAIL_FROM_ADDRESS" - envEmailFromName = "MF_EMAIL_FROM_NAME" - envEmailTemplate = "MF_EMAIL_TEMPLATE" - - envTokenResetEndpoint = "MF_TOKEN_RESET_ENDPOINT" - - envAuthTLS = "MF_AUTH_CLIENT_TLS" - envAuthCACerts = "MF_AUTH_CA_CERTS" - envAuthURL = "MF_AUTH_GRPC_URL" - envAuthTimeout = "MF_AUTH_GRPC_TIMEOUT" - - envSelfRegister = "MF_USERS_ALLOW_SELF_REGISTER" + svcName = "users" + envPrefix = "MF_USERS_" + envPrefixHttp = "MF_USERS_HTTP_" + defDB = "users" + defSvcHttpPort = "8180" ) type config struct { - logLevel string - dbConfig postgres.Config - emailConf email.Config - httpPort string - serverCert string - serverKey string - jaegerURL string - resetURL string - authTLS bool - authCACerts string - authURL string - authTimeout time.Duration - adminEmail string - adminPassword string - passRegex *regexp.Regexp - selfRegister bool + LogLevel string `env:"MF_USERS_LOG_LEVEL" envDefault:"info"` + AdminEmail string `env:"MF_USERS_ADMIN_EMAIL" envDefault:""` + AdminPassword string `env:"MF_USERS_ADMIN_PASSWORD" envDefault:""` + PassRegexText string `env:"MF_USERS_PASS_REGEX" envDefault:"^.{8,}$"` + SelfRegister bool `env:"MF_USERS_ALLOW_SELF_REGISTER" envDefault:"true"` + ResetURL string `env:"MF_TOKEN_RESET_ENDPOINT" envDefault:"email.tmpl"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` + PassRegex *regexp.Regexp } func main() { - cfg := loadConfig() ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) - if err != nil { - log.Fatalf(err.Error()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err.Error()) } - db := connectToDB(cfg.dbConfig, logger) - defer db.Close() - - authTracer, closer := initJaeger("auth", cfg.jaegerURL, logger) - defer closer.Close() - - auth, close := connectToAuth(cfg, authTracer, logger) - if close != nil { - defer close() - } - - tracer, closer := initJaeger("users", cfg.jaegerURL, logger) - defer closer.Close() - - dbTracer, dbCloser := initJaeger("users_db", cfg.jaegerURL, logger) - defer dbCloser.Close() - - svc := newService(db, dbTracer, auth, cfg, logger) - - g.Go(func() error { - return startHTTPServer(ctx, tracer, svc, cfg.httpPort, cfg.serverCert, cfg.serverKey, logger) - }) - - g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("Users service shutdown by signal: %s", sig)) - } - return nil - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Users service terminated: %s", err)) - } -} - -func loadConfig() config { - authTimeout, err := time.ParseDuration(mainflux.Env(envAuthTimeout, defAuthTimeout)) + passRegex, err := regexp.Compile(cfg.PassRegexText) if err != nil { - log.Fatalf("Invalid %s value: %s", envAuthTimeout, err.Error()) + log.Fatalf("Invalid password validation rules %s\n", cfg.PassRegexText) } + cfg.PassRegex = passRegex - tls, err := strconv.ParseBool(mainflux.Env(envAuthTLS, defAuthTLS)) + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { - log.Fatalf("Invalid value passed for %s\n", envAuthTLS) + log.Fatalf(err.Error()) } - passRegex, err := regexp.Compile(mainflux.Env(envPassRegex, defPassRegex)) - if err != nil { - log.Fatalf("Invalid password validation rules %s\n", envPassRegex) + ec := email.Config{} + if err := env.Parse(&ec); err != nil { + log.Fatalf("failed to load email configuration : %s", err.Error()) } - selfRegister, err := strconv.ParseBool(mainflux.Env(envSelfRegister, defSelfRegister)) + dbConfig := pgClient.Config{Name: defDB} + db, err := pgClient.SetupWithConfig(envPrefix, *usersPg.Migration(), dbConfig) if err != nil { - log.Fatalf("Invalid %s value: %s", envSelfRegister, err.Error()) - } - - dbConfig := postgres.Config{ - Host: mainflux.Env(envDBHost, defDBHost), - Port: mainflux.Env(envDBPort, defDBPort), - User: mainflux.Env(envDBUser, defDBUser), - Pass: mainflux.Env(envDBPass, defDBPass), - Name: mainflux.Env(envDB, defDB), - SSLMode: mainflux.Env(envDBSSLMode, defDBSSLMode), - SSLCert: mainflux.Env(envDBSSLCert, defDBSSLCert), - SSLKey: mainflux.Env(envDBSSLKey, defDBSSLKey), - SSLRootCert: mainflux.Env(envDBSSLRootCert, defDBSSLRootCert), + log.Fatal(err) } + defer db.Close() - emailConf := email.Config{ - FromAddress: mainflux.Env(envEmailFromAddress, defEmailFromAddress), - FromName: mainflux.Env(envEmailFromName, defEmailFromName), - Host: mainflux.Env(envEmailHost, defEmailHost), - Port: mainflux.Env(envEmailPort, defEmailPort), - Username: mainflux.Env(envEmailUsername, defEmailUsername), - Password: mainflux.Env(envEmailPassword, defEmailPassword), - Template: mainflux.Env(envEmailTemplate, defEmailTemplate), + auth, authHandler, err := authClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err) } + defer authHandler.Close() + logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - return config{ - logLevel: mainflux.Env(envLogLevel, defLogLevel), - dbConfig: dbConfig, - emailConf: emailConf, - httpPort: mainflux.Env(envHTTPPort, defHTTPPort), - serverCert: mainflux.Env(envServerCert, defServerCert), - serverKey: mainflux.Env(envServerKey, defServerKey), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - resetURL: mainflux.Env(envTokenResetEndpoint, defTokenResetEndpoint), - authTLS: tls, - authCACerts: mainflux.Env(envAuthCACerts, defAuthCACerts), - authURL: mainflux.Env(envAuthURL, defAuthURL), - authTimeout: authTimeout, - adminEmail: mainflux.Env(envAdminEmail, defAdminEmail), - adminPassword: mainflux.Env(envAdminPassword, defAdminPassword), - passRegex: passRegex, - selfRegister: selfRegister, + dbTracer, dbCloser, err := jaegerClient.NewTracer("auth_db", cfg.JaegerURL) + if err != nil { + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer dbCloser.Close() -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } + svc := newService(db, dbTracer, auth, cfg, ec, logger) - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() + tracer, closer, err := jaegerClient.NewTracer("users", cfg.JaegerURL) if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err)) - os.Exit(1) + log.Fatalf("failed to init Jaeger: %s", err.Error()) } + defer closer.Close() - return tracer, closer -} -func connectToDB(dbConfig postgres.Config, logger logger.Logger) *sqlx.DB { - db, err := postgres.Connect(dbConfig) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to postgres: %s", err)) - os.Exit(1) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) } - return db -} + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, tracer, logger), logger) -func connectToAuth(cfg config, tracer opentracing.Tracer, logger logger.Logger) (mainflux.AuthServiceClient, func() error) { - var opts []grpc.DialOption - if cfg.authTLS { - if cfg.authCACerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.authCACerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to create tls credentials: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - opts = append(opts, grpc.WithInsecure()) - logger.Info("gRPC communication is not encrypted") - } + g.Go(func() error { + return hs.Start() + }) - conn, err := grpc.Dial(cfg.authURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to auth service: %s", err)) - os.Exit(1) - } + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) - return authapi.NewClient(tracer, conn, cfg.authTimeout), conn.Close + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("Users service terminated: %s", err)) + } } -func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServiceClient, c config, logger logger.Logger) users.Service { - database := postgres.NewDatabase(db) +func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServiceClient, c config, ec email.Config, logger logger.Logger) users.Service { + database := usersPg.NewDatabase(db) hasher := bcrypt.New() - userRepo := tracing.UserRepositoryMiddleware(postgres.NewUserRepo(database), tracer) + userRepo := tracing.UserRepositoryMiddleware(usersPg.NewUserRepo(database), tracer) - emailer, err := emailer.New(c.resetURL, &c.emailConf) + emailer, err := emailer.New(c.ResetURL, &ec) if err != nil { logger.Error(fmt.Sprintf("Failed to configure e-mailing util: %s", err.Error())) } idProvider := uuid.New() - svc := users.New(userRepo, hasher, auth, emailer, idProvider, c.passRegex) + svc := users.New(userRepo, hasher, auth, emailer, idProvider, c.PassRegex) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "users", - Subsystem: "api", - Name: "request_count", - Help: "Number of requests received.", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "users", - Subsystem: "api", - Name: "request_latency_microseconds", - Help: "Total duration of requests in microseconds.", - }, []string{"method"}), - ) + counter, latency := internal.MakeMetrics(svcName, "api") + svc = api.MetricsMiddleware(svc, counter, latency) + if err := createAdmin(svc, userRepo, c, auth); err != nil { - logger.Error("failed to create admin user: " + err.Error()) - os.Exit(1) + log.Fatalf("failed to create admin user: " + err.Error()) } - switch c.selfRegister { + switch c.SelfRegister { case true: // If MF_USERS_ALLOW_SELF_REGISTER environment variable is "true", // everybody can create a new user. Here, check the existence of that @@ -343,12 +156,10 @@ func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServic // Add a policy that allows anybody to create a user apr, err := auth.AddPolicy(context.Background(), &mainflux.AddPolicyReq{Obj: "user", Act: "create", Sub: "*"}) if err != nil { - logger.Error("failed to add the policy related to MF_USERS_ALLOW_SELF_REGISTER: " + err.Error()) - os.Exit(1) + log.Fatalf("failed to add the policy related to MF_USERS_ALLOW_SELF_REGISTER: " + err.Error()) } if !apr.GetAuthorized() { - logger.Error("failed to authorized the policy result related to MF_USERS_ALLOW_SELF_REGISTER: " + errors.ErrAuthorization.Error()) - os.Exit(1) + log.Fatalf("failed to authorized the policy result related to MF_USERS_ALLOW_SELF_REGISTER: " + errors.ErrAuthorization.Error()) } } default: @@ -357,12 +168,10 @@ func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServic // allows everybody to create a new user. dpr, err := auth.DeletePolicy(context.Background(), &mainflux.DeletePolicyReq{Obj: "user", Act: "create", Sub: "*"}) if err != nil { - logger.Error("failed to delete a policy: " + err.Error()) - os.Exit(1) + log.Fatalf("failed to delete a policy: " + err.Error()) } if !dpr.GetDeleted() { - logger.Error("deleting a policy expected to succeed.") - os.Exit(1) + log.Fatalf("deleting a policy expected to succeed.") } } @@ -371,8 +180,8 @@ func newService(db *sqlx.DB, tracer opentracing.Tracer, auth mainflux.AuthServic func createAdmin(svc users.Service, userRepo users.UserRepository, c config, auth mainflux.AuthServiceClient) error { user := users.User{ - Email: c.adminEmail, - Password: c.adminPassword, + Email: c.AdminEmail, + Password: c.AdminPassword, } if admin, err := userRepo.RetrieveByEmail(context.Background(), user.Email); err == nil { @@ -415,37 +224,3 @@ func createAdmin(svc users.Service, userRepo users.UserRepository, c config, aut return nil } - -func startHTTPServer(ctx context.Context, tracer opentracing.Tracer, svc users.Service, port string, certFile string, keyFile string, logger logger.Logger) error { - p := fmt.Sprintf(":%s", port) - errCh := make(chan error) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, tracer, logger)} - - switch { - case certFile != "" || keyFile != "": - logger.Info(fmt.Sprintf("Users service started using https, cert %s key %s, exposed port %s", certFile, keyFile, port)) - go func() { - errCh <- server.ListenAndServeTLS(certFile, keyFile) - }() - default: - logger.Info(fmt.Sprintf("Users service started using http, exposed port %s", port)) - go func() { - errCh <- server.ListenAndServe() - }() - } - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - logger.Error(fmt.Sprintf("Users service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("users service occurred during shutdown at %s: %w", p, err) - } - logger.Info(fmt.Sprintf("Users service shutdown of http at %s", p)) - return nil - case err := <-errCh: - return err - } - -} diff --git a/cmd/ws/main.go b/cmd/ws/main.go index e48920c7ed..53808f9926 100644 --- a/cmd/ws/main.go +++ b/cmd/ws/main.go @@ -6,105 +6,79 @@ package main import ( "context" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" - "time" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" "golang.org/x/sync/errgroup" + "github.com/mainflux/mainflux/internal" + thingsClient "github.com/mainflux/mainflux/internal/clients/grpc/things" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/internal/server" + httpserver "github.com/mainflux/mainflux/internal/server/http" logger "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/messaging" "github.com/mainflux/mainflux/pkg/messaging/brokers" - thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" adapter "github.com/mainflux/mainflux/ws" "github.com/mainflux/mainflux/ws/api" - opentracing "github.com/opentracing/opentracing-go" - stdprometheus "github.com/prometheus/client_golang/prometheus" - jconfig "github.com/uber/jaeger-client-go/config" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" ) const ( - stopWaitTime = 5 * time.Second - - defPort = "8190" - defBrokerURL = "nats://localhost:4222" - defLogLevel = "error" - defClientTLS = "false" - defCACerts = "" - defJaegerURL = "" - defThingsAuthURL = "localhost:8183" - defThingsAuthTimeout = "1s" - - envPort = "MF_WS_ADAPTER_PORT" - envBrokerURL = "MF_BROKER_URL" - envLogLevel = "MF_WS_ADAPTER_LOG_LEVEL" - envClientTLS = "MF_WS_ADAPTER_CLIENT_TLS" - envCACerts = "MF_WS_ADAPTER_CA_CERTS" - envJaegerURL = "MF_JAEGER_URL" - envThingsAuthURL = "MF_THINGS_AUTH_GRPC_URL" - envThingsTimeout = "MF_THINGS_AUTH_GRPC_TIMEOUT" + svcName = "ws-adapter" + envPrefix = "MF_WS_ADAPTER_" + envPrefixHttp = "MF_WS_ADAPTER_HTTP_" + defSvcHttpPort = "8190" ) type config struct { - port string - brokerURL string - logLevel string - clientTLS bool - caCerts string - jaegerURL string - thingsAuthURL string - thingsAuthTimeout time.Duration + LogLevel string `env:"MF_WS_ADAPTER_LOG_LEVEL" envDefault:"info"` + BrokerURL string `env:"MF_BROKER_URL" envDefault:"nats://localhost:4222"` + JaegerURL string `env:"MF_JAEGER_URL" envDefault:"localhost:6831"` } func main() { - cfg := loadConfig() - ctx, cancel := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) - logger, err := logger.New(os.Stdout, cfg.logLevel) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s service configuration : %s", svcName, err.Error()) + } + + logger, err := logger.New(os.Stdout, cfg.LogLevel) if err != nil { log.Fatalf(err.Error()) } - conn := connectToThings(cfg, logger) - defer conn.Close() - - thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) - defer thingsCloser.Close() - - tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) + tc, tcHandler, err := thingsClient.Setup(envPrefix, cfg.JaegerURL) + if err != nil { + log.Fatal(err.Error()) + } + defer internal.Close(logger, tcHandler) + logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - nps, err := brokers.NewPubSub(cfg.brokerURL, "", logger) + nps, err := brokers.NewPubSub(cfg.BrokerURL, "", logger) if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to message broker: %s", err)) - os.Exit(1) + log.Fatalf("Failed to connect to message broker: %s", err.Error()) + } defer nps.Close() svc := newService(tc, nps, logger) + httpServerConfig := server.Config{Port: defSvcHttpPort} + if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHttp, AltPrefix: envPrefix}); err != nil { + log.Fatalf("failed to load %s HTTP server configuration : %s", svcName, err.Error()) + } + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger), logger) + g.Go(func() error { - return startWSServer(ctx, cfg, svc, logger) + return hs.Start() }) g.Go(func() error { - if sig := errors.SignalHandler(ctx); sig != nil { - cancel() - logger.Info(fmt.Sprintf("WS adapter service shutdown by signal: %s", sig)) - } - - return nil + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) }) if err := g.Wait(); err != nil { @@ -112,121 +86,10 @@ func main() { } } -func loadConfig() config { - tls, err := strconv.ParseBool(mainflux.Env(envClientTLS, defClientTLS)) - if err != nil { - log.Fatalf("Invalid value passed for %s\n", envClientTLS) - } - - authTimeout, err := time.ParseDuration(mainflux.Env(envThingsTimeout, defThingsAuthTimeout)) - if err != nil { - log.Fatalf("Invalid %s value: %s", envThingsTimeout, err.Error()) - } - - return config{ - brokerURL: mainflux.Env(envBrokerURL, defBrokerURL), - port: mainflux.Env(envPort, defPort), - logLevel: mainflux.Env(envLogLevel, defLogLevel), - clientTLS: tls, - caCerts: mainflux.Env(envCACerts, defCACerts), - jaegerURL: mainflux.Env(envJaegerURL, defJaegerURL), - thingsAuthURL: mainflux.Env(envThingsAuthURL, defThingsAuthURL), - thingsAuthTimeout: authTimeout, - } -} - -func connectToThings(cfg config, logger logger.Logger) *grpc.ClientConn { - var opts []grpc.DialOption - if cfg.clientTLS { - if cfg.caCerts != "" { - tpc, err := credentials.NewClientTLSFromFile(cfg.caCerts, "") - if err != nil { - logger.Error(fmt.Sprintf("Failed to load certs: %s", err)) - os.Exit(1) - } - opts = append(opts, grpc.WithTransportCredentials(tpc)) - } - } else { - logger.Info("gRPC communication is not encrypted") - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } - - conn, err := grpc.Dial(cfg.thingsAuthURL, opts...) - if err != nil { - logger.Error(fmt.Sprintf("Failed to connect to things service: %s", err)) - os.Exit(1) - } - - return conn -} - -func initJaeger(svcName, url string, logger logger.Logger) (opentracing.Tracer, io.Closer) { - if url == "" { - return opentracing.NoopTracer{}, ioutil.NopCloser(nil) - } - - tracer, closer, err := jconfig.Configuration{ - ServiceName: svcName, - Sampler: &jconfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jconfig.ReporterConfig{ - LocalAgentHostPort: url, - LogSpans: true, - }, - }.NewTracer() - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger client: %s", err)) - os.Exit(1) - } - - return tracer, closer -} - func newService(tc mainflux.ThingsServiceClient, nps messaging.PubSub, logger logger.Logger) adapter.Service { svc := adapter.New(tc, nps) svc = api.LoggingMiddleware(svc, logger) - svc = api.MetricsMiddleware( - svc, - kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Namespace: "ws_adapter", - Subsystem: "api", - Name: "reqeust_count", - Help: "Number of requests received", - }, []string{"method"}), - kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "ws_adapter", - Subsystem: "api", - Name: "request_latency_microsecond", - Help: "Total duration of requests in microseconds", - }, []string{"method"}), - ) - + counter, latency := internal.MakeMetrics("ws_adapter", "api") + svc = api.MetricsMiddleware(svc, counter, latency) return svc } - -func startWSServer(ctx context.Context, cfg config, svc adapter.Service, l logger.Logger) error { - p := fmt.Sprintf(":%s", cfg.port) - errCh := make(chan error, 2) - server := &http.Server{Addr: p, Handler: api.MakeHandler(svc, l)} - l.Info(fmt.Sprintf("WS adapter service started, exposed port %s", cfg.port)) - - go func() { - errCh <- server.ListenAndServe() - }() - - select { - case <-ctx.Done(): - ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) - defer cancelShutdown() - if err := server.Shutdown(ctxShutdown); err != nil { - l.Error(fmt.Sprintf("WS adapter service error occurred during shutdown at %s: %s", p, err)) - return fmt.Errorf("WS adapter service error occurred during shutdown at %s: %w", p, err) - } - l.Info(fmt.Sprintf("WS adapter service shutdown at %s", p)) - return nil - case err := <-errCh: - return err - } -} diff --git a/coap/README.md b/coap/README.md index c0896edcb2..1a626a8b82 100644 --- a/coap/README.md +++ b/coap/README.md @@ -13,7 +13,7 @@ default values. |--------------------------------|--------------------------------------------------------|-----------------------| | MF_COAP_ADAPTER_PORT | Service listening port | 5683 | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_COAP_ADAPTER_LOG_LEVEL | Service log level | error | +| MF_COAP_ADAPTER_LOG_LEVEL | Service log level | info | | MF_COAP_ADAPTER_CLIENT_TLS | Flag that indicates if TLS should be turned on | false | | MF_COAP_ADAPTER_CA_CERTS | Path to trusted CAs in PEM format | | | MF_COAP_ADAPTER_PING_PERIOD | Hours between 1 and 24 to ping client with ACK message | 12 | diff --git a/consumers/notifiers/postgres/init.go b/consumers/notifiers/postgres/init.go index 1463236b53..5c3713e58c 100644 --- a/consumers/notifiers/postgres/init.go +++ b/consumers/notifiers/postgres/init.go @@ -3,47 +3,10 @@ package postgres -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "subscriptions_1", @@ -62,7 +25,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/consumers/notifiers/postgres/setup_test.go b/consumers/notifiers/postgres/setup_test.go index 92d81553b7..2ee238c446 100644 --- a/consumers/notifiers/postgres/setup_test.go +++ b/consumers/notifiers/postgres/setup_test.go @@ -11,9 +11,9 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux/consumers/notifiers/postgres" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/pkg/ulid" dockertest "github.com/ory/dockertest/v3" ) @@ -52,7 +52,7 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to docker: %s", err) } - dbConfig := postgres.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -64,7 +64,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - if db, err = postgres.Connect(dbConfig); err != nil { + if db, err = pgClient.SetupDB(dbConfig, *postgres.Migration()); err != nil { log.Fatalf("Could not setup test DB connection: %s", err) } diff --git a/consumers/notifiers/smpp/README.md b/consumers/notifiers/smpp/README.md index e7581c6f43..6d78a012ea 100644 --- a/consumers/notifiers/smpp/README.md +++ b/consumers/notifiers/smpp/README.md @@ -10,7 +10,7 @@ default values. | Variable | Description | Default | | ------------------------------------| --------------------------------------------------------------- ----- | --------------------- | -| MF_SMPP_NOTIFIER_LOG_LEVEL | Log level for SMPP Notifier (debug, info, warn, error) | error | +| MF_SMPP_NOTIFIER_LOG_LEVEL | Log level for SMPP Notifier (debug, info, warn, error) | info | | MF_SMPP_NOTIFIER_DB_HOST | Database host address | localhost | | MF_SMPP_NOTIFIER_DB_PORT | Database host port | 5432 | | MF_SMPP_NOTIFIER_DB_USER | Database user | mainflux | diff --git a/consumers/notifiers/smpp/config.go b/consumers/notifiers/smpp/config.go index 87eb973d4b..b9ebdfd9dc 100644 --- a/consumers/notifiers/smpp/config.go +++ b/consumers/notifiers/smpp/config.go @@ -9,13 +9,13 @@ import ( // Config represents SMPP transmitter configuration. type Config struct { - Address string - Username string - Password string - SystemType string - SourceAddrTON uint8 - SourceAddrNPI uint8 - DestAddrTON uint8 - DestAddrNPI uint8 + Address string `env:"MF_SMPP_ADDRESS" envDefault:""` + Username string `env:"MF_SMPP_USERNAME" envDefault:""` + Password string `env:"MF_SMPP_PASSWORD" envDefault:""` + SystemType string `env:"MF_SMPP_SYSTEM_TYPE" envDefault:""` + SourceAddrTON uint8 `env:"MF_SMPP_SRC_ADDR_TON" envDefault:"0"` + SourceAddrNPI uint8 `env:"MF_SMPP_DST_ADDR_TON" envDefault:"0"` + DestAddrTON uint8 `env:"MF_SMPP_SRC_ADDR_NPI" envDefault:"0"` + DestAddrNPI uint8 `env:"MF_SMPP_DST_ADDR_NPI" envDefault:"0"` TLS *tls.Config } diff --git a/consumers/notifiers/smtp/README.md b/consumers/notifiers/smtp/README.md index 27f0558123..8bd8d4e4e9 100644 --- a/consumers/notifiers/smtp/README.md +++ b/consumers/notifiers/smtp/README.md @@ -10,7 +10,7 @@ default values. | Variable | Description | Default | | --------------------------------- | ----------------------------------------------------------------------- | --------------------- | -| MF_SMTP_NOTIFIER_LOG_LEVEL | Log level for SMT Notifier (debug, info, warn, error) | error | +| MF_SMTP_NOTIFIER_LOG_LEVEL | Log level for SMT Notifier (debug, info, warn, error) | info | | MF_SMTP_NOTIFIER_DB_HOST | Database host address | localhost | | MF_SMTP_NOTIFIER_DB_PORT | Database host port | 5432 | | MF_SMTP_NOTIFIER_DB_USER | Database user | mainflux | diff --git a/consumers/writers/cassandra/README.md b/consumers/writers/cassandra/README.md index c3d56a1b1c..5e716672e0 100644 --- a/consumers/writers/cassandra/README.md +++ b/consumers/writers/cassandra/README.md @@ -11,7 +11,7 @@ default values. | Variable | Description | Default | | -------------------------------- | ----------------------------------------------------------------------- | --------------------- | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_CASSANDRA_WRITER_LOG_LEVEL | Log level for Cassandra writer (debug, info, warn, error) | error | +| MF_CASSANDRA_WRITER_LOG_LEVEL | Log level for Cassandra writer (debug, info, warn, error) | info | | MF_CASSANDRA_WRITER_PORT | Service HTTP port | 8180 | | MF_CASSANDRA_WRITER_DB_CLUSTER | Cassandra cluster comma separated addresses | 127.0.0.1 | | MF_CASSANDRA_WRITER_DB_KEYSPACE | Cassandra keyspace name | mainflux | diff --git a/consumers/writers/cassandra/consumer_test.go b/consumers/writers/cassandra/consumer_test.go index 6d45dde82b..91f7ca8fa2 100644 --- a/consumers/writers/cassandra/consumer_test.go +++ b/consumers/writers/cassandra/consumer_test.go @@ -10,6 +10,7 @@ import ( "github.com/gofrs/uuid" "github.com/mainflux/mainflux/consumers/writers/cassandra" + casClient "github.com/mainflux/mainflux/internal/clients/cassandra" "github.com/mainflux/mainflux/pkg/transformers/json" "github.com/mainflux/mainflux/pkg/transformers/senml" "github.com/stretchr/testify/assert" @@ -34,11 +35,13 @@ var ( ) func TestSaveSenml(t *testing.T) { - session, err := cassandra.Connect(cassandra.DBConfig{ + session, err := casClient.Connect(casClient.Config{ Hosts: []string{addr}, Keyspace: keyspace, }) require.Nil(t, err, fmt.Sprintf("failed to connect to Cassandra: %s", err)) + err = casClient.InitDB(session, cassandra.Table) + require.Nil(t, err, fmt.Sprintf("failed to initialize to Cassandra: %s", err)) repo := cassandra.New(session) now := time.Now().Unix() msg := senml.Message{ @@ -74,7 +77,7 @@ func TestSaveSenml(t *testing.T) { } func TestSaveJSON(t *testing.T) { - session, err := cassandra.Connect(cassandra.DBConfig{ + session, err := casClient.Connect(casClient.Config{ Hosts: []string{addr}, Keyspace: keyspace, }) diff --git a/consumers/writers/cassandra/init.go b/consumers/writers/cassandra/init.go index b3aa00896c..444d60853d 100644 --- a/consumers/writers/cassandra/init.go +++ b/consumers/writers/cassandra/init.go @@ -3,10 +3,9 @@ package cassandra -import "github.com/gocql/gocql" - const ( - table = `CREATE TABLE IF NOT EXISTS messages ( + // Table contains query for default table created in cassandra db + Table = `CREATE TABLE IF NOT EXISTS messages ( id uuid, channel text, subtopic text, @@ -35,35 +34,3 @@ const ( PRIMARY KEY (channel, created, id) ) WITH CLUSTERING ORDER BY (created DESC)` ) - -// DBConfig contains Cassandra DB specific parameters. -type DBConfig struct { - Hosts []string - Keyspace string - User string - Pass string - Port int -} - -// Connect establishes connection to the Cassandra cluster. -func Connect(cfg DBConfig) (*gocql.Session, error) { - cluster := gocql.NewCluster(cfg.Hosts...) - cluster.Keyspace = cfg.Keyspace - cluster.Consistency = gocql.Quorum - cluster.Authenticator = gocql.PasswordAuthenticator{ - Username: cfg.User, - Password: cfg.Pass, - } - cluster.Port = cfg.Port - - session, err := cluster.CreateSession() - if err != nil { - return nil, err - } - - if err := session.Query(table).Exec(); err != nil { - return nil, err - } - - return session, nil -} diff --git a/consumers/writers/cassandra/setup_test.go b/consumers/writers/cassandra/setup_test.go index 0ef83b9c75..c0f9bb18e3 100644 --- a/consumers/writers/cassandra/setup_test.go +++ b/consumers/writers/cassandra/setup_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/gocql/gocql" - "github.com/mainflux/mainflux/consumers/writers/cassandra" + casClient "github.com/mainflux/mainflux/internal/clients/cassandra" log "github.com/mainflux/mainflux/logger" dockertest "github.com/ory/dockertest/v3" ) @@ -35,7 +35,7 @@ func TestMain(m *testing.M) { return err } - session, err := cassandra.Connect(cassandra.DBConfig{ + session, err := casClient.Connect(casClient.Config{ Hosts: []string{addr}, Keyspace: keyspace, }) diff --git a/consumers/writers/influxdb/README.md b/consumers/writers/influxdb/README.md index fd94ea5dd5..4a67c72772 100644 --- a/consumers/writers/influxdb/README.md +++ b/consumers/writers/influxdb/README.md @@ -11,7 +11,7 @@ default values. | Variable | Description | Default | | ----------------------------- | --------------------------------------------------------------------------------- | ---------------------- | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_INFLUX_WRITER_LOG_LEVEL | Log level for InfluxDB writer (debug, info, warn, error) | error | +| MF_INFLUX_WRITER_LOG_LEVEL | Log level for InfluxDB writer (debug, info, warn, error) | info | | MF_INFLUX_WRITER_PORT | Service HTTP port | 8180 | | MF_INFLUX_WRITER_DB_HOST | InfluxDB host | localhost | | MF_INFLUXDB_PORT | Default port of InfluxDB database | 8086 | diff --git a/consumers/writers/mongodb/README.md b/consumers/writers/mongodb/README.md index f26ecfb003..aa50200827 100644 --- a/consumers/writers/mongodb/README.md +++ b/consumers/writers/mongodb/README.md @@ -11,7 +11,7 @@ default values. | Variable | Description | Default | | ---------------------------- | --------------------------------------------------------------------------------- | ---------------------- | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_MONGO_WRITER_LOG_LEVEL | Log level for MongoDB writer | error | +| MF_MONGO_WRITER_LOG_LEVEL | Log level for MongoDB writer | info | | MF_MONGO_WRITER_PORT | Service HTTP port | 8180 | | MF_MONGO_WRITER_DB | Default MongoDB database name | messages | | MF_MONGO_WRITER_DB_HOST | Default MongoDB database host | localhost | diff --git a/consumers/writers/postgres/README.md b/consumers/writers/postgres/README.md index 2b7f37a6e0..ca9a13ad71 100644 --- a/consumers/writers/postgres/README.md +++ b/consumers/writers/postgres/README.md @@ -11,8 +11,8 @@ default values. | Variable | Description | Default | | ----------------------------------- | --------------------------------------------------------------------------------- | ---------------------- | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_POSTGRES_WRITER_LOG_LEVEL | Service log level | error | -| MF_POSTGRES_WRITER_PORT | Service HTTP port | 9104 | +| MF_POSTGRES_WRITER_LOG_LEVEL | Service log level | info | +| MF_POSTGRES_WRITER_PORT | Service HTTP port | 8180 | | MF_POSTGRES_WRITER_DB_HOST | Postgres DB host | postgres | | MF_POSTGRES_WRITER_DB_PORT | Postgres DB port | 5432 | | MF_POSTGRES_WRITER_DB_USER | Postgres user | mainflux | diff --git a/consumers/writers/postgres/init.go b/consumers/writers/postgres/init.go index 1a055f87c1..5841786243 100644 --- a/consumers/writers/postgres/init.go +++ b/consumers/writers/postgres/init.go @@ -3,47 +3,11 @@ package postgres -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +// Migration of postgres-writer +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "messages_1", @@ -72,7 +36,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/consumers/writers/postgres/setup_test.go b/consumers/writers/postgres/setup_test.go index bcdad41126..a70462fa46 100644 --- a/consumers/writers/postgres/setup_test.go +++ b/consumers/writers/postgres/setup_test.go @@ -11,9 +11,9 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux/consumers/writers/postgres" + pgclient "github.com/mainflux/mainflux/internal/clients/postgres" dockertest "github.com/ory/dockertest/v3" ) @@ -48,7 +48,7 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to docker: %s", err) } - dbConfig := postgres.Config{ + dbConfig := pgclient.Config{ Host: "localhost", Port: port, User: "test", @@ -60,7 +60,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - db, err = postgres.Connect(dbConfig) + db, err = pgclient.SetupDB(dbConfig, *postgres.Migration()) if err != nil { log.Fatalf("Could not setup test DB connection: %s", err) } diff --git a/consumers/writers/timescale/README.md b/consumers/writers/timescale/README.md index bb749e0885..074f9671f5 100644 --- a/consumers/writers/timescale/README.md +++ b/consumers/writers/timescale/README.md @@ -11,8 +11,8 @@ default values. | Variable | Description | Default | | ----------------------------------- | --------------------------------------------------------- | ---------------------- | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_TIMESCALE_WRITER_LOG_LEVEL | Service log level | error | -| MF_TIMESCALE_WRITER_PORT | Service HTTP port | 9104 | +| MF_TIMESCALE_WRITER_LOG_LEVEL | Service log level | info | +| MF_TIMESCALE_WRITER_PORT | Service HTTP port | 8180 | | MF_TIMESCALE_WRITER_DB_HOST | Timescale DB host | timescale | | MF_TIMESCALE_WRITER_DB_PORT | Timescale DB port | 5432 | | MF_TIMESCALE_WRITER_DB_USER | Timescale user | mainflux | diff --git a/consumers/writers/timescale/init.go b/consumers/writers/timescale/init.go index a4d38240b0..ca4aaa9601 100644 --- a/consumers/writers/timescale/init.go +++ b/consumers/writers/timescale/init.go @@ -3,47 +3,11 @@ package timescale -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a TimescaleSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the TimescaleSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +// Migration of timescale-writer +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "messages_1", @@ -72,7 +36,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/consumers/writers/timescale/setup_test.go b/consumers/writers/timescale/setup_test.go index d220d20fdf..c7c4f01fb3 100644 --- a/consumers/writers/timescale/setup_test.go +++ b/consumers/writers/timescale/setup_test.go @@ -11,10 +11,10 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux/consumers/writers/timescale" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" dockertest "github.com/ory/dockertest/v3" ) @@ -49,7 +49,7 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to docker: %s", err) } - dbConfig := timescale.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -61,7 +61,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - db, err = timescale.Connect(dbConfig) + db, err = pgClient.SetupDB(dbConfig, *timescale.Migration()) if err != nil { log.Fatalf("Could not setup test DB connection: %s", err) } diff --git a/docker/.env b/docker/.env index d5073679a7..ebdb547855 100644 --- a/docker/.env +++ b/docker/.env @@ -124,7 +124,7 @@ MF_COAP_ADAPTER_PORT=5683 ### WS MF_WS_ADAPTER_LOG_LEVEL=debug -MF_WS_ADAPTER_PORT=8190 +MF_WS_ADAPTER_PORT=8186 ## Addons Services ### Bootstrap diff --git a/go.mod b/go.mod index 0c73ebefd3..fe906aa967 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/mainflux/mainflux go 1.19 require ( + github.com/caarlos0/env/v7 v7.0.0 github.com/cenkalti/backoff/v4 v4.1.3 github.com/docker/docker v20.10.21+incompatible github.com/eclipse/paho.mqtt.golang v1.4.2 diff --git a/go.sum b/go.sum index 25f26429d7..944ba5869c 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCS github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/caarlos0/env/v7 v7.0.0 h1:cyczlTd/zREwSr9ch/mwaDl7Hse7kJuUY8hvHfXu5WI= +github.com/caarlos0/env/v7 v7.0.0/go.mod h1:LPPWniDUq4JaO6Q41vtlyikhMknqymCLBw0eX4dcH1E= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= diff --git a/http/README.md b/http/README.md index b3c943c509..0e5511908c 100644 --- a/http/README.md +++ b/http/README.md @@ -10,7 +10,7 @@ default values. | Variable | Description | Default | | --------------------------- | ------------------------------------------------------------- | --------------------- | -| MF_HTTP_ADAPTER_LOG_LEVEL | Log level for the HTTP Adapter | error | +| MF_HTTP_ADAPTER_LOG_LEVEL | Log level for the HTTP Adapter | info | | MF_HTTP_ADAPTER_PORT | Service HTTP port | 8180 | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | | MF_HTTP_ADAPTER_CLIENT_TLS | Flag that indicates if TLS should be turned on | false | diff --git a/internal/clients/cassandra/cassandra.go b/internal/clients/cassandra/cassandra.go new file mode 100644 index 0000000000..ba6c01a459 --- /dev/null +++ b/internal/clients/cassandra/cassandra.go @@ -0,0 +1,72 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package cassandra + +import ( + "github.com/gocql/gocql" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" +) + +var ( + errConfig = errors.New("failed to load Cassandra configuration") + errConnect = errors.New("failed to connect to Cassandra database") + errInit = errors.New("failed to execute initialization query in Cassandra ") +) + +// Config contains Cassandra DB specific parameters +type Config struct { + Hosts []string `env:"DB_CLUSTER" envDefault:"127.0.0.1" envSeparator:","` + Keyspace string `env:"DB_KEYSPACE" envDefault:"mainflux"` + User string `env:"DB_USER" envDefault:""` + Pass string `env:"DB_PASS" envDefault:""` + Port int `env:"DB_PORT" envDefault:"9042"` +} + +// Setup load configuration from environment and creates new cassandra connection +func Setup(envPrefix string) (*gocql.Session, error) { + return SetupDB(envPrefix, "") +} + +// SetupDB load configuration from environment, +// creates new cassandra connection and executes +// the initial query in database. +func SetupDB(envPrefix string, initQuery string) (*gocql.Session, error) { + config := Config{} + if err := env.Parse(&config, env.Options{Prefix: envPrefix}); err != nil { + return nil, errors.Wrap(errConfig, err) + } + cs, err := Connect(config) + if err != nil { + return nil, err + } + if initQuery != "" { + if err := InitDB(cs, initQuery); err != nil { + return nil, errors.Wrap(errInit, err) + } + } + return cs, nil +} + +// Connect establishes connection to the Cassandra cluster. +func Connect(cfg Config) (*gocql.Session, error) { + cluster := gocql.NewCluster(cfg.Hosts...) + cluster.Keyspace = cfg.Keyspace + cluster.Consistency = gocql.Quorum + cluster.Authenticator = gocql.PasswordAuthenticator{ + Username: cfg.User, + Password: cfg.Pass, + } + cluster.Port = cfg.Port + + cassSess, err := cluster.CreateSession() + if err != nil { + return nil, errors.Wrap(errConnect, err) + } + return cassSess, nil +} + +func InitDB(cs *gocql.Session, query string) error { + return cs.Query(query).Exec() +} diff --git a/internal/clients/grpc/auth/client.go b/internal/clients/grpc/auth/client.go new file mode 100644 index 0000000000..58b2bcf3c2 --- /dev/null +++ b/internal/clients/grpc/auth/client.go @@ -0,0 +1,28 @@ +package auth + +import ( + "github.com/mainflux/mainflux" + authapi "github.com/mainflux/mainflux/auth/api/grpc" + grpcClient "github.com/mainflux/mainflux/internal/clients/grpc" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" +) + +const envAuthGrpcPrefix = "MF_AUTH_GRPC_" + +var errGrpcConfig = errors.New("failed to load grpc configuration") + +// Setup loads Auth gRPC configuration from environment variable and creates new Auth gRPC API +func Setup(envPrefix, jaegerURL string) (mainflux.AuthServiceClient, grpcClient.ClientHandler, error) { + config := grpcClient.Config{} + if err := env.Parse(&config, env.Options{Prefix: envAuthGrpcPrefix, AltPrefix: envPrefix}); err != nil { + return nil, nil, errors.Wrap(errGrpcConfig, err) + } + + c, ch, err := grpcClient.Setup(config, "auth", jaegerURL) + if err != nil { + return nil, nil, err + } + + return authapi.NewClient(c.Tracer, c.ClientConn, config.Timeout), ch, nil +} diff --git a/internal/clients/grpc/connect.go b/internal/clients/grpc/connect.go new file mode 100644 index 0000000000..dc9e48b16b --- /dev/null +++ b/internal/clients/grpc/connect.go @@ -0,0 +1,119 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package grpc + +import ( + "io" + "time" + + jaegerClient "github.com/mainflux/mainflux/internal/clients/jaeger" + "github.com/mainflux/mainflux/pkg/errors" + "github.com/opentracing/opentracing-go" + gogrpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +var ( + errGrpcConnect = errors.New("failed to connect to grpc server") + errJaeger = errors.New("failed to initialize jaeger ") + errGrpcClose = errors.New("failed to close grpc connection") + errJaegerClose = errors.New("failed to close jaeger connection") +) + +type Config struct { + ClientTLS bool `env:"CLIENT_TLS" envDefault:"false"` + CACerts string `env:"CA_CERTS" envDefault:""` + URL string `env:"URL" envDefault:""` + Timeout time.Duration `env:"TIMEOUT" envDefault:"1s"` +} + +type ClientHandler interface { + Close() error + IsSecure() bool + Secure() string +} + +type Client struct { + *gogrpc.ClientConn + opentracing.Tracer + io.Closer + secure bool +} + +var _ ClientHandler = (*Client)(nil) + +// NewClientHandler create new client handler for gRPC client +func NewClientHandler(c *Client) ClientHandler { + return c +} + +// Connect creates new gRPC client and connect to gRPC server +func Connect(cfg Config) (*gogrpc.ClientConn, bool, error) { + var opts []gogrpc.DialOption + secure := false + tc := insecure.NewCredentials() + + if cfg.ClientTLS && cfg.CACerts != "" { + var err error + tc, err = credentials.NewClientTLSFromFile(cfg.CACerts, "") + if err != nil { + return nil, secure, err + } + secure = true + } + + opts = append(opts, gogrpc.WithTransportCredentials(tc)) + + conn, err := gogrpc.Dial(cfg.URL, opts...) + if err != nil { + return nil, secure, err + } + return conn, secure, nil +} + +// Setup load gRPC configuration from environment variable, creates new gRPC client and connect to gRPC server +func Setup(config Config, svcName, jaegerURL string) (*Client, ClientHandler, error) { + secure := false + + // connect to auth grpc server + grpcClient, secure, err := Connect(config) + if err != nil { + return nil, nil, errors.Wrap(errGrpcConnect, err) + } + + // initialize auth tracer for auth grpc client + tracer, tracerCloser, err := jaegerClient.NewTracer(svcName, jaegerURL) + if err != nil { + grpcClient.Close() + return nil, nil, errors.Wrap(errJaeger, err) + } + c := &Client{grpcClient, tracer, tracerCloser, secure} + + return c, NewClientHandler(c), nil +} + +func (c *Client) Close() error { + var retErr error + err := c.ClientConn.Close() + if err != nil { + retErr = errors.Wrap(errGrpcClose, err) + } + err = c.Closer.Close() + if err != nil { + retErr = errors.Wrap(retErr, errors.Wrap(errJaegerClose, err)) + } + return retErr +} + +func (c *Client) IsSecure() bool { + return c.secure +} + +func (c *Client) Secure() string { + if c.secure { + return "with TLS" + } + return "without TLS" +} diff --git a/internal/clients/grpc/things/client.go b/internal/clients/grpc/things/client.go new file mode 100644 index 0000000000..581f5821a0 --- /dev/null +++ b/internal/clients/grpc/things/client.go @@ -0,0 +1,28 @@ +package things + +import ( + "github.com/mainflux/mainflux" + grpcClient "github.com/mainflux/mainflux/internal/clients/grpc" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" + thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" +) + +const envThingsAuthGrpcPrefix = "MF_THINGS_AUTH_GRPC_" + +var errGrpcConfig = errors.New("failed to load grpc configuration") + +// Setup loads Things gRPC configuration from environment variable and creates new Things gRPC API +func Setup(envPrefix, jaegerURL string) (mainflux.ThingsServiceClient, grpcClient.ClientHandler, error) { + config := grpcClient.Config{} + if err := env.Parse(&config, env.Options{Prefix: envThingsAuthGrpcPrefix, AltPrefix: envPrefix}); err != nil { + return nil, nil, errors.Wrap(errGrpcConfig, err) + } + + c, ch, err := grpcClient.Setup(config, "things", jaegerURL) + if err != nil { + return nil, nil, err + } + + return thingsapi.NewClient(c.ClientConn, c.Tracer, config.Timeout), ch, nil +} diff --git a/internal/clients/influxdb/influxdb.go b/internal/clients/influxdb/influxdb.go new file mode 100644 index 0000000000..0c5cbe5870 --- /dev/null +++ b/internal/clients/influxdb/influxdb.go @@ -0,0 +1,53 @@ +package influxdb + +import ( + "fmt" + "time" + + "github.com/influxdata/influxdb/client/v2" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" +) + +var ( + errConnect = errors.New("failed to create InfluxDB client") + errConfig = errors.New("failed to load InfluxDB client configuration from environment variable") +) + +type Config struct { + Protocol string `env:"PROTOCOL" envDefault:"http"` + Host string `env:"HOST" envDefault:"localhost"` + Port string `env:"PORT" envDefault:"8086"` + Username string `env:"ADMIN_USER" envDefault:"mainflux"` + Password string `env:"ADMIN_PASSWORD" envDefault:"mainflux"` + DbName string `env:"DB" envDefault:"mainflux"` + UserAgent string `env:"USER_AGENT" envDefault:"InfluxDBClient"` + Timeout time.Duration `env:"TIMEOUT"` // Influxdb client configuration by default there is no timeout duration , this field will not have fallback default timeout duration Reference: https://pkg.go.dev/github.com/influxdata/influxdb@v1.10.0/client/v2#HTTPConfig + InsecureSkipVerify bool `env:"INSECURE_SKIP_VERIFY" envDefault:"false"` +} + +// Setup load configuration from environment variable, create InfluxDB client and connect to InfluxDB server +func Setup(envPrefix string) (client.HTTPClient, error) { + config := Config{} + if err := env.Parse(&config, env.Options{Prefix: envPrefix}); err != nil { + return nil, errors.Wrap(errConfig, err) + } + return Connect(config) +} + +// Connect create InfluxDB client and connect to InfluxDB server +func Connect(config Config) (client.HTTPClient, error) { + address := fmt.Sprintf("%s://%s:%s", config.Protocol, config.Host, config.Port) + clientConfig := client.HTTPConfig{ + Addr: address, + Username: config.Username, + Password: config.Password, + UserAgent: config.UserAgent, + Timeout: config.Timeout, + } + client, err := client.NewHTTPClient(clientConfig) + if err != nil { + return nil, errors.Wrap(errConnect, err) + } + return client, nil +} diff --git a/internal/clients/jaeger/tracer.go b/internal/clients/jaeger/tracer.go new file mode 100644 index 0000000000..5fbaf46d74 --- /dev/null +++ b/internal/clients/jaeger/tracer.go @@ -0,0 +1,41 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package jaeger + +import ( + "errors" + "io" + "io/ioutil" + + "github.com/opentracing/opentracing-go" + jconfig "github.com/uber/jaeger-client-go/config" +) + +var ( + errNoUrl = errors.New("URL is empty") + errNoSvcName = errors.New("Service Name is empty") +) + +// NewTracer initializes Jaeger +func NewTracer(svcName, url string) (opentracing.Tracer, io.Closer, error) { + if url == "" { + return opentracing.NoopTracer{}, ioutil.NopCloser(nil), errNoUrl + } + + if svcName == "" { + return opentracing.NoopTracer{}, ioutil.NopCloser(nil), errNoSvcName + } + + return jconfig.Configuration{ + ServiceName: svcName, + Sampler: &jconfig.SamplerConfig{ + Type: "const", + Param: 1, + }, + Reporter: &jconfig.ReporterConfig{ + LocalAgentHostPort: url, + LogSpans: true, + }, + }.NewTracer() +} diff --git a/internal/clients/mongo/mongo.go b/internal/clients/mongo/mongo.go new file mode 100644 index 0000000000..1d6f01f799 --- /dev/null +++ b/internal/clients/mongo/mongo.go @@ -0,0 +1,51 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package mongodb + +import ( + "context" + "fmt" + + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + errConfig = errors.New("failed to load mongodb configuration") + errConnect = errors.New("failed to connect to mongodb server") +) + +// Config defines the options that are used when connecting to a MongoDB instance +type Config struct { + Host string `env:"HOST" envDefault:"localhost"` + Port string `env:"PORT" envDefault:"27017"` + DB string `env:"DB" envDefault:"messages"` +} + +// Connect creates a connection to the MongoDB instance +func Connect(cfg Config) (*mongo.Database, error) { + addr := fmt.Sprintf("mongodb://%s:%s", cfg.Host, cfg.Port) + client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) + if err != nil { + return nil, errors.Wrap(errConnect, err) + } + + db := client.Database(cfg.DB) + return db, nil +} + +// Setup load configuration from environment, create new MongoDB client and connect to MongoDB server +func Setup(envPrefix string) (*mongo.Database, error) { + cfg := Config{} + if err := env.Parse(&cfg, env.Options{Prefix: envPrefix}); err != nil { + return nil, errors.Wrap(errConfig, err) + } + db, err := Connect(cfg) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/internal/clients/postgres/postgres.go b/internal/clients/postgres/postgres.go new file mode 100644 index 0000000000..80888c267f --- /dev/null +++ b/internal/clients/postgres/postgres.go @@ -0,0 +1,86 @@ +package postgres + +import ( + "fmt" + + _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access + "github.com/jmoiron/sqlx" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" + migrate "github.com/rubenv/sql-migrate" +) + +var ( + errConfig = errors.New("failed to load postgresql configuration") + errConnect = errors.New("failed to connect to postgresql server") + errMigration = errors.New("failed to apply migrations") +) + +type Config struct { + Host string `env:"DB_HOST,notEmpty" envDefault:"localhost"` + Port string `env:"DB_PORT,notEmpty" envDefault:"5432"` + User string `env:"DB_USER,notEmpty" envDefault:"mainflux"` + Pass string `env:"DB_PASS,notEmpty" envDefault:"mainflux"` + Name string `env:"DB" envDefault:""` + SSLMode string `env:"DB_SSL_MODE,notEmpty" envDefault:"disable"` + SSLCert string `env:"DB_SSL_CERT" envDefault:""` + SSLKey string `env:"DB_SSL_KEY" envDefault:""` + SSLRootCert string `env:"DB_SSL_ROOT_CERT" envDefault:""` +} + +// Setup creates a connection to the PostgreSQL instance and applies any +// unapplied database migrations. A non-nil error is returned to indicate failure. +func Setup(prefix string, migrations migrate.MemoryMigrationSource) (*sqlx.DB, error) { + return SetupWithConfig(prefix, migrations, Config{}) +} + +// SetupWithConfig creates a connection to the PostgreSQL instance and applies any +// unapplied database migrations. A non-nil error is returned to indicate failure. +func SetupWithConfig(prefix string, migrations migrate.MemoryMigrationSource, defConfig Config) (*sqlx.DB, error) { + cfg := defConfig + if err := env.Parse(&cfg, env.Options{Prefix: prefix}); err != nil { + return nil, errors.Wrap(errConfig, err) + } + return SetupDB(cfg, migrations) +} + +// SetupDB creates a connection to the PostgreSQL instance and applies any +// unapplied database migrations. A non-nil error is returned to indicate failure. +func SetupDB(cfg Config, migrations migrate.MemoryMigrationSource) (*sqlx.DB, error) { + db, err := Connect(cfg) + if err != nil { + return nil, err + } + if err := MigrateDB(db, migrations); err != nil { + return nil, err + } + return db, nil +} + +// Connect creates a connection to the PostgreSQL instance +func Connect(cfg Config) (*sqlx.DB, error) { + url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) + + db, err := sqlx.Open("pgx", url) + if err != nil { + return nil, errors.Wrap(errConnect, err) + } + + return db, nil +} + +// MigrateDB applies any unapplied database migrations +func MigrateDB(db *sqlx.DB, migrations migrate.MemoryMigrationSource) error { + _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) + if err != nil { + return errors.Wrap(errMigration, err) + } + return nil +} + +func (c *Config) LoadEnv(prefix string) error { + if err := env.Parse(c, env.Options{Prefix: prefix}); err != nil { + return errors.Wrap(errConfig, err) + } + return nil +} diff --git a/internal/clients/redis/redis.go b/internal/clients/redis/redis.go new file mode 100644 index 0000000000..687a1c8d37 --- /dev/null +++ b/internal/clients/redis/redis.go @@ -0,0 +1,51 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package redis + +import ( + "strconv" + + r "github.com/go-redis/redis/v8" + "github.com/mainflux/mainflux/internal/env" + "github.com/mainflux/mainflux/pkg/errors" +) + +var ( + errConfig = errors.New("failed to load redis client configuration") + errConnect = errors.New("failed to connect to redis server") +) + +// Config of RedisDB +type Config struct { + URL string `env:"URL" envDefault:"localhost:6379"` + Pass string `env:"PASS" envDefault:""` + DB string `env:"DB" envDefault:"0"` +} + +// Setup load configuration from environment, creates new RedisDB client and connect to RedisDB Server +func Setup(prefix string) (*r.Client, error) { + cfg := Config{} + if err := env.Parse(&cfg, env.Options{Prefix: prefix}); err != nil { + return nil, errors.Wrap(errConfig, err) + } + client, err := Connect(cfg) + if err != nil { + return nil, errors.Wrap(errConnect, err) + } + return client, nil +} + +// Connect create new RedisDB client and connect to RedisDB server +func Connect(cfg Config) (*r.Client, error) { + db, err := strconv.Atoi(cfg.DB) + if err != nil { + return nil, err + } + + return r.NewClient(&r.Options{ + Addr: cfg.URL, + Password: cfg.Pass, + DB: db, + }), nil +} diff --git a/internal/close.go b/internal/close.go new file mode 100644 index 0000000000..dd66211ed8 --- /dev/null +++ b/internal/close.go @@ -0,0 +1,12 @@ +package internal + +import ( + grpcClient "github.com/mainflux/mainflux/internal/clients/grpc" + logger "github.com/mainflux/mainflux/logger" +) + +func Close(log logger.Logger, clientHandler grpcClient.ClientHandler) { + if err := clientHandler.Close(); err != nil { + log.Warn(err.Error()) + } +} diff --git a/internal/email/email.go b/internal/email/email.go index f4f1394bd9..1f1ec6af93 100644 --- a/internal/email/email.go +++ b/internal/email/email.go @@ -30,15 +30,15 @@ type email struct { Footer string } -// Config email agent configuration. +// Config email agent configuration type Config struct { - Host string - Port string - Username string - Password string - FromAddress string - FromName string - Template string + Host string `env:"MF_EMAIL_HOST" envDefault:"localhost"` + Port string `env:"MF_EMAIL_PORT" envDefault:"25"` + Username string `env:"MF_EMAIL_USERNAME" envDefault:"root"` + Password string `env:"MF_EMAIL_PASSWORD" envDefault:""` + FromAddress string `env:"MF_EMAIL_FROM_ADDRESS" envDefault:""` + FromName string `env:"MF_EMAIL_FROM_NAME" envDefault:""` + Template string `env:"MF_EMAIL_TEMPLATE" envDefault:"email.tmpl"` } // Agent for mailing diff --git a/internal/env/load.go b/internal/env/load.go new file mode 100644 index 0000000000..30f9ceada5 --- /dev/null +++ b/internal/env/load.go @@ -0,0 +1,10 @@ +package env + +// NewConfig gets configuration from environment variable +func NewConfig[T any](opts ...Options) (T, error) { + var cfg T + if err := Parse(&cfg, opts...); err != nil { + return cfg, err + } + return cfg, nil +} diff --git a/internal/env/parser.go b/internal/env/parser.go new file mode 100644 index 0000000000..4bf127d809 --- /dev/null +++ b/internal/env/parser.go @@ -0,0 +1,117 @@ +package env + +import ( + "github.com/caarlos0/env/v7" + "github.com/mainflux/mainflux/internal/clients/grpc" + "github.com/mainflux/mainflux/internal/server" +) + +type Options struct { + // Environment keys and values that will be accessible for the service + Environment map[string]string + + // TagName specifies another tagname to use rather than the default env + TagName string + + // RequiredIfNoDef automatically sets all env as required if they do not declare 'envDefault' + RequiredIfNoDef bool + + // OnSet allows to run a function when a value is set + OnSet env.OnSetFn + + // Prefix define a prefix for each key + Prefix string + + // AltPrefix define a alternate prefix for each key + AltPrefix string +} + +func Parse(v interface{}, opts ...Options) error { + actOpt := []env.Options{} + altPrefix := "" + + for _, opt := range opts { + actOpt = append(actOpt, env.Options{ + Environment: opt.Environment, + TagName: opt.TagName, + RequiredIfNoDef: opt.RequiredIfNoDef, + OnSet: opt.OnSet, + Prefix: opt.Prefix, + }) + if opt.AltPrefix != "" { + altPrefix = opt.AltPrefix + } + } + + if altPrefix == "" { + return env.Parse(v, actOpt...) + } + + switch cfg := v.(type) { + case *grpc.Config: + return parseGrpcConfig(cfg, altPrefix, actOpt...) + case *server.Config: + return parseServerConfig(cfg, altPrefix, actOpt...) + default: + return env.Parse(v, actOpt...) + } +} + +func parseGrpcConfig(cfg *grpc.Config, altPrefix string, opts ...env.Options) error { + if err := env.Parse(cfg, opts...); err != nil { + return err + } + + if !cfg.ClientTLS || cfg.CACerts == "" { + altOpts := []env.Options{} + for _, opt := range opts { + if opt.Prefix != "" { + opt.Prefix = altPrefix + } + altOpts = append(altOpts, opt) + } + altCfg := grpc.Config{} + if err := env.Parse(&altCfg, altOpts...); err != nil { + return err + } + if cfg.CACerts == "" && altCfg.CACerts != "" { + cfg.CACerts = altCfg.CACerts + } + if !cfg.ClientTLS && altCfg.ClientTLS { + cfg.ClientTLS = altCfg.ClientTLS + } + } + + return nil +} + +func parseServerConfig(cfg *server.Config, altPrefix string, opts ...env.Options) error { + copyConfig := cfg + if err := env.Parse(cfg, opts...); err != nil { + return err + } + + if cfg.CertFile == "" || cfg.KeyFile == "" || cfg.Port == "" || cfg.Port == copyConfig.Port { + altOpts := []env.Options{} + for _, opt := range opts { + if opt.Prefix != "" { + opt.Prefix = altPrefix + } + altOpts = append(altOpts, opt) + } + altCfg := server.Config{} + if err := env.Parse(&altCfg, altOpts...); err != nil { + return err + } + if cfg.CertFile == "" && altCfg.CertFile != "" { + cfg.CertFile = altCfg.CertFile + } + if cfg.KeyFile == "" && altCfg.KeyFile != "" { + cfg.KeyFile = altCfg.KeyFile + } + if (cfg.Port == "" || cfg.Port == copyConfig.Port) && altCfg.Port != "" { + cfg.Port = altCfg.Port + } + } + return nil +} diff --git a/internal/metrics.go b/internal/metrics.go new file mode 100644 index 0000000000..a894343278 --- /dev/null +++ b/internal/metrics.go @@ -0,0 +1,26 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + kitprometheus "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +func MakeMetrics(namespace, subsystem string) (*kitprometheus.Counter, *kitprometheus.Summary) { + counter := kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "request_count", + Help: "Number of requests received.", + }, []string{"method"}) + latency := kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "request_latency_microseconds", + Help: "Total duration of requests in microseconds.", + }, []string{"method"}) + + return counter, latency +} diff --git a/internal/server/coap/coap.go b/internal/server/coap/coap.go new file mode 100644 index 0000000000..fcb692d609 --- /dev/null +++ b/internal/server/coap/coap.go @@ -0,0 +1,66 @@ +package coap + +import ( + "context" + "fmt" + "time" + + "github.com/mainflux/mainflux/internal/server" + "github.com/mainflux/mainflux/logger" + gocoap "github.com/plgd-dev/go-coap/v2" + "github.com/plgd-dev/go-coap/v2/mux" +) + +const ( + stopWaitTime = 5 * time.Second +) + +type Server struct { + server.BaseServer + handler mux.HandlerFunc +} + +var _ server.Server = (*Server)(nil) + +func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, handler mux.HandlerFunc, logger logger.Logger) server.Server { + listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port) + return &Server{ + BaseServer: server.BaseServer{ + Ctx: ctx, + Cancel: cancel, + Name: name, + Address: listenFullAddress, + Config: config, + Logger: logger, + }, + handler: handler, + } +} + +func (s *Server) Start() error { + errCh := make(chan error) + s.Logger.Info(fmt.Sprintf("%s service started using http, exposed port %s", s.Name, s.Address)) + go func() { + errCh <- gocoap.ListenAndServe("udp", s.Address, s.handler) + }() + + select { + case <-s.Ctx.Done(): + return s.Stop() + case err := <-errCh: + return err + } + +} + +func (s *Server) Stop() error { + defer s.Cancel() + c := make(chan bool) + defer close(c) + select { + case <-c: + case <-time.After(stopWaitTime): + } + s.Logger.Info(fmt.Sprintf("%s service shutdown of http at %s", s.Name, s.Address)) + return nil +} diff --git a/internal/server/grpc/grpc.go b/internal/server/grpc/grpc.go new file mode 100644 index 0000000000..cc16241b38 --- /dev/null +++ b/internal/server/grpc/grpc.go @@ -0,0 +1,94 @@ +package grpc + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/mainflux/mainflux/internal/server" + "github.com/mainflux/mainflux/logger" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +const ( + stopWaitTime = 5 * time.Second +) + +type Server struct { + server.BaseServer + server *grpc.Server + registerService serviceRegister +} + +type serviceRegister func(srv *grpc.Server) + +var _ server.Server = (*Server)(nil) + +func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger logger.Logger) server.Server { + listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port) + return &Server{ + BaseServer: server.BaseServer{ + Ctx: ctx, + Cancel: cancel, + Name: name, + Address: listenFullAddress, + Config: config, + Logger: logger, + }, + registerService: registerService, + } +} + +func (s *Server) Start() error { + errCh := make(chan error) + + listener, err := net.Listen("tcp", s.Address) + if err != nil { + return fmt.Errorf("failed to listen on port %s: %w", s.Address, err) + } + + switch { + case s.Config.CertFile != "" || s.Config.KeyFile != "": + creds, err := credentials.NewServerTLSFromFile(s.Config.CertFile, s.Config.KeyFile) + if err != nil { + return fmt.Errorf("failed to load auth certificates: %w", err) + } + s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile)) + s.server = grpc.NewServer(grpc.Creds(creds)) + default: + s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s without TLS", s.Name, s.Address)) + s.server = grpc.NewServer() + } + + s.registerService(s.server) + + go func() { + errCh <- s.server.Serve(listener) + }() + + select { + case <-s.Ctx.Done(): + return s.Stop() + case err := <-errCh: + s.Cancel() + return err + } +} + +func (s *Server) Stop() error { + defer s.Cancel() + c := make(chan bool) + go func() { + defer close(c) + s.server.GracefulStop() + }() + select { + case <-c: + case <-time.After(stopWaitTime): + } + s.Logger.Info(fmt.Sprintf("%s gRPC service shutdown at %s", s.Name, s.Address)) + + return nil +} diff --git a/internal/server/http/http.go b/internal/server/http/http.go new file mode 100644 index 0000000000..bf9d83e3bd --- /dev/null +++ b/internal/server/http/http.go @@ -0,0 +1,76 @@ +package http + +import ( + "context" + "fmt" + "net/http" + "time" + + mfserver "github.com/mainflux/mainflux/internal/server" + "github.com/mainflux/mainflux/logger" +) + +const ( + stopWaitTime = 5 * time.Second + httpProtocol = "http" + httpsProtocol = "https" +) + +type Server struct { + mfserver.BaseServer + server *http.Server +} + +var _ mfserver.Server = (*Server)(nil) + +func New(ctx context.Context, cancel context.CancelFunc, name string, config mfserver.Config, handler http.Handler, logger logger.Logger) mfserver.Server { + listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port) + server := &http.Server{Addr: listenFullAddress, Handler: handler} + return &Server{ + BaseServer: mfserver.BaseServer{ + Ctx: ctx, + Cancel: cancel, + Name: name, + Address: listenFullAddress, + Config: config, + Logger: logger, + }, + server: server, + } +} + +func (s *Server) Start() error { + errCh := make(chan error) + s.Protocol = httpProtocol + switch { + case s.Config.CertFile != "" || s.Config.KeyFile != "": + s.Protocol = httpsProtocol + s.Logger.Info(fmt.Sprintf("%s service %s server listening at %s with TLS cert %s and key %s", s.Name, s.Protocol, s.Address, s.Config.CertFile, s.Config.KeyFile)) + go func() { + errCh <- s.server.ListenAndServeTLS(s.Config.CertFile, s.Config.KeyFile) + }() + default: + s.Logger.Info(fmt.Sprintf("%s service %s server listening at %s without TLS", s.Name, s.Protocol, s.Address)) + go func() { + errCh <- s.server.ListenAndServe() + }() + } + select { + case <-s.Ctx.Done(): + return s.Stop() + case err := <-errCh: + return err + } +} + +func (s *Server) Stop() error { + defer s.Cancel() + ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime) + defer cancelShutdown() + if err := s.server.Shutdown(ctxShutdown); err != nil { + s.Logger.Error(fmt.Sprintf("%s service %s server error occurred during shutdown at %s: %s", s.Name, s.Protocol, s.Address, err)) + return fmt.Errorf("%s service %s server error occurred during shutdown at %s: %w", s.Name, s.Protocol, s.Address, err) + } + s.Logger.Info(fmt.Sprintf("%s %s service shutdown of http at %s", s.Name, s.Protocol, s.Address)) + return nil +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000000..f7a9dbee0d --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,66 @@ +package server + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/mainflux/mainflux/logger" +) + +type Server interface { + Start() error + Stop() error +} + +type Config struct { + Host string `env:"HOST" envDefault:""` + Port string `env:"PORT" envDefault:""` + CertFile string `env:"SERVER_CERT" envDefault:""` + KeyFile string `env:"SERVER_KEY" envDefault:""` +} + +type BaseServer struct { + Ctx context.Context + Cancel context.CancelFunc + Name string + Address string + Config Config + Logger logger.Logger + Protocol string +} + +func stopAllServer(servers ...Server) error { + var err error + for _, server := range servers { + err1 := server.Stop() + if err1 != nil { + if err == nil { + err = fmt.Errorf("%w", err1) + } else { + err = fmt.Errorf("%v ; %w", err, err1) + } + } + } + return err +} + +func StopSignalHandler(ctx context.Context, cancel context.CancelFunc, logger logger.Logger, svcName string, servers ...Server) error { + var err error + c := make(chan os.Signal, 2) + signal.Notify(c, syscall.SIGINT, syscall.SIGABRT) + select { + case sig := <-c: + defer cancel() + err = stopAllServer(servers...) + if err != nil { + logger.Error(fmt.Sprintf("%s service error during shutdown: %v", svcName, err)) + } + logger.Info(fmt.Sprintf("%s service shutdown by signal: %s", svcName, sig)) + return err + case <-ctx.Done(): + return nil + } +} diff --git a/lora/README.md b/lora/README.md index 27e5b52612..04f56a22f8 100644 --- a/lora/README.md +++ b/lora/README.md @@ -14,7 +14,7 @@ default values. | Variable | Description | Default | |----------------------------------|---------------------------------------|---------------------------------| | MF_LORA_ADAPTER_HTTP_PORT | Service HTTP port | 8180 | -| MF_LORA_ADAPTER_LOG_LEVEL | Service Log level | error | +| MF_LORA_ADAPTER_LOG_LEVEL | Service Log level | info | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | | MF_LORA_ADAPTER_MESSAGES_URL | LoRa adapter MQTT broker URL | tcp://localhost:1883 | | MF_LORA_ADAPTER_MESSAGES_TOPIC | LoRa adapter MQTT subscriber Topic | application/+/device/+/event/up | diff --git a/mqtt/README.md b/mqtt/README.md index 5ccb8fa02c..118f5ad667 100644 --- a/mqtt/README.md +++ b/mqtt/README.md @@ -12,7 +12,7 @@ default values. | Variable | Description | Default | |------------------------------------------|------------------------------------------------------------------|-----------------------| -| MF_MQTT_ADAPTER_LOG_LEVEL | mProxy Log level | error | +| MF_MQTT_ADAPTER_LOG_LEVEL | mProxy Log level | info | | MF_MQTT_ADAPTER_MQTT_PORT | mProxy port | 1883 | | MF_MQTT_ADAPTER_MQTT_TARGET_HOST | MQTT broker host | 0.0.0.0 | | MF_MQTT_ADAPTER_MQTT_TARGET_PORT | MQTT broker port | 1883 | diff --git a/opcua/README.md b/opcua/README.md index c78399f70f..f0c6d64153 100644 --- a/opcua/README.md +++ b/opcua/README.md @@ -14,7 +14,7 @@ default values. | Variable | Description | Default | |----------------------------------|--------------------------------------------------|----------------------------| | MF_OPCUA_ADAPTER_HTTP_PORT | Service HTTP port | 8180 | -| MF_OPCUA_ADAPTER_LOG_LEVEL | Service Log level | error | +| MF_OPCUA_ADAPTER_LOG_LEVEL | Service Log level | info | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | | MF_OPCUA_ADAPTER_INTERVAL_MS | OPC-UA Server Interval in milliseconds | 1000 | | MF_OPCUA_ADAPTER_POLICY | OPC-UA Server Policy | | diff --git a/opcua/adapter.go b/opcua/adapter.go index 8d7133ba7b..0dc2f317e8 100644 --- a/opcua/adapter.go +++ b/opcua/adapter.go @@ -46,11 +46,11 @@ type Service interface { type Config struct { ServerURI string NodeID string - Interval string - Policy string - Mode string - CertFile string - KeyFile string + Interval string `env:"MF_OPCUA_ADAPTER_INTERVAL_MS" envDefault:"1000"` + Policy string `env:"MF_OPCUA_ADAPTER_POLICY" envDefault:""` + Mode string `env:"MF_OPCUA_ADAPTER_MODE" envDefault:""` + CertFile string `env:"MF_OPCUA_ADAPTER_CERT_FILE" envDefault:""` + KeyFile string `env:"MF_OPCUA_ADAPTER_KEY_FILE" envDefault:""` } var _ Service = (*adapterService)(nil) diff --git a/opcua/db/subs.go b/opcua/db/subs.go index dc2444ef60..99ca7a270f 100644 --- a/opcua/db/subs.go +++ b/opcua/db/subs.go @@ -32,7 +32,7 @@ type Node struct { func Save(serverURI, nodeID string) error { file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm) if err != nil { - errors.Wrap(errWriteFile, err) + return errors.Wrap(errWriteFile, err) } csvWriter := csv.NewWriter(file) csvWriter.Write([]string{serverURI, nodeID}) diff --git a/readers/cassandra/README.md b/readers/cassandra/README.md index 1d2ef40c0d..50ed039670 100644 --- a/readers/cassandra/README.md +++ b/readers/cassandra/README.md @@ -10,6 +10,7 @@ default values. | Variable | Description | Default | |---------------------------------|-----------------------------------------------------|----------------| +| MF_CASSANDRA_READER_LOG_LEVEL | Service log level | info | | MF_CASSANDRA_READER_PORT | Service HTTP port | 8180 | | MF_CASSANDRA_READER_DB_CLUSTER | Cassandra cluster comma separated addresses | 127.0.0.1 | | MF_CASSANDRA_READER_DB_USER | Cassandra DB username | | diff --git a/readers/cassandra/init.go b/readers/cassandra/init.go deleted file mode 100644 index a466bfa095..0000000000 --- a/readers/cassandra/init.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Mainflux -// SPDX-License-Identifier: Apache-2.0 - -package cassandra - -import ( - "github.com/gocql/gocql" -) - -// DBConfig contains Cassandra DB specific parameters. -type DBConfig struct { - Hosts []string - Keyspace string - User string - Pass string - Port int -} - -// Connect establishes connection to the Cassandra cluster. -func Connect(cfg DBConfig) (*gocql.Session, error) { - cluster := gocql.NewCluster(cfg.Hosts...) - cluster.Keyspace = cfg.Keyspace - cluster.Consistency = gocql.Quorum - cluster.Authenticator = gocql.PasswordAuthenticator{ - Username: cfg.User, - Password: cfg.Pass, - } - cluster.Port = cfg.Port - - return cluster.CreateSession() -} diff --git a/readers/cassandra/messages_test.go b/readers/cassandra/messages_test.go index c4c8386820..29d46a73d8 100644 --- a/readers/cassandra/messages_test.go +++ b/readers/cassandra/messages_test.go @@ -8,7 +8,9 @@ import ( "testing" "time" + "github.com/mainflux/mainflux/consumers/writers/cassandra" cwriter "github.com/mainflux/mainflux/consumers/writers/cassandra" + casClient "github.com/mainflux/mainflux/internal/clients/cassandra" "github.com/mainflux/mainflux/pkg/transformers/json" "github.com/mainflux/mainflux/pkg/transformers/senml" "github.com/mainflux/mainflux/pkg/uuid" @@ -45,12 +47,14 @@ var ( ) func TestReadSenml(t *testing.T) { - session, err := creader.Connect(creader.DBConfig{ + session, err := casClient.Connect(casClient.Config{ Hosts: []string{addr}, Keyspace: keyspace, }) require.Nil(t, err, fmt.Sprintf("failed to connect to Cassandra: %s", err)) defer session.Close() + err = casClient.InitDB(session, cassandra.Table) + require.Nil(t, err, fmt.Sprintf("failed to initialize to Cassandra: %s", err)) writer := cwriter.New(session) chanID, err := idProvider.ID() @@ -385,7 +389,7 @@ func TestReadSenml(t *testing.T) { } func TestReadJSON(t *testing.T) { - session, err := creader.Connect(creader.DBConfig{ + session, err := casClient.Connect(casClient.Config{ Hosts: []string{addr}, Keyspace: keyspace, }) diff --git a/readers/cassandra/setup_test.go b/readers/cassandra/setup_test.go index 4de0322d1f..7f4797a54e 100644 --- a/readers/cassandra/setup_test.go +++ b/readers/cassandra/setup_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/gocql/gocql" - "github.com/mainflux/mainflux/consumers/writers/cassandra" + casClient "github.com/mainflux/mainflux/internal/clients/cassandra" log "github.com/mainflux/mainflux/logger" dockertest "github.com/ory/dockertest/v3" ) @@ -35,7 +35,7 @@ func TestMain(m *testing.M) { return err } - session, err := cassandra.Connect(cassandra.DBConfig{ + session, err := casClient.Connect(casClient.Config{ Hosts: []string{addr}, Keyspace: keyspace, }) diff --git a/readers/influxdb/README.md b/readers/influxdb/README.md index 03a89c3649..f74ef62006 100644 --- a/readers/influxdb/README.md +++ b/readers/influxdb/README.md @@ -10,6 +10,7 @@ default values. | Variable | Description | Default | |------------------------------|-----------------------------------------------------|----------------| +| MF_INFLUX_READER_LOG_LEVEL | Service log level | info | | MF_INFLUX_READER_PORT | Service HTTP port | 8180 | | MF_INFLUXDB_HOST | InfluxDB host | localhost | | MF_INFLUXDB_PORT | Default port of InfluxDB database | 8086 | diff --git a/readers/mongodb/README.md b/readers/mongodb/README.md index 0d6826a6b8..600a89c43c 100644 --- a/readers/mongodb/README.md +++ b/readers/mongodb/README.md @@ -10,6 +10,7 @@ default values. | Variable | Description | Default | |-----------------------------|-----------------------------------------------------|----------------| +| MF_MONGO_READER_LOG_LEVEL | Service log level | info | | MF_MONGO_READER_PORT | Service HTTP port | 8180 | | MF_MONGO_READER_DB | MongoDB database name | messages | | MF_MONGO_READER_DB_HOST | MongoDB database host | localhost | diff --git a/readers/postgres/README.md b/readers/postgres/README.md index 037ef1b148..0794787978 100644 --- a/readers/postgres/README.md +++ b/readers/postgres/README.md @@ -10,7 +10,7 @@ default values. | Variable | Description | Default | |-------------------------------------|----------------------------------------------|----------------| -| MF_POSTGRES_READER_LOG_LEVEL | Service log level | debug | +| MF_POSTGRES_READER_LOG_LEVEL | Service log level | info | | MF_POSTGRES_READER_PORT | Service HTTP port | 8180 | | MF_POSTGRES_READER_CLIENT_TLS | TLS mode flag | false | | MF_POSTGRES_READER_CA_CERTS | Path to trusted CAs in PEM format | | diff --git a/readers/postgres/init.go b/readers/postgres/init.go index ed087c4bec..95e3a94bfd 100644 --- a/readers/postgres/init.go +++ b/readers/postgres/init.go @@ -6,7 +6,6 @@ package postgres import ( "fmt" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" migrate "github.com/rubenv/sql-migrate" ) diff --git a/readers/postgres/messages.go b/readers/postgres/messages.go index 5eb021d2ab..486075cc44 100644 --- a/readers/postgres/messages.go +++ b/readers/postgres/messages.go @@ -9,7 +9,7 @@ import ( "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5/pgconn" - "github.com/jmoiron/sqlx" // required for DB access + "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux/pkg/errors" "github.com/mainflux/mainflux/pkg/transformers/senml" "github.com/mainflux/mainflux/readers" diff --git a/readers/postgres/setup_test.go b/readers/postgres/setup_test.go index 589712fc4b..9ca1746638 100644 --- a/readers/postgres/setup_test.go +++ b/readers/postgres/setup_test.go @@ -36,9 +36,9 @@ func TestMain(m *testing.M) { } port := container.GetPort("5432/tcp") + url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port) if err = pool.Retry(func() error { - url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port) db, err = sqlx.Open("pgx", url) if err != nil { return err diff --git a/readers/timescale/README.md b/readers/timescale/README.md index 10f4b4dfe7..5452380ebc 100644 --- a/readers/timescale/README.md +++ b/readers/timescale/README.md @@ -10,11 +10,11 @@ default values. | Variable | Description | Default | |--------------------------------------|---------------------------------------------|----------------| -| MF_TIMESCALE_READER_LOG_LEVEL | Service log level | debug | +| MF_TIMESCALE_READER_LOG_LEVEL | Service log level | info | | MF_TIMESCALE_READER_PORT | Service HTTP port | 8180 | | MF_TIMESCALE_READER_CLIENT_TLS | TLS mode flag | false | | MF_TIMESCALE_READER_CA_CERTS | Path to trusted CAs in PEM format | | -| MF_TIMESCALE_READER_DB_HOST | Timescale DB host | timescale | +| MF_TIMESCALE_READER_DB_HOST | Timescale DB host | localhost | | MF_TIMESCALE_READER_DB_PORT | Timescale DB port | 5432 | | MF_TIMESCALE_READER_DB_USER | Timescale user | mainflux | | MF_TIMESCALE_READER_DB_PASS | Timescale password | mainflux | diff --git a/readers/timescale/init.go b/readers/timescale/init.go index dcfe5fc4d3..9958bc6abe 100644 --- a/readers/timescale/init.go +++ b/readers/timescale/init.go @@ -6,7 +6,6 @@ package timescale import ( "fmt" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" migrate "github.com/rubenv/sql-migrate" ) diff --git a/readers/timescale/setup_test.go b/readers/timescale/setup_test.go index dda152dc0a..02b4c4b06a 100644 --- a/readers/timescale/setup_test.go +++ b/readers/timescale/setup_test.go @@ -36,9 +36,9 @@ func TestMain(m *testing.M) { } port := container.GetPort("5432/tcp") + url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port) if err = pool.Retry(func() error { - url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port) db, err = sqlx.Open("pgx", url) if err != nil { return err diff --git a/things/README.md b/things/README.md index 106df49e0d..6680d77b90 100644 --- a/things/README.md +++ b/things/README.md @@ -18,7 +18,7 @@ default values. | Variable | Description | Default | | -------------------------- | ----------------------------------------------------------------------- | -------------- | -| MF_THINGS_LOG_LEVEL | Log level for Things (debug, info, warn, error) | error | +| MF_THINGS_LOG_LEVEL | Log level for Things (debug, info, warn, error) | info | | MF_THINGS_DB_HOST | Database host address | localhost | | MF_THINGS_DB_PORT | Database host port | 5432 | | MF_THINGS_DB_USER | Database user | mainflux | diff --git a/things/postgres/init.go b/things/postgres/init.go index 1a83015cb2..be1a7b350f 100644 --- a/things/postgres/init.go +++ b/things/postgres/init.go @@ -3,47 +3,11 @@ package postgres -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - migrations := &migrate.MemoryMigrationSource{ +// Migration of Things service +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "things_1", @@ -102,6 +66,4 @@ func migrateDB(db *sqlx.DB) error { }, } - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/things/postgres/setup_test.go b/things/postgres/setup_test.go index cf2eb2aa65..823fea1028 100644 --- a/things/postgres/setup_test.go +++ b/things/postgres/setup_test.go @@ -11,8 +11,8 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/things/postgres" dockertest "github.com/ory/dockertest/v3" ) @@ -54,7 +54,7 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to docker: %s", err) } - dbConfig := postgres.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -66,7 +66,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - if db, err = postgres.Connect(dbConfig); err != nil { + if db, err = pgClient.SetupDB(dbConfig, *postgres.Migration()); err != nil { log.Fatalf("Could not setup test DB connection: %s", err) } diff --git a/twins/README.md b/twins/README.md index d8cb281f48..1cf5ac8a28 100644 --- a/twins/README.md +++ b/twins/README.md @@ -14,8 +14,8 @@ default values. | Variable | Description | Default | |----------------------------|----------------------------------------------------------------------|-----------------------| -| MF_TWINS_LOG_LEVEL | Log level for twin service (debug, info, warn, error) | error | -| MF_TWINS_HTTP_PORT | Twins service HTTP port | 9021 | +| MF_TWINS_LOG_LEVEL | Log level for twin service (debug, info, warn, error) | info | +| MF_TWINS_HTTP_PORT | Twins service HTTP port | 8180 | | MF_TWINS_SERVER_CERT | Path to server certificate in PEM format | | | MF_TWINS_SERVER_KEY | Path to server key in PEM format | | | MF_JAEGER_URL | Jaeger server URL | | diff --git a/users/README.md b/users/README.md index 5c0ec4a52d..4a011e555f 100644 --- a/users/README.md +++ b/users/README.md @@ -18,7 +18,7 @@ default values. | Variable | Description | Default | | ------------------------- | ----------------------------------------------------------------------- | -------------- | -| MF_USERS_LOG_LEVEL | Log level for Users (debug, info, warn, error) | error | +| MF_USERS_LOG_LEVEL | Log level for Users (debug, info, warn, error) | info | | MF_USERS_DB_HOST | Database host address | localhost | | MF_USERS_DB_PORT | Database host port | 5432 | | MF_USERS_DB_USER | Database user | mainflux | diff --git a/users/postgres/init.go b/users/postgres/init.go index c45a595be8..5bb917e421 100644 --- a/users/postgres/init.go +++ b/users/postgres/init.go @@ -3,48 +3,11 @@ package postgres -import ( - "fmt" +import migrate "github.com/rubenv/sql-migrate" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access - "github.com/jmoiron/sqlx" - - migrate "github.com/rubenv/sql-migrate" -) - -// Config defines the options that are used when connecting to a PostgreSQL instance -type Config struct { - Host string - Port string - User string - Pass string - Name string - SSLMode string - SSLCert string - SSLKey string - SSLRootCert string -} - -// Connect creates a connection to the PostgreSQL instance and applies any -// unapplied database migrations. A non-nil error is returned to indicate -// failure. -func Connect(cfg Config) (*sqlx.DB, error) { - url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert) - - db, err := sqlx.Open("pgx", url) - if err != nil { - return nil, err - } - - if err := migrateDB(db); err != nil { - return nil, err - } - return db, nil -} - -func migrateDB(db *sqlx.DB) error { - - migrations := &migrate.MemoryMigrationSource{ +// Migration of Users service +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ Migrations: []*migrate.Migration{ { Id: "users_1", @@ -93,7 +56,4 @@ func migrateDB(db *sqlx.DB) error { }, }, } - - _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up) - return err } diff --git a/users/postgres/setup_test.go b/users/postgres/setup_test.go index 7da75df4ef..1b366a8e74 100644 --- a/users/postgres/setup_test.go +++ b/users/postgres/setup_test.go @@ -12,9 +12,9 @@ import ( "os" "testing" - _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access "github.com/jmoiron/sqlx" + pgClient "github.com/mainflux/mainflux/internal/clients/postgres" "github.com/mainflux/mainflux/users/postgres" dockertest "github.com/ory/dockertest/v3" ) @@ -50,7 +50,7 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to docker: %s", err) } - dbConfig := postgres.Config{ + dbConfig := pgClient.Config{ Host: "localhost", Port: port, User: "test", @@ -62,7 +62,7 @@ func TestMain(m *testing.M) { SSLRootCert: "", } - if db, err = postgres.Connect(dbConfig); err != nil { + if db, err = pgClient.SetupDB(dbConfig, *postgres.Migration()); err != nil { log.Fatalf("Could not setup test DB connection: %s", err) } diff --git a/vendor/github.com/caarlos0/env/v7/.gitignore b/vendor/github.com/caarlos0/env/v7/.gitignore new file mode 100644 index 0000000000..ca6a0ff8cf --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/.gitignore @@ -0,0 +1,4 @@ +coverage.txt +bin +card.png +dist diff --git a/vendor/github.com/caarlos0/env/v7/.golangci.yml b/vendor/github.com/caarlos0/env/v7/.golangci.yml new file mode 100644 index 0000000000..ff791f86ea --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/.golangci.yml @@ -0,0 +1,8 @@ +linters: + enable: + - thelper + - gofumpt + - tparallel + - unconvert + - unparam + - wastedassign diff --git a/vendor/github.com/caarlos0/env/v7/.goreleaser.yml b/vendor/github.com/caarlos0/env/v7/.goreleaser.yml new file mode 100644 index 0000000000..4688983c27 --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/.goreleaser.yml @@ -0,0 +1,3 @@ +includes: + - from_url: + url: https://raw.githubusercontent.com/caarlos0/.goreleaserfiles/main/lib.yml diff --git a/vendor/github.com/caarlos0/env/v7/LICENSE.md b/vendor/github.com/caarlos0/env/v7/LICENSE.md new file mode 100644 index 0000000000..3a59b6b384 --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2022 Carlos Alexandro Becker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/caarlos0/env/v7/Makefile b/vendor/github.com/caarlos0/env/v7/Makefile new file mode 100644 index 0000000000..da8595fb9a --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/Makefile @@ -0,0 +1,37 @@ +SOURCE_FILES?=./... +TEST_PATTERN?=. + +export GO111MODULE := on + +setup: + go mod tidy +.PHONY: setup + +build: + go build +.PHONY: build + +test: + go test -v -failfast -race -coverpkg=./... -covermode=atomic -coverprofile=coverage.txt $(SOURCE_FILES) -run $(TEST_PATTERN) -timeout=2m +.PHONY: test + +cover: test + go tool cover -html=coverage.txt +.PHONY: cover + +fmt: + gofumpt -w -l . +.PHONY: fmt + +lint: + golangci-lint run ./... +.PHONY: lint + +ci: build test +.PHONY: ci + +card: + wget -O card.png -c "https://og.caarlos0.dev/**env**: parse envs to structs.png?theme=light&md=1&fontSize=100px&images=https://github.com/caarlos0.png" +.PHONY: card + +.DEFAULT_GOAL := ci diff --git a/vendor/github.com/caarlos0/env/v7/README.md b/vendor/github.com/caarlos0/env/v7/README.md new file mode 100644 index 0000000000..fd167e47bf --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/README.md @@ -0,0 +1,454 @@ +# env + +[![Build Status](https://img.shields.io/github/actions/workflow/status/caarlos0/env/build.yml?branch=main&style=for-the-badge)](https://github.com/caarlos0/env/actions?workflow=build) +[![Coverage Status](https://img.shields.io/codecov/c/gh/caarlos0/env.svg?logo=codecov&style=for-the-badge)](https://codecov.io/gh/caarlos0/env) +[![](http://img.shields.io/badge/godoc-reference-5272B4.svg?style=for-the-badge)](https://pkg.go.dev/github.com/caarlos0/env/v6) + +A simple and zero-dependencies library to parse environment variables into structs. + +## Example + +Get the module with: + +```sh +go get github.com/caarlos0/env/v6 +``` + +The usage looks like this: + +```go +package main + +import ( + "fmt" + "time" + + "github.com/caarlos0/env/v6" +) + +type config struct { + Home string `env:"HOME"` + Port int `env:"PORT" envDefault:"3000"` + Password string `env:"PASSWORD,unset"` + IsProduction bool `env:"PRODUCTION"` + Hosts []string `env:"HOSTS" envSeparator:":"` + Duration time.Duration `env:"DURATION"` + TempFolder string `env:"TEMP_FOLDER" envDefault:"${HOME}/tmp" envExpand:"true"` +} + +func main() { + cfg := config{} + if err := env.Parse(&cfg); err != nil { + fmt.Printf("%+v\n", err) + } + + fmt.Printf("%+v\n", cfg) +} +``` + +You can run it like this: + +```sh +$ PRODUCTION=true HOSTS="host1:host2:host3" DURATION=1s go run main.go +{Home:/your/home Port:3000 IsProduction:true Hosts:[host1 host2 host3] Duration:1s} +``` + +> **Warning** +> +> _Unexported fields_ are **ignored** by `env`. + +## Supported types and defaults + +Out of the box all built-in types are supported, plus a few others that +are commonly used. + +Complete list: + +- `string` +- `bool` +- `int` +- `int8` +- `int16` +- `int32` +- `int64` +- `uint` +- `uint8` +- `uint16` +- `uint32` +- `uint64` +- `float32` +- `float64` +- `time.Duration` +- `encoding.TextUnmarshaler` +- `url.URL` + +Pointers, slices and slices of pointers of those types are also supported. + +You can also use/define a [custom parser func](#custom-parser-funcs) for any +other type you want. + +If you set the `envDefault` tag for something, this value will be used in the +case of absence of it in the environment. + +By default, slice types will split the environment value on `,`; you can change +this behavior by setting the `envSeparator` tag. + +If you set the `envExpand` tag, environment variables (either in `${var}` or +`$var` format) in the string will be replaced according with the actual value +of the variable. + +## Custom Parser Funcs + +If you have a type that is not supported out of the box by the lib, you are able +to use (or define) and pass custom parsers (and their associated `reflect.Type`) +to the `env.ParseWithFuncs()` function. + +In addition to accepting a struct pointer (same as `Parse()`), this function +also accepts a `map[reflect.Type]env.ParserFunc`. + +If you add a custom parser for, say `Foo`, it will also be used to parse +`*Foo` and `[]Foo` types. + +Check the examples in the [go doc](http://pkg.go.dev/github.com/caarlos0/env/v6) +for more info. + +### A note about `TextUnmarshaler` and `time.Time` + +Env supports by default anything that implements the `TextUnmarshaler` interface. +That includes things like `time.Time` for example. +The upside is that depending on the format you need, you don't need to change anything. +The downside is that if you do need time in another format, you'll need to create your own type. + +Its fairly straightforward: + +```go +type MyTime time.Time + +func (t *MyTime) UnmarshalText(text []byte) error { + tt, err := time.Parse("2006-01-02", string(text)) + *t = MyTime(tt) + return err +} + +type Config struct { + SomeTime MyTime `env:"SOME_TIME"` +} +``` + +And then you can parse `Config` with `env.Parse`. + +## Required fields + +The `env` tag option `required` (e.g., `env:"tagKey,required"`) can be added to ensure that some environment variable is set. +In the example above, an error is returned if the `config` struct is changed to: + +```go +type config struct { + SecretKey string `env:"SECRET_KEY,required"` +} +``` + +## Not Empty fields + +While `required` demands the environment variable to be set, it doesn't check its value. +If you want to make sure the environment is set and not empty, you need to use the `notEmpty` tag option instead (`env:"SOME_ENV,notEmpty"`). + +Example: + +```go +type config struct { + SecretKey string `env:"SECRET_KEY,notEmpty"` +} +``` + +## Unset environment variable after reading it + +The `env` tag option `unset` (e.g., `env:"tagKey,unset"`) can be added +to ensure that some environment variable is unset after reading it. + +Example: + +```go +type config struct { + SecretKey string `env:"SECRET_KEY,unset"` +} +``` + +## From file + +The `env` tag option `file` (e.g., `env:"tagKey,file"`) can be added +to in order to indicate that the value of the variable shall be loaded from a file. The path of that file is given +by the environment variable associated with it +Example below + +```go +package main + +import ( + "fmt" + "time" + "github.com/caarlos0/env/v6" +) + +type config struct { + Secret string `env:"SECRET,file"` + Password string `env:"PASSWORD,file" envDefault:"/tmp/password"` + Certificate string `env:"CERTIFICATE,file" envDefault:"${CERTIFICATE_FILE}" envExpand:"true"` +} + +func main() { + cfg := config{} + if err := env.Parse(&cfg); err != nil { + fmt.Printf("%+v\n", err) + } + + fmt.Printf("%+v\n", cfg) +} +``` + +```sh +$ echo qwerty > /tmp/secret +$ echo dvorak > /tmp/password +$ echo coleman > /tmp/certificate + +$ SECRET=/tmp/secret \ + CERTIFICATE_FILE=/tmp/certificate \ + go run main.go +{Secret:qwerty Password:dvorak Certificate:coleman} +``` + +## Options + +### Environment + +By setting the `Options.Environment` map you can tell `Parse` to add those `keys` and `values` +as env vars before parsing is done. These envs are stored in the map and never actually set by `os.Setenv`. +This option effectively makes `env` ignore the OS environment variables: only the ones provided in the option are used. + +This can make your testing scenarios a bit more clean and easy to handle. + +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v6" +) + +type Config struct { + Password string `env:"PASSWORD"` +} + +func main() { + cfg := &Config{} + opts := &env.Options{Environment: map[string]string{ + "PASSWORD": "MY_PASSWORD", + }} + + // Load env vars. + if err := env.Parse(cfg, opts); err != nil { + log.Fatal(err) + } + + // Print the loaded data. + fmt.Printf("%+v\n", cfg.envData) +} +``` + +### Changing default tag name + +You can change what tag name to use for setting the env vars by setting the `Options.TagName` +variable. + +For example +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v6" +) + +type Config struct { + Password string `json:"PASSWORD"` +} + +func main() { + cfg := &Config{} + opts := &env.Options{TagName: "json"} + + // Load env vars. + if err := env.Parse(cfg, opts); err != nil { + log.Fatal(err) + } + + // Print the loaded data. + fmt.Printf("%+v\n", cfg.envData) +} +``` + +### Prefixes + +You can prefix sub-structs env tags, as well as a whole `env.Parse` call. + +Here's an example flexing it a bit: + +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v6" +) + +type Config struct { + Home string `env:"HOME"` +} + +type ComplexConfig struct { + Foo Config `envPrefix:"FOO_"` + Clean Config + Bar Config `envPrefix:"BAR_"` + Blah string `env:"BLAH"` +} + +func main() { + cfg := ComplexConfig{} + if err := Parse(&cfg, Options{ + Prefix: "T_", + Environment: map[string]string{ + "T_FOO_HOME": "/foo", + "T_BAR_HOME": "/bar", + "T_BLAH": "blahhh", + "T_HOME": "/clean", + }, + }); err != nil { + log.Fatal(err) + } + + // Load env vars. + if err := env.Parse(cfg, opts); err != nil { + log.Fatal(err) + } + + // Print the loaded data. + fmt.Printf("%+v\n", cfg.envData) +} +``` + +### On set hooks + +You might want to listen to value sets and, for example, log something or do some other kind of logic. +You can do this by passing a `OnSet` option: + +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v6" +) + +type Config struct { + Username string `env:"USERNAME" envDefault:"admin"` + Password string `env:"PASSWORD"` +} + +func main() { + cfg := &Config{} + opts := &env.Options{ + OnSet: func(tag string, value interface{}, isDefault bool) { + fmt.Printf("Set %s to %v (default? %v)\n", tag, value, isDefault) + }, + } + + // Load env vars. + if err := env.Parse(cfg, opts); err != nil { + log.Fatal(err) + } + + // Print the loaded data. + fmt.Printf("%+v\n", cfg.envData) +} +``` + +## Making all fields to required + +You can make all fields that don't have a default value be required by setting the `RequiredIfNoDef: true` in the `Options`. + +For example + +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v6" +) + +type Config struct { + Username string `env:"USERNAME" envDefault:"admin"` + Password string `env:"PASSWORD"` +} + +func main() { + cfg := &Config{} + opts := &env.Options{RequiredIfNoDef: true} + + // Load env vars. + if err := env.Parse(cfg, opts); err != nil { + log.Fatal(err) + } + + // Print the loaded data. + fmt.Printf("%+v\n", cfg.envData) +} +``` + +## Defaults from code + +You may define default value also in code, by initialising the config data before it's filled by `env.Parse`. +Default values defined as struct tags will overwrite existing values during Parse. + +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v6" +) + +type Config struct { + Username string `env:"USERNAME" envDefault:"admin"` + Password string `env:"PASSWORD"` +} + +func main() { + var cfg = Config{ + Username: "test", + Password: "123456", + } + + if err := env.Parse(&cfg); err != nil { + fmt.Println("failed:", err) + } + + fmt.Printf("%+v", cfg) // {Username:admin Password:123456} +} +``` + +## Stargazers over time + +[![Stargazers over time](https://starchart.cc/caarlos0/env.svg)](https://starchart.cc/caarlos0/env) diff --git a/vendor/github.com/caarlos0/env/v7/env.go b/vendor/github.com/caarlos0/env/v7/env.go new file mode 100644 index 0000000000..a97befb9db --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/env.go @@ -0,0 +1,506 @@ +package env + +import ( + "encoding" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "strconv" + "strings" + "time" +) + +// nolint: gochecknoglobals +var ( + // ErrNotAStructPtr is returned if you pass something that is not a pointer to a + // Struct to Parse. + ErrNotAStructPtr = errors.New("env: expected a pointer to a Struct") + + defaultBuiltInParsers = map[reflect.Kind]ParserFunc{ + reflect.Bool: func(v string) (interface{}, error) { + return strconv.ParseBool(v) + }, + reflect.String: func(v string) (interface{}, error) { + return v, nil + }, + reflect.Int: func(v string) (interface{}, error) { + i, err := strconv.ParseInt(v, 10, 32) + return int(i), err + }, + reflect.Int16: func(v string) (interface{}, error) { + i, err := strconv.ParseInt(v, 10, 16) + return int16(i), err + }, + reflect.Int32: func(v string) (interface{}, error) { + i, err := strconv.ParseInt(v, 10, 32) + return int32(i), err + }, + reflect.Int64: func(v string) (interface{}, error) { + return strconv.ParseInt(v, 10, 64) + }, + reflect.Int8: func(v string) (interface{}, error) { + i, err := strconv.ParseInt(v, 10, 8) + return int8(i), err + }, + reflect.Uint: func(v string) (interface{}, error) { + i, err := strconv.ParseUint(v, 10, 32) + return uint(i), err + }, + reflect.Uint16: func(v string) (interface{}, error) { + i, err := strconv.ParseUint(v, 10, 16) + return uint16(i), err + }, + reflect.Uint32: func(v string) (interface{}, error) { + i, err := strconv.ParseUint(v, 10, 32) + return uint32(i), err + }, + reflect.Uint64: func(v string) (interface{}, error) { + i, err := strconv.ParseUint(v, 10, 64) + return i, err + }, + reflect.Uint8: func(v string) (interface{}, error) { + i, err := strconv.ParseUint(v, 10, 8) + return uint8(i), err + }, + reflect.Float64: func(v string) (interface{}, error) { + return strconv.ParseFloat(v, 64) + }, + reflect.Float32: func(v string) (interface{}, error) { + f, err := strconv.ParseFloat(v, 32) + return float32(f), err + }, + } +) + +func defaultTypeParsers() map[reflect.Type]ParserFunc { + return map[reflect.Type]ParserFunc{ + reflect.TypeOf(url.URL{}): func(v string) (interface{}, error) { + u, err := url.Parse(v) + if err != nil { + return nil, fmt.Errorf("unable to parse URL: %v", err) + } + return *u, nil + }, + reflect.TypeOf(time.Nanosecond): func(v string) (interface{}, error) { + s, err := time.ParseDuration(v) + if err != nil { + return nil, fmt.Errorf("unable to parse duration: %v", err) + } + return s, err + }, + } +} + +// ParserFunc defines the signature of a function that can be used within `CustomParsers`. +type ParserFunc func(v string) (interface{}, error) + +// OnSetFn is a hook that can be run when a value is set. +type OnSetFn func(tag string, value interface{}, isDefault bool) + +// Options for the parser. +type Options struct { + // Environment keys and values that will be accessible for the service. + Environment map[string]string + + // TagName specifies another tagname to use rather than the default env. + TagName string + + // RequiredIfNoDef automatically sets all env as required if they do not declare 'envDefault' + RequiredIfNoDef bool + + // OnSet allows to run a function when a value is set + OnSet OnSetFn + + // Prefix define a prefix for each key + Prefix string + + // Sets to true if we have already configured once. + configured bool +} + +// configure will do the basic configurations and defaults. +func configure(opts []Options) []Options { + // If we have already configured the first item + // of options will have been configured set to true. + if len(opts) > 0 && opts[0].configured { + return opts + } + + // Created options with defaults. + opt := Options{ + TagName: "env", + Environment: toMap(os.Environ()), + configured: true, + } + + // Loop over all opts structs and set + // to opt if value is not default/empty. + for _, item := range opts { + if item.Environment != nil { + opt.Environment = item.Environment + } + if item.TagName != "" { + opt.TagName = item.TagName + } + if item.OnSet != nil { + opt.OnSet = item.OnSet + } + if item.Prefix != "" { + opt.Prefix = item.Prefix + } + opt.RequiredIfNoDef = item.RequiredIfNoDef + } + + return []Options{opt} +} + +func getOnSetFn(opts []Options) OnSetFn { + return opts[0].OnSet +} + +// getTagName returns the tag name. +func getTagName(opts []Options) string { + return opts[0].TagName +} + +// getEnvironment returns the environment map. +func getEnvironment(opts []Options) map[string]string { + return opts[0].Environment +} + +// Parse parses a struct containing `env` tags and loads its values from +// environment variables. +func Parse(v interface{}, opts ...Options) error { + return ParseWithFuncs(v, map[reflect.Type]ParserFunc{}, opts...) +} + +// ParseWithFuncs is the same as `Parse` except it also allows the user to pass +// in custom parsers. +func ParseWithFuncs(v interface{}, funcMap map[reflect.Type]ParserFunc, opts ...Options) error { + opts = configure(opts) + + ptrRef := reflect.ValueOf(v) + if ptrRef.Kind() != reflect.Ptr { + return ErrNotAStructPtr + } + ref := ptrRef.Elem() + if ref.Kind() != reflect.Struct { + return ErrNotAStructPtr + } + parsers := defaultTypeParsers() + for k, v := range funcMap { + parsers[k] = v + } + + return doParse(ref, parsers, opts) +} + +func doParse(ref reflect.Value, funcMap map[reflect.Type]ParserFunc, opts []Options) error { + refType := ref.Type() + + var agrErr aggregateError + + for i := 0; i < refType.NumField(); i++ { + refField := ref.Field(i) + refTypeField := refType.Field(i) + + if err := doParseField(refField, refTypeField, funcMap, opts); err != nil { + if val, ok := err.(aggregateError); ok { + agrErr.errors = append(agrErr.errors, val.errors...) + } else { + agrErr.errors = append(agrErr.errors, err) + } + } + } + + if len(agrErr.errors) == 0 { + return nil + } + + return agrErr +} + +func doParseField(refField reflect.Value, refTypeField reflect.StructField, funcMap map[reflect.Type]ParserFunc, opts []Options) error { + if !refField.CanSet() { + return nil + } + if reflect.Ptr == refField.Kind() && refField.Elem().Kind() == reflect.Struct { + return ParseWithFuncs(refField.Interface(), funcMap, optsWithPrefix(refTypeField, opts)...) + } + if reflect.Struct == refField.Kind() && refField.CanAddr() && refField.Type().Name() == "" { + return ParseWithFuncs(refField.Addr().Interface(), funcMap, optsWithPrefix(refTypeField, opts)...) + } + value, err := get(refTypeField, opts) + if err != nil { + return err + } + + if value != "" { + return set(refField, refTypeField, value, funcMap) + } + + if reflect.Struct == refField.Kind() { + return doParse(refField, funcMap, optsWithPrefix(refTypeField, opts)) + } + + return nil +} + +func get(field reflect.StructField, opts []Options) (val string, err error) { + var exists bool + var isDefault bool + var loadFile bool + var unset bool + var notEmpty bool + + required := opts[0].RequiredIfNoDef + prefix := opts[0].Prefix + ownKey, tags := parseKeyForOption(field.Tag.Get(getTagName(opts))) + key := prefix + ownKey + for _, tag := range tags { + switch tag { + case "": + continue + case "file": + loadFile = true + case "required": + required = true + case "unset": + unset = true + case "notEmpty": + notEmpty = true + default: + return "", fmt.Errorf("tag option %q not supported", tag) + } + } + expand := strings.EqualFold(field.Tag.Get("envExpand"), "true") + defaultValue, defExists := field.Tag.Lookup("envDefault") + val, exists, isDefault = getOr(key, defaultValue, defExists, getEnvironment(opts)) + + if expand { + val = os.ExpandEnv(val) + } + + if unset { + defer os.Unsetenv(key) + } + + if required && !exists && len(ownKey) > 0 { + return "", fmt.Errorf(`required environment variable %q is not set`, key) + } + + if notEmpty && val == "" { + return "", fmt.Errorf("environment variable %q should not be empty", key) + } + + if loadFile && val != "" { + filename := val + val, err = getFromFile(filename) + if err != nil { + return "", fmt.Errorf(`could not load content of file "%s" from variable %s: %v`, filename, key, err) + } + } + + if onSetFn := getOnSetFn(opts); onSetFn != nil { + onSetFn(key, val, isDefault) + } + return val, err +} + +// split the env tag's key into the expected key and desired option, if any. +func parseKeyForOption(key string) (string, []string) { + opts := strings.Split(key, ",") + return opts[0], opts[1:] +} + +func getFromFile(filename string) (value string, err error) { + b, err := os.ReadFile(filename) + return string(b), err +} + +func getOr(key, defaultValue string, defExists bool, envs map[string]string) (string, bool, bool) { + value, exists := envs[key] + switch { + case (!exists || key == "") && defExists: + return defaultValue, true, true + case exists && value == "" && defExists: + return defaultValue, true, true + case !exists: + return "", false, false + } + + return value, true, false +} + +func set(field reflect.Value, sf reflect.StructField, value string, funcMap map[reflect.Type]ParserFunc) error { + if tm := asTextUnmarshaler(field); tm != nil { + if err := tm.UnmarshalText([]byte(value)); err != nil { + return newParseError(sf, err) + } + return nil + } + + typee := sf.Type + fieldee := field + if typee.Kind() == reflect.Ptr { + typee = typee.Elem() + fieldee = field.Elem() + } + + parserFunc, ok := funcMap[typee] + if ok { + val, err := parserFunc(value) + if err != nil { + return newParseError(sf, err) + } + + fieldee.Set(reflect.ValueOf(val)) + return nil + } + + parserFunc, ok = defaultBuiltInParsers[typee.Kind()] + if ok { + val, err := parserFunc(value) + if err != nil { + return newParseError(sf, err) + } + + fieldee.Set(reflect.ValueOf(val).Convert(typee)) + return nil + } + + if field.Kind() == reflect.Slice { + return handleSlice(field, value, sf, funcMap) + } + + return newNoParserError(sf) +} + +func handleSlice(field reflect.Value, value string, sf reflect.StructField, funcMap map[reflect.Type]ParserFunc) error { + separator := sf.Tag.Get("envSeparator") + if separator == "" { + separator = "," + } + parts := strings.Split(value, separator) + + typee := sf.Type.Elem() + if typee.Kind() == reflect.Ptr { + typee = typee.Elem() + } + + if _, ok := reflect.New(typee).Interface().(encoding.TextUnmarshaler); ok { + return parseTextUnmarshalers(field, parts, sf) + } + + parserFunc, ok := funcMap[typee] + if !ok { + parserFunc, ok = defaultBuiltInParsers[typee.Kind()] + if !ok { + return newNoParserError(sf) + } + } + + result := reflect.MakeSlice(sf.Type, 0, len(parts)) + for _, part := range parts { + r, err := parserFunc(part) + if err != nil { + return newParseError(sf, err) + } + v := reflect.ValueOf(r).Convert(typee) + if sf.Type.Elem().Kind() == reflect.Ptr { + v = reflect.New(typee) + v.Elem().Set(reflect.ValueOf(r).Convert(typee)) + } + result = reflect.Append(result, v) + } + field.Set(result) + return nil +} + +func asTextUnmarshaler(field reflect.Value) encoding.TextUnmarshaler { + if reflect.Ptr == field.Kind() { + if field.IsNil() { + field.Set(reflect.New(field.Type().Elem())) + } + } else if field.CanAddr() { + field = field.Addr() + } + + tm, ok := field.Interface().(encoding.TextUnmarshaler) + if !ok { + return nil + } + return tm +} + +func parseTextUnmarshalers(field reflect.Value, data []string, sf reflect.StructField) error { + s := len(data) + elemType := field.Type().Elem() + slice := reflect.MakeSlice(reflect.SliceOf(elemType), s, s) + for i, v := range data { + sv := slice.Index(i) + kind := sv.Kind() + if kind == reflect.Ptr { + sv = reflect.New(elemType.Elem()) + } else { + sv = sv.Addr() + } + tm := sv.Interface().(encoding.TextUnmarshaler) + if err := tm.UnmarshalText([]byte(v)); err != nil { + return newParseError(sf, err) + } + if kind == reflect.Ptr { + slice.Index(i).Set(sv) + } + } + + field.Set(slice) + + return nil +} + +func newParseError(sf reflect.StructField, err error) error { + return parseError{ + sf: sf, + err: err, + } +} + +type parseError struct { + sf reflect.StructField + err error +} + +func (e parseError) Error() string { + return fmt.Sprintf(`parse error on field "%s" of type "%s": %v`, e.sf.Name, e.sf.Type, e.err) +} + +func newNoParserError(sf reflect.StructField) error { + return fmt.Errorf(`no parser found for field "%s" of type "%s"`, sf.Name, sf.Type) +} + +func optsWithPrefix(field reflect.StructField, opts []Options) []Options { + subOpts := make([]Options, len(opts)) + copy(subOpts, opts) + if prefix := field.Tag.Get("envPrefix"); prefix != "" { + subOpts[0].Prefix += prefix + } + return subOpts +} + +type aggregateError struct { + errors []error +} + +func (e aggregateError) Error() string { + var sb strings.Builder + sb.WriteString("env:") + + for _, err := range e.errors { + sb.WriteString(fmt.Sprintf(" %v;", err.Error())) + } + + return strings.TrimRight(sb.String(), ";") +} diff --git a/vendor/github.com/caarlos0/env/v7/env_unix.go b/vendor/github.com/caarlos0/env/v7/env_unix.go new file mode 100644 index 0000000000..411d4385a0 --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/env_unix.go @@ -0,0 +1,15 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package env + +import "strings" + +func toMap(env []string) map[string]string { + r := map[string]string{} + for _, e := range env { + p := strings.SplitN(e, "=", 2) + r[p[0]] = p[1] + } + return r +} diff --git a/vendor/github.com/caarlos0/env/v7/env_windows.go b/vendor/github.com/caarlos0/env/v7/env_windows.go new file mode 100644 index 0000000000..e12123cd38 --- /dev/null +++ b/vendor/github.com/caarlos0/env/v7/env_windows.go @@ -0,0 +1,25 @@ +package env + +import "strings" + +func toMap(env []string) map[string]string { + r := map[string]string{} + for _, e := range env { + p := strings.SplitN(e, "=", 2) + + // On Windows, environment variables can start with '='. If so, Split at next character. + // See env_windows.go in the Go source: https://github.com/golang/go/blob/master/src/syscall/env_windows.go#L58 + prefixEqualSign := false + if len(e) > 0 && e[0] == '=' { + e = e[1:] + prefixEqualSign = true + } + p = strings.SplitN(e, "=", 2) + if prefixEqualSign { + p[0] = "=" + p[0] + } + + r[p[0]] = p[1] + } + return r +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 275dca7c8e..670d4e5c7e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,6 +19,9 @@ github.com/armon/go-radix # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile +# github.com/caarlos0/env/v7 v7.0.0 +## explicit; go 1.17 +github.com/caarlos0/env/v7 # github.com/cenkalti/backoff/v3 v3.2.2 ## explicit; go 1.12 github.com/cenkalti/backoff/v3 diff --git a/ws/README.md b/ws/README.md index 753a4a88d4..42281f2da9 100644 --- a/ws/README.md +++ b/ws/README.md @@ -12,7 +12,7 @@ default values. |------------------------------|-----------------------------------------------------|-----------------------| | MF_WS_ADAPTER_PORT | Service WS port | 8190 | | MF_BROKER_URL | Message broker instance URL | nats://localhost:4222 | -| MF_WS_ADAPTER_LOG_LEVEL | Log level for the WS Adapter | error | +| MF_WS_ADAPTER_LOG_LEVEL | Log level for the WS Adapter | info | | MF_WS_ADAPTER_CLIENT_TLS | Flag that indicates if TLS should be turned on | false | | MF_WS_ADAPTER_CA_CERTS | Path to trusted CAs in PEM format | | | MF_JAEGER_URL | Jaeger server URL | localhost:6831 |