From 4ae0f1a97428d693dac6b7c2ada3459046f87377 Mon Sep 17 00:00:00 2001 From: Andrey Stolyarov Date: Thu, 12 Dec 2024 23:28:29 +0300 Subject: [PATCH] Feeds (#21) * Add yaml config for feeds --- .../d10xa/jsonlogviewer/shell/ShellImpl.scala | 4 +- .../decline/ConfigInitImpl.scala | 1 + .../d10xa/jsonlogviewer/shell/ShellImpl.scala | 30 ++- .../ru/d10xa/jsonlogviewer/Application.scala | 30 +-- .../jsonlogviewer/ConfigYamlReader.scala | 4 +- .../d10xa/jsonlogviewer/LogViewerStream.scala | 104 +++++++++-- .../d10xa/jsonlogviewer/decline/Config.scala | 1 + .../jsonlogviewer/decline/ConfigYaml.scala | 12 -- .../decline/ConfigYamlLoader.scala | 82 -------- .../decline/yaml/ConfigYaml.scala | 16 ++ .../decline/yaml/ConfigYamlLoader.scala | 175 ++++++++++++++++++ .../jsonlogviewer/decline/yaml/Feed.scala | 11 ++ .../formatout/ColorLineFormatter.scala | 9 +- .../ru/d10xa/jsonlogviewer/shell/Shell.scala | 4 +- .../decline/yaml/ConfigYamlLoaderTest.scala | 85 +++++++++ 15 files changed, 415 insertions(+), 153 deletions(-) delete mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYaml.scala delete mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYamlLoader.scala create mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYaml.scala create mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoader.scala create mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala create mode 100644 json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderTest.scala diff --git a/json-log-viewer/js/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala b/json-log-viewer/js/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala index 106b21b..553ec5d 100644 --- a/json-log-viewer/js/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala +++ b/json-log-viewer/js/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala @@ -5,8 +5,8 @@ import fs2.* import java.io.* -class ShellImpl extends Shell { +class ShellImpl[F[_]] extends Shell[F] { - def mergeCommands(commands: List[String]): Stream[IO, String] = Stream.empty + def mergeCommands(commands: List[String]): Stream[F, String] = Stream.empty } diff --git a/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigInitImpl.scala b/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigInitImpl.scala index 6c4877b..c240d8e 100644 --- a/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigInitImpl.scala +++ b/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigInitImpl.scala @@ -5,6 +5,7 @@ import cats.effect.IO import ru.d10xa.jsonlogviewer.ConfigYamlReader import ru.d10xa.jsonlogviewer.decline.Config.FormatIn import cats.syntax.all.* +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml import java.io.File diff --git a/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala b/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala index 4571de3..180571d 100644 --- a/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala +++ b/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/shell/ShellImpl.scala @@ -1,37 +1,35 @@ package ru.d10xa.jsonlogviewer.shell import cats.effect.* -import fs2.* +import cats.syntax.all.* -import java.io.* +class ShellImpl[F[_]: Async] extends Shell[F] { -class ShellImpl extends Shell { - - def createProcess(command: String): Resource[IO, Process] = - Resource.make(IO { + def createProcess(command: String): Resource[F, Process] = + Resource.make(Async[F].delay { new ProcessBuilder("sh", "-c", command) .redirectErrorStream(true) .start() - })(process => IO(process.destroy()).void) + })(process => Async[F].delay(process.destroy())) - def runInfiniteCommand(command: String): Stream[IO, String] = - Stream.resource(createProcess(command)).flatMap { process => + def runInfiniteCommand(command: String): fs2.Stream[F, String] = + fs2.Stream.resource(createProcess(command)).flatMap { process => fs2.io .readInputStream( - IO(process.getInputStream), + Async[F].delay(process.getInputStream), 4096, closeAfterUse = false ) - .through(text.utf8.decode) - .through(text.lines) - .onFinalize(IO { + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + .onFinalize(Async[F].delay { process.waitFor() }.void) } - def mergeCommands(commands: List[String]): Stream[IO, String] = { + def mergeCommands(commands: List[String]): fs2.Stream[F, String] = { val streams = commands.map(runInfiniteCommand) - Stream.emits(streams).parJoin(math.max(1, commands.length)) + fs2.Stream.emits(streams).parJoin(math.max(1, commands.length)) } -} +} \ No newline at end of file diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/Application.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/Application.scala index aedc3da..a104752 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/Application.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/Application.scala @@ -1,6 +1,5 @@ package ru.d10xa.jsonlogviewer -import cats.data.Validated import cats.effect.* import com.monovore.decline.Opts import com.monovore.decline.effect.CommandIOApp @@ -8,13 +7,11 @@ import fs2.* import fs2.io.* import ru.d10xa.jsonlogviewer.decline.Config import ru.d10xa.jsonlogviewer.decline.Config.FormatIn -import ru.d10xa.jsonlogviewer.decline.DeclineOpts -import ru.d10xa.jsonlogviewer.logfmt.LogfmtLogLineParser -import _root_.io.circe.yaml.scalayaml.parser -import cats.syntax.all.* import ru.d10xa.jsonlogviewer.decline.ConfigInit import ru.d10xa.jsonlogviewer.decline.ConfigInitImpl -import ru.d10xa.jsonlogviewer.decline.ConfigYaml +import ru.d10xa.jsonlogviewer.decline.DeclineOpts +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml +import ru.d10xa.jsonlogviewer.logfmt.LogfmtLogLineParser import ru.d10xa.jsonlogviewer.shell.ShellImpl object Application @@ -23,30 +20,13 @@ object Application "Print json logs in human-readable form" ): - private val stdinLinesStream: Stream[IO, String] = - stdinUtf8[IO](1024 * 1024 * 10) - .repartition(s => Chunk.array(s.split("\n", -1))) - .filter(_.nonEmpty) - private val configInit: ConfigInit = new ConfigInitImpl def main: Opts[IO[ExitCode]] = DeclineOpts.config.map { c => configInit.initConfig(c).flatMap { updatedConfig => IO { - val jsonPrefixPostfix = JsonPrefixPostfix(JsonDetector()) - val logLineParser = updatedConfig.formatIn match { - case Some(FormatIn.Logfmt) => LogfmtLogLineParser(updatedConfig) - case _ => JsonLogLineParser(updatedConfig, jsonPrefixPostfix) - } - val commandsOpt = updatedConfig.configYaml.flatMap(_.commands).filter(_.nonEmpty) - val stream = commandsOpt match { - case Some(cmds) if cmds.nonEmpty => - new ShellImpl().mergeCommands(cmds) - case _ => - stdinLinesStream - } - stream - .through(LogViewerStream.stream[IO](updatedConfig, logLineParser)) + LogViewerStream + .stream[IO](updatedConfig) .through(text.utf8.encode) .through(io.stdout) .compile diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/ConfigYamlReader.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/ConfigYamlReader.scala index 6874b4e..9d91e90 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/ConfigYamlReader.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/ConfigYamlReader.scala @@ -2,8 +2,8 @@ package ru.d10xa.jsonlogviewer import cats.effect.IO import cats.data.ValidatedNel -import ru.d10xa.jsonlogviewer.decline.ConfigYaml -import ru.d10xa.jsonlogviewer.decline.ConfigYamlLoader +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYamlLoader import scala.io.Source diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala index 9c2ef63..d75e472 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala @@ -1,33 +1,115 @@ package ru.d10xa.jsonlogviewer +import cats.effect.Async +import cats.syntax.all.* import fs2.* import fs2.io.* import ru.d10xa.jsonlogviewer.decline.Config +import ru.d10xa.jsonlogviewer.decline.Config.FormatIn +import ru.d10xa.jsonlogviewer.decline.ConfigInit +import ru.d10xa.jsonlogviewer.decline.ConfigInitImpl +import ru.d10xa.jsonlogviewer.decline.DeclineOpts +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml +import ru.d10xa.jsonlogviewer.decline.yaml.Feed import ru.d10xa.jsonlogviewer.formatout.ColorLineFormatter import ru.d10xa.jsonlogviewer.formatout.RawFormatter +import ru.d10xa.jsonlogviewer.logfmt.LogfmtLogLineParser +import ru.d10xa.jsonlogviewer.query.QueryAST +import ru.d10xa.jsonlogviewer.shell.ShellImpl + object LogViewerStream { - def stream[F[_]]( + private def makeLogLineParser( config: Config, - logLineParser: LogLineParser - ): Pipe[F, String, String] = stream => + optFormatIn: Option[FormatIn] + ): LogLineParser = { + val jsonPrefixPostfix = JsonPrefixPostfix(JsonDetector()) + optFormatIn match { + case Some(FormatIn.Logfmt) => LogfmtLogLineParser(config) + case _ => JsonLogLineParser(config, jsonPrefixPostfix) + } + } + + private def commandsToStream[F[_]: Async]( + commands: List[String] + ): Stream[F, String] = { + new ShellImpl[F]().mergeCommands(commands) + } + + private def stdinLinesStream[F[_]: Async]: Stream[F, String] = + stdinUtf8[F](1024 * 1024 * 10) + .repartition(s => Chunk.array(s.split("\n", -1))) + .filter(_.nonEmpty) + + private def processStream[F[_]: Async]( + baseConfig: Config, + lines: Stream[F, String], + feedFilter: Option[QueryAST], + feedFormatIn: Option[FormatIn], + feedName: Option[String] + ): Stream[F, String] = { + val effectiveFormatIn = feedFormatIn.orElse(baseConfig.formatIn) + val effectiveFilter = feedFilter.orElse(baseConfig.filter) + val effectiveConfig = baseConfig.copy( + filter = effectiveFilter, + formatIn = effectiveFormatIn + ) + val timestampFilter = TimestampFilter() - val outputLineFormatter = config.formatOut match + val parseResultKeys = ParseResultKeys(effectiveConfig) + val logLineFilter = LogLineFilter(effectiveConfig, parseResultKeys) + val logLineParser = makeLogLineParser(effectiveConfig, effectiveFormatIn) + val outputLineFormatter = effectiveConfig.formatOut match case Some(Config.FormatOut.Raw) => RawFormatter() - case Some(Config.FormatOut.Pretty) | None => ColorLineFormatter(config) - - val parseResultKeys = ParseResultKeys(config) - val logLineFilter = LogLineFilter(config, parseResultKeys) - stream + case Some(Config.FormatOut.Pretty) | None => + ColorLineFormatter(effectiveConfig, feedName) + + lines .map(logLineParser.parse) .filter(logLineFilter.grep) .filter(logLineFilter.logLineQueryPredicate) - .through(timestampFilter.filterTimestampAfter[F](config.timestamp.after)) .through( - timestampFilter.filterTimestampBefore[F](config.timestamp.before) + timestampFilter.filterTimestampAfter[F](effectiveConfig.timestamp.after) + ) + .through( + timestampFilter.filterTimestampBefore[F]( + effectiveConfig.timestamp.before + ) ) .map(outputLineFormatter.formatLine) .map(_.toString) + } + + def stream[F[_]: Async](config: Config): Stream[F, String] = { + val topCommandsOpt: Option[List[String]] = + config.configYaml.flatMap(_.commands).filter(_.nonEmpty) + val feedsOpt: Option[List[Feed]] = + config.configYaml.flatMap(_.feeds).filter(_.nonEmpty) + + val finalStream = feedsOpt match { + case Some(feeds) => + val feedStreams = feeds.map { feed => + val feedStream = commandsToStream[F](feed.commands) + processStream( + config, + feedStream, + feed.filter, + feed.formatIn, + feed.name.some + ) + } + Stream.emits(feedStreams).parJoin(feedStreams.size) + + case None => + val baseStream = topCommandsOpt match { + case Some(cmds) => commandsToStream[F](cmds) + case None => stdinLinesStream[F] + } + processStream(config, baseStream, None, None, None) + } + + finalStream .intersperse("\n") .append(Stream.emit("\n")) + } } diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/Config.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/Config.scala index 371a43b..3d10b44 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/Config.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/Config.scala @@ -4,6 +4,7 @@ import ru.d10xa.jsonlogviewer.decline.Config import ru.d10xa.jsonlogviewer.decline.Config.ConfigGrep import ru.d10xa.jsonlogviewer.decline.ConfigFile import ru.d10xa.jsonlogviewer.decline.TimestampConfig +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml import ru.d10xa.jsonlogviewer.query.QueryAST import scala.util.matching.Regex diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYaml.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYaml.scala deleted file mode 100644 index 77394f5..0000000 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYaml.scala +++ /dev/null @@ -1,12 +0,0 @@ -package ru.d10xa.jsonlogviewer.decline - -import ru.d10xa.jsonlogviewer.query.QueryAST - -case class ConfigYaml( - filter: Option[QueryAST], - formatIn: Option[Config.FormatIn], - commands: Option[List[String]] -) - -object ConfigYaml: - val empty: ConfigYaml = ConfigYaml(None, None, None) diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYamlLoader.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYamlLoader.scala deleted file mode 100644 index 52d9300..0000000 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/ConfigYamlLoader.scala +++ /dev/null @@ -1,82 +0,0 @@ -package ru.d10xa.jsonlogviewer.decline - -import cats.data.Validated -import cats.data.ValidatedNel -import cats.syntax.all.* -import io.circe.* -import io.circe.generic.auto.* -import io.circe.yaml.scalayaml.parser -import ru.d10xa.jsonlogviewer.decline.Config.FormatIn -import ru.d10xa.jsonlogviewer.query.QueryAST - -object ConfigYamlLoader { - def parseYamlFile(content: String): ValidatedNel[String, ConfigYaml] = { - val uncommentedContent = content.linesIterator - .filterNot(line => line.trim.startsWith("#")) - .mkString("\n") - .trim - if (uncommentedContent.isEmpty) { - Validated.valid(ConfigYaml.empty) - } else { - parser.parse(content) match { - case Left(error) => - Validated.invalidNel(s"YAML parsing error: ${error.getMessage}") - case Right(json) => - json.asObject.map(_.toMap) match { - case None => Validated.invalidNel("YAML is not a valid JSON object") - case Some(fields) => - val filterValidated: ValidatedNel[String, Option[QueryAST]] = - fields.get("filter") match { - case Some(jsonValue) => - jsonValue.as[String] match { - case Left(_) => - Validated.invalidNel("Invalid 'filter' field format") - case Right(filterStr) => - val trimmedStr = filterStr.linesIterator - .filterNot(line => - line.trim.startsWith("#") || line.trim.startsWith( - "//" - ) - ) - .mkString("\n") - .replace("\\n", " ") - .trim - QueryASTValidator - .toValidatedQueryAST(trimmedStr) - .map(Some(_)) - } - case None => Validated.valid(None) - } - - val formatInValidated: ValidatedNel[String, Option[FormatIn]] = - fields.get("formatIn") match { - case Some(jsonValue) => - jsonValue.as[String] match { - case Left(_) => - Validated.invalidNel("Invalid 'formatIn' field format") - case Right(formatStr) => - FormatInValidator - .toValidatedFormatIn(formatStr) - .map(Some(_)) - } - case None => Validated.valid(None) - } - val commandsValidated: ValidatedNel[String, Option[List[String]]] = - fields.get("commands") match { - case Some(jsonValue) => - jsonValue.as[List[String]] match { - case Left(_) => - Validated.invalidNel("Invalid 'commands' field format") - case Right(cmds) => - Validated.valid(Some(cmds)) - } - case None => Validated.valid(None) - } - - (filterValidated, formatInValidated, commandsValidated) - .mapN(ConfigYaml.apply) - } - } - } - } -} diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYaml.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYaml.scala new file mode 100644 index 0000000..9e717cf --- /dev/null +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYaml.scala @@ -0,0 +1,16 @@ +package ru.d10xa.jsonlogviewer.decline.yaml + +import ru.d10xa.jsonlogviewer.decline.Config +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml +import ru.d10xa.jsonlogviewer.query.QueryAST + +case class ConfigYaml( + filter: Option[QueryAST], + formatIn: Option[Config.FormatIn], + commands: Option[List[String]], + feeds: Option[List[Feed]] +) + +object ConfigYaml: + val empty: ConfigYaml = ConfigYaml(None, None, None, None) + diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoader.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoader.scala new file mode 100644 index 0000000..086817b --- /dev/null +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoader.scala @@ -0,0 +1,175 @@ +package ru.d10xa.jsonlogviewer.decline.yaml + +import cats.data.NonEmptyList +import cats.data.Validated +import cats.data.ValidatedNel +import cats.syntax.all.* +import io.circe.* +import io.circe.generic.auto.* +import io.circe.yaml.scalayaml.parser +import ru.d10xa.jsonlogviewer.decline.Config.FormatIn +import ru.d10xa.jsonlogviewer.decline.FormatInValidator +import ru.d10xa.jsonlogviewer.decline.QueryASTValidator +import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml +import ru.d10xa.jsonlogviewer.query.QueryAST + +object ConfigYamlLoader { + private def trimCommentedLines(str: String): String = + str.linesIterator + .filterNot(line => + line.trim.startsWith("#") || line.trim.startsWith("//") + ) + .mkString("\n") + .replace("\\n", " ") + .trim + + private def parseOptionalQueryAST( + fields: Map[String, Json], + fieldName: String + ): ValidatedNel[String, Option[QueryAST]] = + parseOptionalStringField( + fields, + fieldName, + s"Invalid '$fieldName' field format" + ).andThen { + case Some(str) => + val trimmed = trimCommentedLines(str) + QueryASTValidator.toValidatedQueryAST(trimmed).map(Some(_)) + case None => Validated.valid(None) + } + + private def parseOptionalFormatIn( + fields: Map[String, Json], + fieldName: String + ): ValidatedNel[String, Option[FormatIn]] = + parseOptionalStringField( + fields, + fieldName, + s"Invalid '$fieldName' field format" + ).andThen { + case Some(formatStr) => + FormatInValidator.toValidatedFormatIn(formatStr).map(Some(_)) + case None => Validated.valid(None) + } + + private def parseOptionalListString( + fields: Map[String, Json], + fieldName: String + ): ValidatedNel[String, Option[List[String]]] = + fields.get(fieldName) match { + case Some(jsonValue) => + jsonValue + .as[List[String]] + .leftMap(_ => s"Invalid '$fieldName' field format") + .toValidatedNel + .map(Some(_)) + case None => Validated.valid(None) + } + + private def parseOptionalFeeds( + fields: Map[String, Json], + fieldName: String + ): ValidatedNel[String, Option[List[Feed]]] = + fields.get(fieldName) match { + case Some(jsonValue) => + jsonValue + .as[List[Json]] + .leftMap(_ => s"Invalid '$fieldName' field format, should be a list") + .toValidatedNel + .andThen(_.traverse(parseFeed)) + .map(Some(_)) + case None => Validated.valid(None) + } + + private def parseOptionalStringField( + fields: Map[String, Json], + fieldName: String, + errorMsg: String + ): ValidatedNel[String, Option[String]] = + fields.get(fieldName) match { + case Some(jsonValue) => + jsonValue.as[String].leftMap(_ => errorMsg).toValidatedNel.map(Some(_)) + case None => Validated.valid(None) + } + + private def parseString( + fields: Map[String, Json], + fieldName: String, + errorMsg: String + ): ValidatedNel[String, String] = + fields.get(fieldName) match { + case Some(j) => + j.as[String].leftMap(_ => errorMsg).toValidatedNel + case None => + Validated.invalidNel(s"Missing '$fieldName' field in feed") + } + + private def parseListString( + fields: Map[String, Json], + fieldName: String + ): ValidatedNel[String, List[String]] = + fields.get(fieldName) match { + case Some(c) => + c.as[List[String]] + .leftMap(_ => s"Invalid '$fieldName' field in feed") + .toValidatedNel + case None => + Validated.invalidNel(s"Missing '$fieldName' field in feed") + } + + private def parseFeed(feedJson: Json): ValidatedNel[String, Feed] = + feedJson.asObject.map(_.toMap) match { + case None => Validated.invalidNel("Feed entry is not a valid JSON object") + case Some(feedFields) => + val nameValidated = parseString( + feedFields, + "name", + "Invalid 'name' field in feed" + ) + val commandsValidated = parseListString(feedFields, "commands") + val filterValidated = parseOptionalQueryAST(feedFields, "filter") + val formatInValidated + : Validated[NonEmptyList[String], Option[FormatIn]] = + parseOptionalFormatIn(feedFields, "formatIn") + + (nameValidated, commandsValidated, filterValidated, formatInValidated) + .mapN(Feed.apply) + } + + def parseYamlFile(content: String): ValidatedNel[String, ConfigYaml] = { + val uncommentedContent = content.linesIterator + .filterNot(line => line.trim.startsWith("#")) + .mkString("\n") + .trim + if (uncommentedContent.isEmpty) { + Validated.valid(ConfigYaml.empty) + } else { + parser.parse(content) match { + case Left(error) => + Validated.invalidNel(s"YAML parsing error: ${error.getMessage}") + case Right(json) => + json.asObject.map(_.toMap) match { + case None => Validated.invalidNel("YAML is not a valid JSON object") + case Some(fields) => + val filterValidated: ValidatedNel[String, Option[QueryAST]] = + parseOptionalQueryAST(fields, "filter") + + val formatInValidated: ValidatedNel[String, Option[FormatIn]] = + parseOptionalFormatIn(fields, "formatIn") + val commandsValidated + : ValidatedNel[String, Option[List[String]]] = + parseOptionalListString(fields, "commands") + val feedsValidated: ValidatedNel[String, Option[List[Feed]]] = + parseOptionalFeeds(fields, "feeds") + + ( + filterValidated, + formatInValidated, + commandsValidated, + feedsValidated + ).mapN(ConfigYaml.apply) + } + } + } + } +} diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala new file mode 100644 index 0000000..b61de99 --- /dev/null +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala @@ -0,0 +1,11 @@ +package ru.d10xa.jsonlogviewer.decline.yaml + +import ru.d10xa.jsonlogviewer.decline.Config.FormatIn +import ru.d10xa.jsonlogviewer.query.QueryAST + +case class Feed( + name: String, + commands: List[String], + filter: Option[QueryAST], + formatIn: Option[FormatIn] +) diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatter.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatter.scala index 32dcd83..0634ca0 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatter.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatter.scala @@ -7,7 +7,7 @@ import ru.d10xa.jsonlogviewer.OutputLineFormatter import ru.d10xa.jsonlogviewer.ParseResult import ru.d10xa.jsonlogviewer.decline.Config -class ColorLineFormatter(c: Config) extends OutputLineFormatter: +class ColorLineFormatter(c: Config, feedName: Option[String]) extends OutputLineFormatter: private val strEmpty: Str = Str("") private val strSpace: Str = Str(" ") private val strNewLine: Str = Str("\n") @@ -92,6 +92,12 @@ class ColorLineFormatter(c: Config) extends OutputLineFormatter: fansi.Color.White(prefix.ansiStrip) :: strSpace :: Nil case None => Nil + def strFeedName(s: Option[String]): Seq[Str] = + s match + case Some(feedName) => + fansi.Color.White(feedName.ansiStrip) :: strSpace :: Nil + case None => Nil + def strPostfix(s: Option[String]): Seq[Str] = s match case Some(postfix) => @@ -103,6 +109,7 @@ class ColorLineFormatter(c: Config) extends OutputLineFormatter: case Some(line) => val color = line.level.map(levelToColor).getOrElse(fansi.Color.White) val substrings1 = Seq( + strPrefix(feedName), strPrefix(p.prefix), strTimestamp(line.timestamp, fansi.Color.Green), strThreadName(line.threadName, color), diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/shell/Shell.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/shell/Shell.scala index 79dfb98..928f3ec 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/shell/Shell.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/shell/Shell.scala @@ -2,6 +2,6 @@ package ru.d10xa.jsonlogviewer.shell import fs2.* import cats.effect.* -trait Shell { - def mergeCommands(commands: List[String]): Stream[IO, String] +trait Shell[F[_]] { + def mergeCommands(commands: List[String]): Stream[F, String] } diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderTest.scala new file mode 100644 index 0000000..3078b61 --- /dev/null +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderTest.scala @@ -0,0 +1,85 @@ +package ru.d10xa.jsonlogviewer.decline.yaml + +import cats.data.Validated +import munit.FunSuite +import ru.d10xa.jsonlogviewer.decline.Config.FormatIn +import ru.d10xa.jsonlogviewer.query.QueryAST + +class ConfigYamlLoaderTest extends FunSuite { + + test("parse valid yaml with feeds") { + val yaml = + """|commands: + | - ./mock-logs.sh pod1 + | - ./mock-logs.sh pod2 + |filter: | + | message = 'first line' + |formatIn: json + |feeds: + | - name: "pod-logs" + | commands: + | - "./mock-logs.sh pod1" + | - "./mock-logs.sh pod2" + | filter: | + | message = 'first line' + | formatIn: json + | - name: "service-logs" + | commands: + | - "./mock-logs.sh service1" + | filter: | + | message = 'first line' + | formatIn: logfmt + |""".stripMargin + + val result = ConfigYamlLoader.parseYamlFile(yaml) + assert(result.isValid, s"Result should be valid: $result") + + val config = result.toOption.get + assertEquals(config.formatIn, Some(FormatIn.Json)) + assertEquals( + config.commands.get, + List("./mock-logs.sh pod1", "./mock-logs.sh pod2") + ) + assert(config.filter.isDefined) + + val feeds = config.feeds.get + assertEquals(feeds.size, 2) + + val feed1 = feeds.head + assertEquals(feed1.name, "pod-logs") + assertEquals( + feed1.commands, + List("./mock-logs.sh pod1", "./mock-logs.sh pod2") + ) + assertEquals(feed1.formatIn, Some(FormatIn.Json)) + + val feed2 = feeds(1) + assertEquals(feed2.name, "service-logs") + assertEquals(feed2.commands, List("./mock-logs.sh service1")) + assertEquals(feed2.formatIn, Some(FormatIn.Logfmt)) + } + + test("parse empty yaml") { + val yaml = "" + val result = ConfigYamlLoader.parseYamlFile(yaml) + assert(result.isValid, s"Result should be valid for empty yaml: $result") + + val config = result.toOption.get + assert(config.filter.isEmpty) + assert(config.formatIn.isEmpty) + assert(config.commands.isEmpty) + assert(config.feeds.isEmpty) + } + + test("parse invalid yaml") { + val yaml = + """formatIn: + | - not a string + |""".stripMargin + val result = ConfigYamlLoader.parseYamlFile(yaml) + assert(result.isInvalid, s"Result should be invalid: $result") + + val errors = result.swap.toOption.get + assert(errors.exists(_.contains("Invalid 'formatIn' field format"))) + } +}