From f58ff0f34ec8ee1ebc01a7c32028f04e50aa5989 Mon Sep 17 00:00:00 2001 From: itaigilo Date: Wed, 26 Jun 2024 15:21:30 +0300 Subject: [PATCH] Add Spark client user metadata search example to spark-client.md (#7918) * Add a metadata search example to spark-client.md * Fix numbering --- docs/reference/spark-client.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/reference/spark-client.md b/docs/reference/spark-client.md index db2eb282b68..b1bc02bbf05 100644 --- a/docs/reference/spark-client.md +++ b/docs/reference/spark-client.md @@ -117,5 +117,27 @@ Include this assembled jar (an "Überjar") from S3, from */ ``` +1. Search by user metadata: + + ```scala + import io.treeverse.clients.LakeFSContext + + val namespace = "s3://bucket/repo/path/" + val df = LakeFSContext.newDF(spark, namespace) + + val key = "SomeKey" + val searchedValue = "val3" + df.select("key", "user_metadata") + .filter(_.getMap[String, String](1).toMap.get(s"X-Amz-Meta-${key}").getOrElse("") == searchedValue) + .show() + /* output example: + +---------+-----------------------------------------------------+ + |key |user_metadata | + +---------+-----------------------------------------------------+ + |file1.txt|{X-Amz-Meta-SomeKey -> val3, X-Amz-Meta-Tag -> blue} | + |file8.txt|{X-Amz-Meta-SomeKey -> val3, X-Amz-Meta-Tag -> green}| + +---------+-----------------------------------------------------+ + */ + ``` [s3a-assumed-role]: https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/assumed_roles.html#Configuring_Assumed_Roles