Merge branch 'master' into consumer-e2e

.github/workflows/docker.yml

@@ -31,7 +31,9 @@ jobs:
       with:
         gradle-version: "8.10.2"
     - name: Execute Gradle build
-      run: ./gradlew clean build copyDependencies :server:copyJacocoAgent -x test
+      run: ./gradlew clean build copyDependencies -x test
+    - name: Create empty config directory
+      run: mkdir -p setup/docker/configs/varadhi-auto-generated
     - name: Build the Docker image
       run: docker build . --file setup/docker/Dockerfile --tag varadhi/varadhi:${{ inputs.releaseVersion }}
     - name: Push to DockerHub

.github/workflows/e2e.yml

@@ -28,9 +28,9 @@ jobs:
         with:
           gradle-version: "8.10.2"
       - name: Execute Gradle build
-        run: ./gradlew clean build copyDependencies :server:copyJacocoAgent -x test
+        run: ./gradlew clean build copyDependencies copyConfigForE2E -x test
       - name: Prepare mount directory for Web container - to write code coverage file.
-        run: chmod a+w server/build/jacoco
+        run: chmod a+w setup/docker/coverage
       - name: Build the Varadhi Docker image
         run: docker build . --file setup/docker/Dockerfile --tag varadhi.docker.registry/varadhi:latest --build-arg ENV=test
       - name: Setup the environment

.gitignore

@@ -6,7 +6,6 @@ build
 
 # IntelliJ
 .idea/*
-.run/*
 !.idea/vcs.xml
 *.iml
 *.iws
@@ -41,3 +40,9 @@ scratch
 
 scripts/benchmark/infra/
 nohup.out
+
+# auto generated configs
+setup/docker/configs/varadhi-auto-generated/*
+
+# coverage results
+**/*testE2E.exec

.run/Dev, start pulsar and zk.run.xml → .run/Start pulsar and zk (dev mode).run.xml

@@ -1,11 +1,11 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dev, start pulsar and zk" type="docker-deploy" factoryName="docker-compose.yml" server-name="Docker">
+  <configuration default="false" name="Start pulsar and zk (dev mode)" type="docker-deploy" factoryName="docker-compose.yml" server-name="Docker">
     <deployment type="docker-compose.yml">
       <settings>
         <option name="envFilePath" value="" />
         <option name="profiles">
           <list>
-            <option value="dev" />
+            <option value="dev-metric" />
           </list>
         </option>
         <option name="removeOrphansOnComposeDown" value="false" />

.run/varadhi local [run].run.xml

@@ -11,10 +11,9 @@
       <option name="taskNames">
         <list>
           <option value="run" />
-          <option value="--args=&quot;src/main/resources/configuration.yml&quot;" />
         </list>
       </option>
-      <option name="vmOptions" value="-Dvertx.disableDnsResolver=true -Dlog4j2.configurationFile=src/main/resources/log4j2.xml" />
+      <option name="vmOptions" value="" />
     </ExternalSystemSettings>
     <ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess>
     <ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>

README.md

@@ -22,23 +22,33 @@ spec and other detailed information.
 
 ## Build
 
-./gradlew build
+```bash
+./gradlew build test
+```
 
-## Run
+## Integration Tests
+
+```bash
+./gradlew copyDependencies copyConfigForE2E -x test
+
+docker build . --file setup/docker/Dockerfile --tag varadhi.docker.registry/varadhi:latest --build-arg
+
+docker compose --profile test -f setup/docker/compose.yml up -d --wait --wait-timeout 180
+
+./gradlew testE2E
+```
 
 ### Dependencies
 
-To provide the required functionality Varadhi takes a dependencies on various tech stack.
+To provide the required functionality, Varadhi takes dependencies on various tech stack.
 Current development environment supports below option for these.
 
 Run the following to start the below dependencies.
 
-```docker compose -f setup/docker/compose.yml -p docker --profile dev up -d```
+```docker compose --profile dev -f setup/docker/compose.yml up -d --wait --wait-timeout 180```
 
 OR
 
-Start [Dev, start pulsar and zk](.run%2FDev%2C%20start%20pulsar%20and%20zk.run.xml) IntelliJ run profile.
-
 #### Message Broker
 
 Varadhi needs a message broker for message persistence and delivery. [Apache Pulsar](https://pulsar.apache.org/) is used
@@ -54,13 +64,20 @@ used in a standalone mode. Details can be found [here](https://hub.docker.com/_/
 
 ### Varadhi Server
 
-To run the Varadhi server execute below from repo root.
+Finally, to run the Varadhi server, provide the custom zk & pulsar endpoints at `server/src/main/resources/config.overrides` and then do
+```./gradlew run```
 
-```./gradlew run --args="src/main/resources/configuration.yml```
+If you are using the dev profile to start the zk and pulsar, then simply do `./gradlew run`. No config overrides are required.
 
-OR
+## k8s Deployment
+
+```bash
+cd setup/helm/varadhi
+
+helm install varadhi-server . -f values/common.values.yaml -f values/local.server.values.yaml
 
-Start [varadhi local \[run\]](.run%2Fvaradhi%20local%20%5Brun%5D.run.xml) IntelliJ run profile.
+helm install varadhi-controller . -f values/common.values.yaml -f values/local.controller.values.yaml
+```
 
 ## Modules
 

build.gradle

@@ -0,0 +1,11 @@
+plugins {
+    id "com.flipkart.varadhi.java-common-conventions"
+}
+
+tasks.register('version') {
+    doLast {
+        println("Version: ${version}")
+    }
+}
+
+tasks.register("copyConfigForE2E", Copy, copyConfigToDir("setup/docker/config.overrides", "setup/docker/configs/varadhi-auto-generated"))

buildSrc/src/main/groovy/com.flipkart.varadhi.java-application-conventions.gradle

@@ -5,3 +5,11 @@ plugins {
     // Apply the application plugin to add support for building a CLI application in Java.
     id 'application'
 }
+
+ext {
+    copyConfigForModule = { scope ->
+        destination = layout.buildDirectory.dir("resources/$scope")
+        overridesFile = layout.projectDirectory.file("src/$scope/resources/config.overrides")
+        return copyConfigToDir(overridesFile, destination)
+    }
+}

buildSrc/src/main/groovy/com.flipkart.varadhi.java-common-conventions.gradle

@@ -7,13 +7,25 @@ plugins {
 }
 
 group = projectGroupId
-version = "${projectVersion}.${majorVersion}.${minorVersion}.${patchId}"
+version = "${majorVersion}.${minorVersion}.${patchId}"
 
 
 repositories {
     mavenCentral()
 }
 
+def loadProperties(path) {
+    def properties = new Properties()
+    def file = file(path)
+    if (file.exists()) {
+        file.withInputStream { stream ->
+            properties.load(stream)
+        }
+        return properties
+    }
+    return null
+}
+
 ext {
     lombok_version = "1.18.34"
     slf4j_version = "2.0.16"
@@ -42,6 +54,23 @@ ext {
     jmh_version = "1.37"
     awaitility_version = "4.2.2"
     jacoco_version = "0.8.10"
+
+    copyConfigToDir = {overridesFile, destination ->
+        return {
+            overrides = loadProperties(overridesFile)
+            if (!overrides) {
+                return
+            }
+
+            from("$rootDir/conf")
+            into(destination)
+            inputs.file(overridesFile)
+
+            filesMatching("*") {
+                expand(overrides)
+            }
+        }
+    }
 }
 
 sourceSets {
@@ -170,9 +199,15 @@ tasks.withType(JacocoReport).configureEach {
     getSourceDirectories().from(sourceSets.main.allSource.srcDirs)
     getClassDirectories().from(sourceSets.main.output)
     getExecutionData().from(fileTree(buildDir).include("/jacoco/*.exec"))
+    getExecutionData().from(file("$rootDir/setup/docker/coverage/testE2E.exec"))
 }
 
 tasks.withType(Test).configureEach {
+    // check if tasks named 'copyConfig' is present in the project
+    if (tasks.findByName('copyTestConfig')) {
+        dependsOn 'copyTestConfig'
+    }
+
     useJUnitPlatform()
     testLogging {
         events "PASSED", "SKIPPED", "FAILED", "STANDARD_OUT", "STANDARD_ERROR"

common/src/main/java/com/flipkart/varadhi/Constants.java

@@ -6,6 +6,7 @@ public class Constants {
     public static final int RANDOM_PARTITION_KEY_LENGTH = 5;
     public static final String CONTEXT_KEY_BODY = "varadhi.body";
     public static final String CONTEXT_KEY_RESOURCE_HIERARCHY = "varadhi.resourceHierarchy";
+    public static final String CONTEXT_KEY_IS_SUPER_USER = "varadhi.isSuperUser";
 
     // TODO: this header is only for testing and x_ convention may cause it to be sent to the destination during consume
     public static final String USER_ID_HEADER = "x_user_id";

common/src/main/java/com/flipkart/varadhi/reflect/RecursiveFieldUpdater.java

@@ -0,0 +1,48 @@
+package com.flipkart.varadhi.reflect;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
+import java.util.Objects;
+import java.util.function.BiFunction;
+
+public class RecursiveFieldUpdater {
+
+    public static <T, V> void visit(T object, Class<? extends Annotation> annotationClass, BiFunction<String, V, V> updateFunction) {
+        Objects.requireNonNull(object, "Object cannot be null");
+        traverseObject(object, annotationClass, updateFunction, "");
+    }
+
+    private static <T, V> void traverseObject(T obj, Class<? extends Annotation> annotationClass, BiFunction<String, V, V> updateFunction, String path) {
+        if (obj == null) return;
+
+        Class<?> clazz = obj.getClass();
+        for (Field field : clazz.getDeclaredFields()) {
+            field.setAccessible(true);
+            String fieldPath = path.isEmpty() ? field.getName() : path + "." + field.getName();
+
+            try {
+                Object fieldValue = field.get(obj);
+
+                // If the field is annotated with the specified annotation, process it
+                if (field.isAnnotationPresent(annotationClass)) {
+                    // Apply the update function to get the new value
+                    V newValue = updateFunction.apply(fieldPath, (V)fieldValue);
+                    setFieldValue(obj, field, newValue);
+                } else if (!field.getType().isPrimitive() && field.getType().getName().startsWith("com.flipkart.varadhi.")) {
+                    // Recursively visit nested POJOs
+                    traverseObject(fieldValue, annotationClass, updateFunction, fieldPath);
+                }
+            } catch (IllegalAccessException e) {
+                throw new RuntimeException("Failed to access field: " + field.getName(), e);
+            }
+        }
+    }
+
+    private static void setFieldValue(Object obj, Field field, Object newValue) {
+        try {
+            field.set(obj, newValue);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException("Failed to set field value for: " + field.getName(), e);
+        }
+    }
+}

common/src/main/java/com/flipkart/varadhi/utils/HostUtils.java

@@ -1,18 +1,24 @@
 package com.flipkart.varadhi.utils;
 
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 @Slf4j
 public class HostUtils {
+    @Getter
+    private static String hostName;
+    @Getter
+    private static String hostAddress;
 
-    public static String getHostName() throws UnknownHostException {
-        // debug to see how much time it takes, in case DNS resolution is taking time.
-        log.debug("getHostName: started");
-        String host = InetAddress.getLocalHost().getHostName();
-        log.debug("getHostName: completed");
-        return host;
+    public static void initHostUtils() throws UnknownHostException {
+        if(hostName == null) {
+            hostName = InetAddress.getLocalHost().getHostName();
+        }
+        if(hostAddress == null) {
+            hostAddress = InetAddress.getLocalHost().getHostAddress();
+        }
     }
 }

common/src/main/java/com/flipkart/varadhi/utils/YamlLoader.java

@@ -7,6 +7,9 @@
 import com.flipkart.varadhi.exceptions.InvalidConfigException;
 
 import java.io.File;
+import java.net.URL;
+import java.nio.file.Paths;
+import java.util.Collections;
 import java.util.List;
 
 public class YamlLoader {
@@ -20,7 +23,17 @@ public class YamlLoader {
     public static <T> T loadConfig(String configFile, Class<T> clazz) {
         T config;
         try {
-            config = mapper.readValue(new File(configFile), clazz);
+            File file = new File(configFile);
+            if (file.exists()) {
+                config = mapper.readValue(new File(configFile), clazz);
+            } else {
+                String fileName = Paths.get(configFile).getFileName().toString();
+                URL url = Collections.list(clazz.getClassLoader().getResources(fileName)).stream()
+                        .filter(e -> e.toString().endsWith(configFile))
+                        .findFirst()
+                        .orElseThrow(() -> new InvalidConfigException("Config file not found: " + configFile));
+                config = mapper.readValue(url, clazz);
+            }
         } catch (Exception e) {
             throw new InvalidConfigException(
                     String.format("Failed to load config file: %s as %s.", configFile, clazz), e);

common/src/main/java/module-info.java

@@ -11,4 +11,5 @@
 
     exports com.flipkart.varadhi;
     exports com.flipkart.varadhi.exceptions;
+    exports com.flipkart.varadhi.reflect;
 }

common/src/test/java/com/flipkart/varadhi/utils/HostUtilsTest.java

@@ -1,18 +1,33 @@
 package com.flipkart.varadhi.utils;
 
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 public class HostUtilsTest {
 
+    @BeforeAll
+    public static void init() throws UnknownHostException {
+        HostUtils.initHostUtils();
+    }
+
     @Test
     public void TestGetHostName() throws UnknownHostException {
         // dummy test.. no validations here.
         String host = HostUtils.getHostName();
         Assertions.assertNotNull(host);
         Assertions.assertEquals(InetAddress.getLocalHost().getHostName(), host);
     }
+
+    @Test
+    public void TestGetHostAddress() throws UnknownHostException {
+        // dummy test.. no validations here.
+        String address = HostUtils.getHostAddress();
+        Assertions.assertNotNull(address);
+        Assertions.assertEquals(InetAddress.getLocalHost().getHostAddress(), address);
+    }
+
 }