Skip to content

Commit

Permalink
refactor: migrate target
Browse files Browse the repository at this point in the history
migrate abstract target
AdswerveBigQuery
DatamillCoPostgres
GenericTarget
MeltanoSnowflake
Oracle
  • Loading branch information
mgabelle committed Dec 17, 2024
1 parent 7d1d67a commit 2b07dff
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 46 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ public class GenericTarget extends AbstractPythonTarget implements RunnableTask<
description = "Will be save on config.json and used as arguments"
)
@PluginProperty(dynamic = true)
private Map<String, Object> configs;
private Property<Map<String, Object>> configs;

@Override
public Map<String, Object> configuration(RunContext runContext) throws IllegalVariableEvaluationException, IOException {
return runContext.render(this.configs);
return runContext.render(this.configs).asMap(String.class, Object.class);
}

@Override
Expand Down
30 changes: 12 additions & 18 deletions src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ public class MeltanoSnowflake extends AbstractPythonTarget implements RunnableTa
@Schema(
title = "The database user's password."
)
@PluginProperty(dynamic = true)
private String password;
private Property<String> password;

@NotNull
@NotEmpty
Expand All @@ -76,59 +75,54 @@ public class MeltanoSnowflake extends AbstractPythonTarget implements RunnableTa
@Schema(
title = "The initial role for the session."
)
@PluginProperty(dynamic = true)
private String role;
private Property<String> role;

@Schema(
title = "Whether to add metadata columns."
)
@PluginProperty
@Builder.Default
private Boolean addRecordMetadata = true;
private Property<Boolean> addRecordMetadata = Property.of(true);

@Schema(
title = "The default target database schema name to use for all streams."
)
@PluginProperty(dynamic = true)
private String defaultTargetSchema;
private Property<String> defaultTargetSchema;

@Schema(
title = "'True' to enable schema flattening and automatically expand nested properties."
)
@PluginProperty
private Boolean flatteningEnabled;
private Property<Boolean> flatteningEnabled;

@Schema(
title = "The max depth to flatten schemas."
)
@PluginProperty
private Integer flatteningMaxDepth;
private Property<Integer> flatteningMaxDepth;

@Override
public Map<String, Object> configuration(RunContext runContext) throws IllegalVariableEvaluationException {
ImmutableMap.Builder<String, Object> builder = ImmutableMap.<String, Object>builder()
.put("account", runContext.render(this.account))
.put("database", runContext.render(this.database))
.put("username", runContext.render(this.username))
.put("password", runContext.render(this.password))
.put("password", runContext.render(this.password).as(String.class).orElseThrow())
.put("warehouse", runContext.render(this.warehouse))
.put("schema", runContext.render(this.schema))
.put("add_record_metadata", this.addRecordMetadata.toString());
.put("add_record_metadata", runContext.render(this.addRecordMetadata).as(Boolean.class).orElseThrow().toString());

if (this.role != null) {
builder.put("role", runContext.render(this.role));
builder.put("role", runContext.render(this.role).as(String.class).orElseThrow());
}

if (this.defaultTargetSchema != null) {
builder.put("default_target_schema", runContext.render(this.defaultTargetSchema));
builder.put("default_target_schema", runContext.render(this.defaultTargetSchema).as(String.class).orElseThrow());
}

if (this.flatteningEnabled != null) {
builder.put("flattening_enabled", this.flatteningEnabled.toString());
builder.put("flattening_enabled", runContext.render(this.flatteningEnabled).as(Boolean.class).orElseThrow().toString());
}

if (this.flatteningMaxDepth != null) {
builder.put("flattening_max_depth", this.flatteningMaxDepth);
builder.put("flattening_max_depth", runContext.render(this.flatteningMaxDepth).as(Integer.class).orElseThrow());
}

return builder.build();
Expand Down
40 changes: 16 additions & 24 deletions src/main/java/io/kestra/plugin/singer/targets/Oracle.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,13 @@ public class Oracle extends AbstractPythonTarget implements RunnableTask<Abstrac
@Schema(
title = "The database name."
)
@PluginProperty(dynamic = true)
private String database;
private Property<String> database;

@NotNull
@Schema(
title = "The database port."
)
@PluginProperty
private Integer port;
private Property<Integer> port;

@NotEmpty
@Schema(
Expand All @@ -66,70 +64,64 @@ public class Oracle extends AbstractPythonTarget implements RunnableTask<Abstrac
@Schema(
title = "SQLAlchemy driver name."
)
@PluginProperty(dynamic = true)
private String driverName;
private Property<String> driverName;

@Schema(
title = "Use float data type for numbers (otherwise number type is used)."
)
@PluginProperty
private Boolean preferFloatOverNumeric;
private Property<Boolean> preferFloatOverNumeric;

@Schema(
title = "Config object for stream maps capability."
)
@PluginProperty(dynamic = true)
private String streamMaps;
private Property<String> streamMaps;

@Schema(
title = "User-defined config values to be used within map expressions."
)
@PluginProperty(dynamic = true)
private String streamMapConfig;
private Property<String> streamMapConfig;

@Schema(
title = "Enable schema flattening and automatically expand nested properties."
)
@PluginProperty
private Boolean flatteningEnabled;
private Property<Boolean> flatteningEnabled;

@Schema(
title = "The max depth to flatten schemas."
)
@PluginProperty
private Integer flatteningMaxDepth;
private Property<Integer> flatteningMaxDepth;

@Override
public Map<String, Object> configuration(RunContext runContext) throws IllegalVariableEvaluationException {
ImmutableMap.Builder<String, Object> builder = ImmutableMap.<String, Object>builder()
.put("user", runContext.render(this.username))
.put("password", runContext.render(this.password))
.put("host", runContext.render(this.host))
.put("port", String.valueOf(this.port))
.put("database", runContext.render(this.database));
.put("port", String.valueOf(runContext.render(this.port).as(Integer.class).orElseThrow()))
.put("database", runContext.render(this.database).as(String.class).orElse(null));

if (this.driverName != null) {
builder.put("driver_name", runContext.render(this.driverName));
builder.put("driver_name", runContext.render(this.driverName).as(String.class).orElseThrow());
}

if (this.preferFloatOverNumeric != null) {
builder.put("prefer_float_over_numeric", StringUtils.capitalize(this.preferFloatOverNumeric.toString()));
builder.put("prefer_float_over_numeric", StringUtils.capitalize(runContext.render(this.preferFloatOverNumeric).as(Boolean.class).orElseThrow().toString()));
}

if (this.streamMaps != null) {
builder.put("stream_maps", runContext.render(this.streamMaps));
builder.put("stream_maps", runContext.render(this.streamMaps).as(String.class).orElseThrow());
}

if (this.streamMapConfig != null) {
builder.put("stream_map_config", runContext.render(this.streamMapConfig));
builder.put("stream_map_config", runContext.render(this.streamMapConfig).as(String.class).orElseThrow());
}

if (this.flatteningEnabled != null) {
builder.put("flattening_enabled", StringUtils.capitalize(this.flatteningEnabled.toString()));
builder.put("flattening_enabled", StringUtils.capitalize(runContext.render(this.flatteningEnabled).as(Boolean.class).orElseThrow().toString()));
}

if (this.flatteningMaxDepth != null) {
builder.put("flattening_max_depth", this.flatteningMaxDepth);
builder.put("flattening_max_depth", runContext.render(this.flatteningMaxDepth).as(Integer.class).orElseThrow());
}

return builder.build();
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/io/kestra/plugin/singer/targets/OracleTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,10 @@ void run() throws Exception {
.type(Oracle.class.getName())
.from(Property.of(tapOutput.getRaw().toString()))
.host("172.17.0.1")
.database("FREE")
.database(Property.of("FREE"))
.username("system")
.password("oracle_passwd")
.port(57057)
.port(Property.of(57057))
.build();

runContext = TestsUtils.mockRunContext(runContextFactory, target, ImmutableMap.of());
Expand Down

0 comments on commit 2b07dff

Please sign in to comment.