Skip to content

Commit 0d6a4c4

Browse files
piotrczarnaspiotrczarnas
committed
Python rules receive a path to the dqops home and the user's home to register packages.
1 parent a8ac59b commit 0d6a4c4

20 files changed

+192
-40
lines changed

‎dqops/src/main/java/com/dqops/cli/CliInitializerImpl.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.dqops.core.domains.LocalDataDomainManager;
2727
import com.dqops.core.dqocloud.apikey.DqoCloudApiKey;
2828
import com.dqops.core.dqocloud.apikey.DqoCloudApiKeyProvider;
29+
import com.dqops.core.filesystem.cache.LocalFileSystemCache;
2930
import com.dqops.core.jobqueue.DqoJobQueue;
3031
import com.dqops.core.jobqueue.ParentDqoJobQueue;
3132
import com.dqops.core.jobqueue.monitoring.DqoJobQueueMonitoringService;
@@ -81,6 +82,7 @@ public class CliInitializerImpl implements CliInitializer {
8182
private DataDomainsService dataDomainsService;
8283
private TableSimilarityReconciliationService tableSimilarityReconciliationService;
8384
private TableSimilarityRefreshService tableSimilarityRefreshService;
85+
private LocalFileSystemCache localFileSystemCache;
8486

8587
/**
8688
* Called by the dependency injection container to provide dependencies.
@@ -108,6 +110,7 @@ public class CliInitializerImpl implements CliInitializer {
108110
* @param dataDomainsService Data domains service to synchronize domains from the SaaS backend.
109111
* @param tableSimilarityReconciliationService Table similarity reindexing service.
110112
* @param tableSimilarityRefreshService Table similarity dynamic refresh service.
113+
* @param localFileSystemCache Local file system cache.
111114
*/
112115
@Autowired
113116
public CliInitializerImpl(LocalUserHomeCreator localUserHomeCreator,
@@ -133,7 +136,8 @@ public CliInitializerImpl(LocalUserHomeCreator localUserHomeCreator,
133136
LocalDataDomainManager localDataDomainManager,
134137
DataDomainsService dataDomainsService,
135138
TableSimilarityReconciliationService tableSimilarityReconciliationService,
136-
TableSimilarityRefreshService tableSimilarityRefreshService) {
139+
TableSimilarityRefreshService tableSimilarityRefreshService,
140+
LocalFileSystemCache localFileSystemCache) {
137141
this.localUserHomeCreator = localUserHomeCreator;
138142
this.dqoCloudApiKeyProvider = dqoCloudApiKeyProvider;
139143
this.terminalReader = terminalReader;
@@ -158,6 +162,7 @@ public CliInitializerImpl(LocalUserHomeCreator localUserHomeCreator,
158162
this.dataDomainsService = dataDomainsService;
159163
this.tableSimilarityReconciliationService = tableSimilarityReconciliationService;
160164
this.tableSimilarityRefreshService = tableSimilarityRefreshService;
165+
this.localFileSystemCache = localFileSystemCache;
161166
}
162167

163168
/**
@@ -220,6 +225,7 @@ public void initializeApp(String[] args) {
220225
JdbcTypeColumnMapping.ensureInitializedJdbc();
221226

222227
boolean isHeadless = Arrays.stream(args).anyMatch(arg -> Objects.equals(arg, "--headless") || Objects.equals(arg, "-hl"));
228+
this.localFileSystemCache.start();
223229
this.labelsIndexer.start();
224230
this.tableStatusCache.start();
225231
this.tableLineageCache.start();

‎dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbSourceConnection.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -282,10 +282,10 @@ private List<String> getAvailableExtensions() {
282282
*/
283283
private void registerExtensions() {
284284
try {
285-
File userLocationFile = Path.of(this.homeLocationFindService.getUserHomePath()).resolve("bin/.duckdb/extensions").toFile();
285+
File userLocationFile = Path.of(this.homeLocationFindService.getRootUserHomePath()).resolve("bin/.duckdb/extensions").toFile();
286286
String setExtensionsQuery =
287287
userLocationFile.exists() && userLocationFile.isDirectory() ?
288-
DuckdbQueriesProvider.provideSetExtensionsQuery(this.homeLocationFindService.getUserHomePath()) :
288+
DuckdbQueriesProvider.provideSetExtensionsQuery(this.homeLocationFindService.getRootUserHomePath()) :
289289
DuckdbQueriesProvider.provideSetExtensionsQuery(this.homeLocationFindService.getDqoHomePath());
290290
this.executeCommand(setExtensionsQuery, JobCancellationToken.createDummyJobCancellationToken());
291291

‎dqops/src/main/java/com/dqops/core/domains/LocalDataDomainManagerImpl.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public void start() {
7878
public void initializeLocalDataDomain(LocalDataDomainSpec dataDomainSpec) {
7979
String dataDomainName = dataDomainSpec.getDataDomainName();
8080
HomeFolderPath domainRootVirtualFolder = new HomeFolderPath(dataDomainName);
81-
Path pathToRootUserHome = Path.of(this.homeLocationFindService.getUserHomePath());
81+
Path pathToRootUserHome = Path.of(this.homeLocationFindService.getRootUserHomePath());
8282
Path dataDomainRootFolder = pathToRootUserHome.resolve(domainRootVirtualFolder.toRelativePath());
8383
String pathToDataDomainFolder = dataDomainRootFolder.toString();
8484

‎dqops/src/main/java/com/dqops/core/filesystem/cache/LocalFileSystemCache.java

+5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@
3030
* Local file system cache.
3131
*/
3232
public interface LocalFileSystemCache extends DisposableBean {
33+
/**
34+
* Starts a local file system cache.
35+
*/
36+
void start();
37+
3338
/**
3439
* Retrieves a list of folders, calling the load callback to load the data if it is missing in the cache.
3540
*

‎dqops/src/main/java/com/dqops/core/filesystem/cache/LocalFileSystemCacheImpl.java

+28-13
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@
2323
import com.dqops.core.filesystem.virtual.FileNameSanitizer;
2424
import com.dqops.core.filesystem.virtual.HomeFilePath;
2525
import com.dqops.core.filesystem.virtual.HomeFolderPath;
26+
import com.dqops.core.principal.DqoUserPrincipal;
27+
import com.dqops.core.principal.DqoUserPrincipalProvider;
28+
import com.dqops.core.principal.UserDomainIdentity;
2629
import com.dqops.core.similarity.TableSimilarityRefreshService;
2730
import com.dqops.core.similarity.TableSimilarityRefreshServiceProvider;
2831
import com.dqops.data.checkresults.statuscache.DomainConnectionTableKey;
@@ -79,7 +82,7 @@ public class LocalFileSystemCacheImpl implements LocalFileSystemCache, Disposabl
7982
private final TableLineageCacheProvider tableLineageCacheProvider;
8083
private final TableSimilarityRefreshServiceProvider tableSimilarityRefreshServiceProvider;
8184
private final HomeLocationFindService homeLocationFindService;
82-
private final Path userHomeRootPath;
85+
private Path userHomeRootPath;
8386
private Instant nextFileChangeDetectionAt = Instant.now().minus(100L, ChronoUnit.MILLIS);
8487
private boolean wasRecentlyInvalidated;
8588

@@ -105,17 +108,6 @@ public LocalFileSystemCacheImpl(DqoCacheConfigurationProperties dqoCacheConfigur
105108
this.tableLineageCacheProvider = tableLineageCacheProvider;
106109
this.tableSimilarityRefreshServiceProvider = tableSimilarityRefreshServiceProvider;
107110
this.homeLocationFindService = homeLocationFindService;
108-
this.userHomeRootPath = homeLocationFindService.getUserHomePath() != null ? Path.of(homeLocationFindService.getUserHomePath()) : Path.of(".");
109-
110-
WatchService newWatchService = null;
111-
if (dqoCacheConfigurationProperties.isEnable() && dqoCacheConfigurationProperties.isWatchFileSystemChanges()) {
112-
try {
113-
newWatchService = FileSystems.getDefault().newWatchService();
114-
} catch (IOException ioe) {
115-
throw new DqoRuntimeException("Cannot create a watch service.");
116-
}
117-
}
118-
this.watchService = newWatchService;
119111

120112
this.folderListsCache = Caffeine.newBuilder()
121113
.maximumSize(dqoCacheConfigurationProperties.getYamlFilesLimit())
@@ -140,6 +132,25 @@ public LocalFileSystemCacheImpl(DqoCacheConfigurationProperties dqoCacheConfigur
140132
.build();
141133
}
142134

135+
/**
136+
* Starts a local file system cache.
137+
*/
138+
@Override
139+
public void start() {
140+
this.userHomeRootPath = homeLocationFindService.getRootUserHomePath() != null ?
141+
Path.of(homeLocationFindService.getRootUserHomePath()) : Path.of(".");
142+
143+
WatchService newWatchService = null;
144+
if (dqoCacheConfigurationProperties.isEnable() && dqoCacheConfigurationProperties.isWatchFileSystemChanges()) {
145+
try {
146+
newWatchService = FileSystems.getDefault().newWatchService();
147+
} catch (IOException ioe) {
148+
throw new DqoRuntimeException("Cannot create a watch service.");
149+
}
150+
}
151+
this.watchService = newWatchService;
152+
}
153+
143154
/**
144155
* Called when the object is destroyed. Stops the watcher.
145156
* @throws Exception
@@ -157,7 +168,7 @@ public void destroy() throws Exception {
157168
* @param folderPath Folder path.
158169
*/
159170
public void startFolderWatcher(Path folderPath) {
160-
if (folderPath == null) {
171+
if (folderPath == null || this.watchService == null) {
161172
return;
162173
}
163174

@@ -481,6 +492,10 @@ public void invalidateFile(Path filePath) {
481492
* and it is not a real invalidation, but just a notification that a file was just cached.
482493
*/
483494
public void invalidateTableCaches(Path filePath, boolean replacingCachedFile) {
495+
if (this.userHomeRootPath == null) {
496+
return; // not initialized
497+
}
498+
484499
if (!filePath.startsWith(this.userHomeRootPath)) {
485500
return;
486501
}

‎dqops/src/main/java/com/dqops/core/filesystem/localfiles/HomeLocationFindService.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.dqops.core.filesystem.localfiles;
1717

18+
import com.dqops.core.principal.UserDomainIdentity;
1819
import com.dqops.metadata.storage.localfiles.HomeType;
1920

2021
/**
@@ -25,9 +26,16 @@ public interface HomeLocationFindService {
2526

2627
/**
2728
* Returns an absolute path to the user home.
29+
* @param userDomainIdentity User home identity to identify the data domain.
2830
* @return Absolute path to the user home. May return null if the user home is not enabled.
2931
*/
30-
String getUserHomePath();
32+
String getUserHomePath(UserDomainIdentity userDomainIdentity);
33+
34+
/**
35+
* Returns an absolute path to the root user home.
36+
* @return Absolute path to the user home. May return null if the user home is not enabled.
37+
*/
38+
String getRootUserHomePath();
3139

3240
/**
3341
* Returns an absolute path to the dqo home.
@@ -38,7 +46,8 @@ public interface HomeLocationFindService {
3846
/**
3947
* Returns the absolute path to a home of choice (user home or DQO_HOME).
4048
* @param homeType Home type (user home or dqo system home).
49+
* @param userDomainIdentity User domain identity to identify the data domain when the user home is requested.
4150
* @return Absolute path to home.
4251
*/
43-
String getHomePath(HomeType homeType);
52+
String getHomePath(HomeType homeType, UserDomainIdentity userDomainIdentity);
4453
}

‎dqops/src/main/java/com/dqops/core/filesystem/localfiles/HomeLocationFindServiceImpl.java

+33-3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
import com.dqops.core.configuration.DqoConfigurationProperties;
1919
import com.dqops.core.configuration.DqoUserConfigurationProperties;
20+
import com.dqops.core.filesystem.BuiltInFolderNames;
21+
import com.dqops.core.principal.UserDomainIdentity;
2022
import com.dqops.metadata.storage.localfiles.HomeType;
2123
import com.google.common.base.Strings;
2224
import org.springframework.beans.factory.annotation.Autowired;
@@ -25,6 +27,7 @@
2527
import java.io.IOException;
2628
import java.nio.file.Files;
2729
import java.nio.file.Path;
30+
import java.util.Objects;
2831

2932
/**
3033
* Simple service that returns the location of the user home or the DQO_HOME system home.
@@ -70,10 +73,36 @@ private Path detectParentDqoUserHome(Path userHomePath) {
7073

7174
/**
7275
* Returns an absolute path to the user home.
76+
* @param userDomainIdentity User domain identity to identify a data domain.
7377
* @return Absolute path to the user home. May return null if the user home is not enabled.
7478
*/
7579
@Override
76-
public String getUserHomePath() {
80+
public String getUserHomePath(UserDomainIdentity userDomainIdentity) {
81+
String rootUserHomePath = this.getRootUserHomePath();
82+
83+
if (rootUserHomePath == null) {
84+
return null;
85+
}
86+
87+
if (Objects.equals(userDomainIdentity.getDataDomainFolder(), UserDomainIdentity.ROOT_DATA_DOMAIN)) {
88+
return rootUserHomePath;
89+
}
90+
91+
String nestedDataDomainUserHomePath = Path.of(rootUserHomePath)
92+
.resolve(BuiltInFolderNames.DATA_DOMAINS)
93+
.resolve(userDomainIdentity.getDataDomainFolder())
94+
.toString();
95+
96+
return nestedDataDomainUserHomePath;
97+
}
98+
99+
/**
100+
* Returns an absolute path to the root user home.
101+
*
102+
* @return Absolute path to the user home. May return null if the user home is not enabled.
103+
*/
104+
@Override
105+
public String getRootUserHomePath() {
77106
if (this.userConfigurationProperties.isHasLocalHome()) {
78107
if (this.userHomePath != null) {
79108
return this.userHomePath;
@@ -129,13 +158,14 @@ public String getDqoHomePath() {
129158
/**
130159
* Returns the absolute path to a home of choice (user home or DQO_HOME).
131160
* @param homeType Home type (user home or dqo system home).
161+
* @param userDomainIdentity User domain identity.
132162
* @return Absolute path to home.
133163
*/
134164
@Override
135-
public String getHomePath(HomeType homeType) {
165+
public String getHomePath(HomeType homeType, UserDomainIdentity userDomainIdentity) {
136166
switch (homeType) {
137167
case USER_HOME:
138-
return getUserHomePath();
168+
return getUserHomePath(userDomainIdentity);
139169
case DQO_HOME:
140170
return getDqoHomePath();
141171
default:

‎dqops/src/main/java/com/dqops/execution/rules/runners/python/PythonRuleCallInput.java

+34
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ public class PythonRuleCallInput {
3333
private String dataDomainModule;
3434
private String ruleModulePath;
3535
private String homePath;
36+
private String dqoHomePath;
37+
private String dqoRootUserHomePath;
3638
private Instant ruleModuleLastModified;
3739
private RuleExecutionRunParameters ruleParameters;
3840

@@ -84,6 +86,38 @@ public void setHomePath(String homePath) {
8486
this.homePath = homePath;
8587
}
8688

89+
/**
90+
* Returns the path to the DQOps Home folder.
91+
* @return DQOps home folder path.
92+
*/
93+
public String getDqoHomePath() {
94+
return dqoHomePath;
95+
}
96+
97+
/**
98+
* Sets a path to the DQOps home folder.
99+
* @param dqoHomePath Path to the DQOps home folder.
100+
*/
101+
public void setDqoHomePath(String dqoHomePath) {
102+
this.dqoHomePath = dqoHomePath;
103+
}
104+
105+
/**
106+
* Returns the path to the root DQOps user home folder.
107+
* @return Path to the root user folder.
108+
*/
109+
public String getDqoRootUserHomePath() {
110+
return dqoRootUserHomePath;
111+
}
112+
113+
/**
114+
* Sets a path to a root DQOps user home folder that is used to resolve additional python packages.
115+
* @param dqoRootUserHomePath DQOps root user home path.
116+
*/
117+
public void setDqoRootUserHomePath(String dqoRootUserHomePath) {
118+
this.dqoRootUserHomePath = dqoRootUserHomePath;
119+
}
120+
87121
/**
88122
* Returns the timestamp when the Python rule file was last modified.
89123
* @return The last modification timestamp of the rule module.

‎dqops/src/main/java/com/dqops/execution/rules/runners/python/PythonRuleRunner.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.dqops.core.configuration.DqoPythonConfigurationProperties;
1919
import com.dqops.core.filesystem.localfiles.HomeLocationFindService;
2020
import com.dqops.core.filesystem.virtual.HomeFilePath;
21+
import com.dqops.core.principal.UserDomainIdentity;
2122
import com.dqops.execution.ExecutionContext;
2223
import com.dqops.execution.rules.RuleExecutionResult;
2324
import com.dqops.execution.rules.RuleExecutionRunParameters;
@@ -79,10 +80,14 @@ public RuleExecutionResult executeRule(ExecutionContext executionContext,
7980
String dataDomainModule = Strings.isNullOrEmpty(dataDomainFolder) ? "default" : dataDomainFolder.replace(' ', '_');
8081
ruleInput.setDataDomainModule(dataDomainModule);
8182
ruleInput.setRuleParameters(ruleRunParameters);
82-
String pathToHome = this.homeLocationFindService.getHomePath(ruleDefinitionFindResult.getHome());
83+
84+
UserDomainIdentity userIdentity = executionContext.getUserHomeContext() != null ? executionContext.getUserHomeContext().getUserIdentity() : null;
85+
String pathToHome = this.homeLocationFindService.getHomePath(ruleDefinitionFindResult.getHome(), userIdentity);
8386
String absolutePathToPythonRule = Path.of(pathToHome).resolve(ruleHomeRelativePath.toRelativePath()).toAbsolutePath().toString();
8487
ruleInput.setRuleModulePath(absolutePathToPythonRule);
8588
ruleInput.setHomePath(pathToHome);
89+
ruleInput.setDqoHomePath(this.homeLocationFindService.getDqoHomePath());
90+
ruleInput.setDqoRootUserHomePath(this.homeLocationFindService.getRootUserHomePath());
8691
ruleInput.setRuleModuleLastModified(ruleDefinitionFindResult.getRulePythonFileLastModified());
8792

8893
PythonRuleCallOutput output = this.pythonCallerService.executePythonHomeScript(ruleInput, evaluateRulesModule, PythonRuleCallOutput.class);

‎dqops/src/main/java/com/dqops/metadata/storage/localfiles/userhome/LocalUserHomeCreatorImpl.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ public void initializeEmptyFolder(Path path) {
159159
*/
160160
@Override
161161
public boolean initializeDefaultDqoUserHome() {
162-
String userHomePathString = this.homeLocationFindService.getUserHomePath();
162+
String userHomePathString = this.homeLocationFindService.getRootUserHomePath();
163163
if (userHomePathString == null) {
164164
return false;
165165
}
@@ -196,7 +196,7 @@ public boolean initializeDefaultDqoUserHome() {
196196
*/
197197
@Override
198198
public boolean isDefaultDqoUserHomeInitialized() {
199-
String userHomePathString = this.homeLocationFindService.getUserHomePath();
199+
String userHomePathString = this.homeLocationFindService.getRootUserHomePath();
200200

201201
if (userHomePathString == null) {
202202
return false;
@@ -409,7 +409,7 @@ protected boolean isUninitializedInUnmountedDockerVolume(Path userHomePath) {
409409
*/
410410
@Override
411411
public boolean ensureDefaultUserHomeIsInitialized(boolean isHeadless) {
412-
String userHomePathString = this.homeLocationFindService.getUserHomePath();
412+
String userHomePathString = this.homeLocationFindService.getRootUserHomePath();
413413
if (userHomePathString == null) {
414414
return false; // the dqo user home is not required for some reason (configurable)
415415
}
@@ -577,7 +577,7 @@ public void activateFileLoggingInUserHome() {
577577
return;
578578
}
579579

580-
String userHomePath = this.homeLocationFindService.getUserHomePath();
580+
String userHomePath = this.homeLocationFindService.getRootUserHomePath();
581581
Path logsFolderPath = Path.of(userHomePath).resolve(BuiltInFolderNames.LOGS);
582582
if (!Files.exists(logsFolderPath)) {
583583
initializeEmptyFolder(logsFolderPath);

‎dqops/src/main/java/com/dqops/metadata/storage/localfiles/userhome/LocalUserHomeFileStorageServiceImpl.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public LocalUserHomeFileStorageServiceImpl(HomeLocationFindService homeLocationF
5454
UserHomeLockManager userHomeLockManager,
5555
SynchronizationStatusTracker synchronizationStatusTracker,
5656
LocalFileSystemCache localFileSystemCache) {
57-
super(homeLocationFindService.getUserHomePath(), HomeType.USER_HOME, localFileSystemCache);
57+
super(homeLocationFindService.getRootUserHomePath(), HomeType.USER_HOME, localFileSystemCache);
5858
this.userHomeLockManager = userHomeLockManager;
5959
this.synchronizationStatusTracker = synchronizationStatusTracker;
6060
this.localFileSystemCache = localFileSystemCache;

0 commit comments

Comments
 (0)