Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -187,4 +187,10 @@ public interface GorDriverConfig extends Config {
@Key(GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL)
@DefaultValue("")
String managedLinkDataFilesURL();

String GOR_DRIVER_LINK_INFER_REPLACE = "GOR_DRIVER_LINK_INFER_REPLACE";
@Documentation("Replacement patterns when inferring link file paths (<match pattern>[;<replace string>]).")
@Key(GOR_DRIVER_LINK_INFER_REPLACE)
@DefaultValue("")
String linkeInferReplace();
}
17 changes: 15 additions & 2 deletions model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public abstract class LinkFile {

public static final int LINK_FILE_MAX_SIZE = 10000;

private static final boolean USE_LINK_CACHE = Boolean.parseBoolean(System.getProperty("gor.driver.cache.link", "true"));
private static final boolean USE_LINK_CACHE = Boolean.parseBoolean(System.getProperty("gor.driver.link.cache", "true"));
private static final Cache<StreamSource, String> linkCache = Caffeine.newBuilder()
.maximumSize(10000)
.expireAfterWrite(2, TimeUnit.HOURS).build();
Expand Down Expand Up @@ -101,11 +101,21 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri

var linkPath = linkSource.getSourceReference().getUrl();

// Remove common the root if set.
var pathReplacements = System.getenv("GOR_DRIVER_LINK_INFER_REPLACE");
if (!Strings.isNullOrEmpty(pathReplacements)) {
var parts = pathReplacements.split(";", 2);
linkPath = linkPath.replaceAll(parts[0], parts.length > 1 ? parts[1] : "");
}

// Adjust link path so it suitable as part of data file path.
if (PathUtils.isAbsolutePath(linkPath)) {
throw new IllegalArgumentException("Link file path is absolute. Can not infer data file name: " + linkSource.getFullPath());
throw new IllegalArgumentException("Link file path is absolute and gor.driver.link.common.root is not set. Can not infer data file name: " + linkSource.getFullPath());
}

var dataFileRootPath = "";

// Get root from the link file
var link = linkSource.exists()
? load(linkSource).appendMeta(linkFileMeta)
: create(linkSource, linkFileMeta);
Expand All @@ -114,6 +124,7 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
dataFileRootPath = linkDataFileRootPath;
}

// Get root from global const
if (Strings.isNullOrEmpty(dataFileRootPath)) {
dataFileRootPath = System.getenv(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL);
}
Expand All @@ -122,6 +133,7 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
throw new IllegalArgumentException("Link file data root path is not set. Can not infer data file name from link file: " + linkSource.getFullPath());
}

// Create file name
String randomString = RandomStringUtils.random(8, true, true);
var linkPathSplit = linkPath.indexOf('.');
if (linkPathSplit > 0) {
Expand All @@ -135,6 +147,7 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri

linkPath = linkPath.replaceAll("\\.link$", "");

// Insert project
var project = linkSource.getSourceReference().getCommonRoot() != null
? PathUtils.getFileName(linkSource.getSourceReference().getCommonRoot()) : "";
if (!Strings.isNullOrEmpty(project)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.rules.TemporaryFolder;

import java.io.ByteArrayInputStream;
Expand All @@ -26,6 +27,9 @@ public class LinkFileTest {
@Rule
public TemporaryFolder workDir = new TemporaryFolder();

@Rule
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();

@Rule
public final EnvironmentVariables environmentVariables
= new EnvironmentVariables();
Expand Down Expand Up @@ -237,4 +241,28 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam_ExistingFile() throw
assertNotNull(result);
assertTrue(result.matches((paramroot + "/x\\..*\\.gor").replace("/", "\\/")));
}

@Test
public void testInferDataFileNameFromLinkFile_PathReplace() throws Exception {
String root = "/managed/root";
environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL, root);
environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_INFER_REPLACE, "wont;will");

String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource("wont/x.gor.link"), null);

assertNotNull(result);
assertTrue(result.matches((root + "/will/x\\..*\\.gor").replace("/", "\\/")));
}

@Test
public void testInferDataFileNameFromLinkFile_AbsolutePathReplace() throws Exception {
String root = "/managed/root";
environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL, root);
environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_INFER_REPLACE, "\\/abs\\/");

String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource("/abs/path/x.gor.link"), null);

assertNotNull(result);
assertTrue(result.matches((root + "/path/x\\..*\\.gor").replace("/", "\\/")));
}
}
Loading