Skip to content

fix(ENGKNOW-2961): Feature flag, and refactor. #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gmagnu merged 1 commit into from
Dec 11, 2025
Merged

fix(ENGKNOW-2961): Feature flag, and refactor. #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions gortools/src/test/java/gorsat/UTestGorWrite.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.gorpipe.gor.driver.GorDriverConfig;
import org.gorpipe.gor.driver.linkfile.LinkFile;
import org.gorpipe.gor.driver.linkfile.LinkFileMeta;
import org.gorpipe.gor.driver.linkfile.LinkFileV1;
import org.gorpipe.gor.driver.meta.DataType;
import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
import org.gorpipe.gor.model.BaseMeta;
Expand Down Expand Up @@ -80,7 +81,7 @@ public void setupTest() throws IOException {
tempRootPath = tempRoot.getRoot().toPath();

var meta = new LinkFileMeta();
meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1");
meta.loadAndMergeMeta(LinkFileV1.getDefaultMetaContent());
meta.setProperty(BaseMeta.HEADER_SERIAL_KEY, "1");
defaultV1LinkFileHeader = meta.formatHeader();
}
Expand Down Expand Up @@ -158,19 +159,21 @@ public void testWritePathWithExistingBadLinkFile() throws IOException {
Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor"));
Files.writeString(link, "");
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
var linkUrl = LinkFile.load(new FileSource(link)).getLatestEntryUrl();
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkUrl);
var linkFile = LinkFile.load(new FileSource(link));
Assert.assertEquals("1", linkFile.getMeta().getVersion());
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl());
}

@Test
public void testWritePathWithExistingBadVersionedLinkFile() throws IOException {
Path p = Paths.get("../tests/data/gor/dbsnp_test.gor");
Files.copy(p, workDirPath.resolve("dbsnp.gor"));
Files.writeString(workDirPath.resolve("dbsnp3.gor.link"), "");
Path link = workDirPath.resolve("dbsnp3.gor.link");
Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor"));
Files.writeString(link, "## VERSION = 1");
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());

Assert.assertTrue(Files.readString(workDirPath.resolve("dbsnp3.gor.link")).startsWith(
defaultV1LinkFileHeader + workDirPath.resolve("dbsnp2.gor") + "\t"));
var linkFile = LinkFile.load(new FileSource(link));
Assert.assertEquals("1", linkFile.getMeta().getVersion());
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl());
}

@Test
Expand Down
46 changes: 30 additions & 16 deletions model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,39 @@ public abstract class LinkFile {

public static LinkFile load(StreamSource source) throws IOException {
var content = loadContentFromSource(source);
return create(source, content);
var meta = LinkFileMeta.createOrLoad(content, null, false);
return create(source, meta, content);
}

public static LinkFile create(StreamSource source, String content) {
var meta = LinkFileMeta.createAndLoad(content);
var meta = LinkFileMeta.createOrLoad(content, null, true);
return create(source, meta, content);
}

if ("0".equals(meta.getVersion())) {
return new LinkFileV0(source, meta, content);
} else {
return new LinkFileV1(source, meta, content);
}
public static LinkFile create(StreamSource source, LinkFileMeta meta, String content) {
return switch (meta.getVersion()) {
case "0" -> new LinkFileV0(source, meta, content);
case "1" -> new LinkFileV1(source, meta, content);
default -> throw new GorResourceException("Unsupported link file version: " + meta.getVersion(), source.getFullPath());
};
}

public static LinkFile loadV0(StreamSource source) throws IOException {
var content = loadContentFromSource(source);
return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content);
}

public static LinkFile loadV1(StreamSource source) throws IOException {
var content = loadContentFromSource(source);
return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content);
}

public static LinkFile createV0(StreamSource source, String content) throws IOException {
return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content);
}

public static LinkFile createV1(StreamSource source, String content) throws IOException {
return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content);
}

public static String validateAndUpdateLinkFileName(String linkFilePath) {
Expand Down Expand Up @@ -148,15 +170,6 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
protected final LinkFileMeta meta;
protected final List<LinkFileEntry> entries; // Entries sorted by time (oldest first)

/**
* Create a new link file from source and content.
*
* @param source the source to create the link file from
* @param content the content of the link file, can be empty or null to create an empty link file.
*/
protected LinkFile(StreamSource source, String content) {
this(source, LinkFileMeta.createAndLoad(content), content);
}

protected LinkFile(StreamSource source, LinkFileMeta meta, String content) {
this.source = source;
Expand Down Expand Up @@ -321,6 +334,7 @@ private void save(OutputStream os, long timestamp) {
.filter(entry -> entry.timestamp() <= 0 || currentTimestamp - entry.timestamp() <= getEntriesAgeMax())
.forEach(entry -> content.append(entry.format()).append("\n"));
}

try {
os.write(content.toString().getBytes());
} catch (IOException e) {
Expand Down
40 changes: 26 additions & 14 deletions model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import org.gorpipe.gor.model.FileReader;
import org.gorpipe.util.Strings;

import java.util.stream.Collectors;

public class LinkFileMeta extends BaseMeta {

// Max number of entries to keep track of in the link file.
Expand All @@ -16,24 +18,39 @@ public class LinkFileMeta extends BaseMeta {
// Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false).
public static final String HEADER_CONTENT_LIFECYCLE_MANAGED_KEY = "CONTENT_LIFECYCLE_MANAGED";

public static final String[] DEFAULT_TABLE_HEADER = new String[] {"File", "Timestamp", "MD5", "Serial", "Info"};
private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1");

public static final int DEFAULT_ENTRIES_COUNT_MAX = 100;
public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE;

public static LinkFileMeta createAndLoad(String metaContent) {
/**
* Create or load link file meta from content.
* @param content
* @param version version if known, otherwise null. Only used if content is null or empty.
* @param isNew true if creating new link file meta, false if loading existing.
* @return
*/
public static LinkFileMeta createOrLoad(String content, String version, boolean isNew) {
var metaContent = !Strings.isNullOrEmpty(content) ? content.lines().filter(line -> line.startsWith("#")).collect(Collectors.joining("\n")) : "";
LinkFileMeta meta = new LinkFileMeta();
if (Strings.isNullOrEmpty(metaContent)) {
meta.loadAndMergeMeta(getDefaultMetaContent());
} else {
meta.loadAndMergeMeta(metaContent);
if (Strings.isNullOrEmpty(metaContent) ) {
// No meta, determine version to use
if (Strings.isNullOrEmpty(version)) {
version = Strings.isNullOrEmpty(content) || isNew ? DEFAULT_VERSION : LinkFileV0.VERSION;
}

metaContent = switch(version) {
case "0" -> LinkFileV0.getDefaultMetaContent();
case "1" -> LinkFileV1.getDefaultMetaContent();
default -> throw new IllegalArgumentException("Unsupported link file version: " + meta.getVersion());
};
}
meta.loadAndMergeMeta(metaContent);
return meta;
}

public LinkFileMeta() {
super();
setFileHeader(DEFAULT_TABLE_HEADER);
saveHeaderLine = true;
}

Expand Down Expand Up @@ -63,13 +80,8 @@ public void setEntriesAgeMax(int entriesAgeMax) {

@Override
public String getVersion() {
return getProperty(HEADER_VERSION_KEY, "0");
return getProperty(HEADER_VERSION_KEY, DEFAULT_VERSION);
}

public static String getDefaultMetaContent() {
return String.format("""
## SERIAL = 0
## VERSION = 1
""");
}

}
16 changes: 8 additions & 8 deletions model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,8 @@
*/
public class LinkFileV0 extends LinkFile {

/**
* Load from a source, if it exists, otherwise create an empty link file.
*
* @param source the source to load from
*/
public LinkFileV0(StreamSource source) throws IOException {
super(source, loadContentFromSource(source));
}
public static final String VERSION = "0";


protected LinkFileV0(StreamSource source, LinkFileMeta meta, String content) {
super(source, meta, content);
Expand All @@ -40,4 +34,10 @@ public LinkFile appendEntry(String link, String md5, String info, FileReader rea
entries.add(new LinkFileEntryV0(link));
return this;
}

public static String getDefaultMetaContent() {
return String.format("""
## VERSION = 0
""");
}
}
26 changes: 13 additions & 13 deletions model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,12 @@
*/
public class LinkFileV1 extends LinkFile {

public static final String VERSION = "1";
public static final String DEFAULT_TABLE_HEADER = "#File\tTimestamp\tMD5\tSerial\tInfo";

private static boolean allowOverwriteOfTargets
= Boolean.parseBoolean(System.getProperty("gor.link.versioned.allow.overwrite", "false"));

/**
* Load from a source, if it exists, otherwise create an empty link file.
*
* @param source the source to load from
*/
public LinkFileV1(StreamSource source) throws IOException {
super(source, loadContentFromSource(source));
checkDefaultMeta();
}

protected LinkFileV1(StreamSource source, LinkFileMeta meta, String content) {
super(source, meta, content);
checkDefaultMeta();
Expand Down Expand Up @@ -79,9 +72,16 @@ private boolean canReuseEntryWithSameUrl(LinkFileEntry oldEntry, LinkFileEntry n
}

private void checkDefaultMeta() {
if (!meta.containsProperty(BaseMeta.HEADER_VERSION_KEY)) {
getMeta().loadAndMergeMeta(LinkFileMeta.getDefaultMetaContent());
meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1");
if (!meta.getVersion().equals(VERSION)) {
meta.loadAndMergeMeta(getDefaultMetaContent());
}
}

public static String getDefaultMetaContent() {
return String.format("""
## SERIAL = 0
## VERSION = %s
%s
""", VERSION, DEFAULT_TABLE_HEADER);
}
}
42 changes: 36 additions & 6 deletions model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
import org.gorpipe.gor.driver.providers.stream.sources.StreamSource;
import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.rules.TemporaryFolder;

import java.io.ByteArrayInputStream;
Expand All @@ -30,6 +32,10 @@ public class LinkFileTest {
public final EnvironmentVariables environmentVariables
= new EnvironmentVariables();

@Rule
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();


private StreamSource mockSource;
private final String v1LinkFileContent = """
## SERIAL = 1
Expand All @@ -55,10 +61,34 @@ public void setUp() {
public void testCreateLinkFile() {
LinkFile linkFile = LinkFile.create(mockSource, v1LinkFileContent);
assertNotNull(linkFile);
assertTrue(linkFile instanceof LinkFileV1);
assertEquals("1", linkFile.getMeta().getVersion());
assertEquals(2, linkFile.getEntries().size());
assertEquals(100, linkFile.getEntriesCountMax());
}

@Test
public void testCreateLinkFileSimple() {
LinkFile linkFile = LinkFile.create(mockSource, "test.gorz");
assertNotNull(linkFile);
assertTrue(linkFile instanceof LinkFileV1);
assertEquals("1", linkFile.getMeta().getVersion());
assertEquals(1, linkFile.getEntries().size());
assertEquals(100, linkFile.getEntriesCountMax());
}

@Ignore("Fiddly test depending on system properties, ignore for now. Can run in isolation to verify.")
@Test
public void testCreateLinkFileSimpleWithDefault0() {
System.setProperty("gor.driver.link.default.version", "0");
LinkFile linkFile = LinkFile.create(mockSource, "test.gorz");
assertNotNull(linkFile);
assertTrue(linkFile instanceof LinkFileV0);
assertEquals("0", linkFile.getMeta().getVersion());
assertEquals(1, linkFile.getEntries().size());
assertEquals(100, linkFile.getEntriesCountMax());
}

@Test
public void testLoadLinkFile() throws IOException {
when(mockSource.exists()).thenReturn(true);
Expand Down Expand Up @@ -101,7 +131,7 @@ public void testGetTimedPath() {
@Test
public void testSaveNewV1LinkFile() throws IOException {
var linkPath = workPath.resolve("test.link");
LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString()));
LinkFile linkFile = LinkFile.createV1(new FileSource(linkPath.toString()), "");
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
linkFile.save();
String savedContent = Files.readString(linkPath);
Expand All @@ -112,7 +142,7 @@ public void testSaveNewV1LinkFile() throws IOException {
@Test
public void testSaveNewV0LinkFile() throws IOException {
var linkPath = workPath.resolve("test.link");
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
LinkFile linkFile = LinkFile.createV0(new FileSource(linkPath.toString()), "");
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
linkFile.save();
String savedContent = Files.readString(linkPath);
Expand All @@ -135,7 +165,7 @@ public void testSaveLinkFileV1ToV1() throws IOException {
public void testSaveLinkFileV0ToV0() throws IOException {
var linkPath = workPath.resolve("test.link");
Files.writeString(linkPath, "a/b/c.gorz");
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
LinkFile linkFile = LinkFile.load(new FileSource(linkPath.toString()));
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
linkFile.save();
String savedContent = Files.readString(linkPath);
Expand All @@ -146,26 +176,26 @@ public void testSaveLinkFileV0ToV0() throws IOException {
public void testSaveLinkFileV0ToV1() throws IOException {
var linkPath = workPath.resolve("test.link");
Files.writeString(linkPath, "a/b/c.gorz");
LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString()));
LinkFile linkFile = LinkFile.loadV1(new FileSource(linkPath.toString()));
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
linkFile.save();
String savedContent = Files.readString(linkPath);
assertTrue(savedContent.contains("## VERSION = 1"));
assertEquals(2, linkFile.getEntries().size());
assertTrue(savedContent.contains(simpleFile));
}

@Test
public void testSaveLinkFileV1ToV0() throws IOException {
var linkPath = workPath.resolve("test.link");
Files.writeString(linkPath, v1LinkFileContent);
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
LinkFile linkFile = LinkFile.loadV0(new FileSource(linkPath.toString()));
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
linkFile.save();
String savedContent = Files.readString(linkPath);
assertEquals(simpleFile, savedContent.trim());
}


@Test(expected = IllegalArgumentException.class)
public void testInferDataFileNameFromLinkFile_NullOrEmptyPath() throws Exception {
LinkFile.inferDataFileNameFromLinkFile(new FileSource(""), null);
Expand Down
Loading