From 8b0b86827b3d54945c7021cb7cd4c3f5c7f063ff Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Thu, 11 Dec 2025 01:42:05 +0000 Subject: [PATCH 1/2] fix(ENGKNOW-2961): Feature flag, and refactor. --- .../src/test/java/gorsat/UTestGorWrite.java | 19 ++++---- .../gorpipe/gor/driver/linkfile/LinkFile.java | 46 ++++++++++++------- .../gor/driver/linkfile/LinkFileMeta.java | 40 ++++++++++------ .../gor/driver/linkfile/LinkFileV0.java | 16 +++---- .../gor/driver/linkfile/LinkFileV1.java | 26 +++++------ .../gor/driver/linkfile/LinkFileTest.java | 42 ++++++++++++++--- 6 files changed, 124 insertions(+), 65 deletions(-) diff --git a/gortools/src/test/java/gorsat/UTestGorWrite.java b/gortools/src/test/java/gorsat/UTestGorWrite.java index 3e78f21b..24ece8b6 100644 --- a/gortools/src/test/java/gorsat/UTestGorWrite.java +++ b/gortools/src/test/java/gorsat/UTestGorWrite.java @@ -30,6 +30,7 @@ import org.gorpipe.gor.driver.GorDriverConfig; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.gor.driver.linkfile.LinkFileMeta; +import org.gorpipe.gor.driver.linkfile.LinkFileV1; import org.gorpipe.gor.driver.meta.DataType; import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; import org.gorpipe.gor.model.BaseMeta; @@ -80,7 +81,7 @@ public void setupTest() throws IOException { tempRootPath = tempRoot.getRoot().toPath(); var meta = new LinkFileMeta(); - meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1"); + meta.loadAndMergeMeta(LinkFileV1.getDefaultMetaContent()); meta.setProperty(BaseMeta.HEADER_SERIAL_KEY, "1"); defaultV1LinkFileHeader = meta.formatHeader(); } @@ -158,19 +159,21 @@ public void testWritePathWithExistingBadLinkFile() throws IOException { Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor")); Files.writeString(link, ""); TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); - var linkUrl = LinkFile.load(new FileSource(link)).getLatestEntryUrl(); - Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkUrl); + var linkFile = LinkFile.load(new FileSource(link)); + Assert.assertEquals("1", linkFile.getMeta().getVersion()); + Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl()); } @Test public void testWritePathWithExistingBadVersionedLinkFile() throws IOException { - Path p = Paths.get("../tests/data/gor/dbsnp_test.gor"); - Files.copy(p, workDirPath.resolve("dbsnp.gor")); - Files.writeString(workDirPath.resolve("dbsnp3.gor.link"), ""); + Path link = workDirPath.resolve("dbsnp3.gor.link"); + Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor")); + Files.writeString(link, "## VERSION = 1"); TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); - Assert.assertTrue(Files.readString(workDirPath.resolve("dbsnp3.gor.link")).startsWith( - defaultV1LinkFileHeader + workDirPath.resolve("dbsnp2.gor") + "\t")); + var linkFile = LinkFile.load(new FileSource(link)); + Assert.assertEquals("1", linkFile.getMeta().getVersion()); + Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl()); } @Test diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java index 8efe2413..876d3bae 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java @@ -66,17 +66,39 @@ public abstract class LinkFile { public static LinkFile load(StreamSource source) throws IOException { var content = loadContentFromSource(source); - return create(source, content); + var meta = LinkFileMeta.createOrLoad(content, null, false); + return create(source, meta, content); } public static LinkFile create(StreamSource source, String content) { - var meta = LinkFileMeta.createAndLoad(content); + var meta = LinkFileMeta.createOrLoad(content, null, true); + return create(source, meta, content); + } - if ("0".equals(meta.getVersion())) { - return new LinkFileV0(source, meta, content); - } else { - return new LinkFileV1(source, meta, content); - } + public static LinkFile create(StreamSource source, LinkFileMeta meta, String content) { + return switch (meta.getVersion()) { + case "0" -> new LinkFileV0(source, meta, content); + case "1" -> new LinkFileV1(source, meta, content); + default -> throw new GorResourceException("Unsupported link file version: " + meta.getVersion(), source.getFullPath()); + }; + } + + public static LinkFile loadV0(StreamSource source) throws IOException { + var content = loadContentFromSource(source); + return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content); + } + + public static LinkFile loadV1(StreamSource source) throws IOException { + var content = loadContentFromSource(source); + return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content); + } + + public static LinkFile createV0(StreamSource source, String content) throws IOException { + return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content); + } + + public static LinkFile createV1(StreamSource source, String content) throws IOException { + return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content); } public static String validateAndUpdateLinkFileName(String linkFilePath) { @@ -148,15 +170,6 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri protected final LinkFileMeta meta; protected final List entries; // Entries sorted by time (oldest first) - /** - * Create a new link file from source and content. - * - * @param source the source to create the link file from - * @param content the content of the link file, can be empty or null to create an empty link file. - */ - protected LinkFile(StreamSource source, String content) { - this(source, LinkFileMeta.createAndLoad(content), content); - } protected LinkFile(StreamSource source, LinkFileMeta meta, String content) { this.source = source; @@ -321,6 +334,7 @@ private void save(OutputStream os, long timestamp) { .filter(entry -> entry.timestamp() <= 0 || currentTimestamp - entry.timestamp() <= getEntriesAgeMax()) .forEach(entry -> content.append(entry.format()).append("\n")); } + try { os.write(content.toString().getBytes()); } catch (IOException e) { diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java index 1dc7df45..66ef262e 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java @@ -5,6 +5,8 @@ import org.gorpipe.gor.model.FileReader; import org.gorpipe.util.Strings; +import java.util.stream.Collectors; + public class LinkFileMeta extends BaseMeta { // Max number of entries to keep track of in the link file. @@ -16,24 +18,39 @@ public class LinkFileMeta extends BaseMeta { // Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false). public static final String HEADER_CONTENT_LIFECYCLE_MANAGED_KEY = "CONTENT_LIFECYCLE_MANAGED"; - public static final String[] DEFAULT_TABLE_HEADER = new String[] {"File", "Timestamp", "MD5", "Serial", "Info"}; + private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1"); public static final int DEFAULT_ENTRIES_COUNT_MAX = 100; public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE; - public static LinkFileMeta createAndLoad(String metaContent) { + /** + * Create or load link file meta from content. + * @param content + * @param version version if known, otherwise null. Only used if content is null or empty. + * @param isNew true if creating new link file meta, false if loading existing. + * @return + */ + public static LinkFileMeta createOrLoad(String content, String version, boolean isNew) { + var metaContent = !Strings.isNullOrEmpty(content) ? content.lines().filter(line -> line.startsWith("#")).collect(Collectors.joining("\n")) : ""; LinkFileMeta meta = new LinkFileMeta(); - if (Strings.isNullOrEmpty(metaContent)) { - meta.loadAndMergeMeta(getDefaultMetaContent()); - } else { - meta.loadAndMergeMeta(metaContent); + if (Strings.isNullOrEmpty(metaContent) ) { + // No meta, determine version to use + if (Strings.isNullOrEmpty(version)) { + version = Strings.isNullOrEmpty(content) || isNew ? DEFAULT_VERSION : LinkFileV0.VERSION; + } + + metaContent = switch(version) { + case "0" -> LinkFileV0.getDefaultMetaContent(); + case "1" -> LinkFileV1.getDefaultMetaContent(); + default -> throw new IllegalArgumentException("Unsupported link file version: " + meta.getVersion()); + }; } + meta.loadAndMergeMeta(metaContent); return meta; } public LinkFileMeta() { super(); - setFileHeader(DEFAULT_TABLE_HEADER); saveHeaderLine = true; } @@ -63,13 +80,8 @@ public void setEntriesAgeMax(int entriesAgeMax) { @Override public String getVersion() { - return getProperty(HEADER_VERSION_KEY, "0"); + return getProperty(HEADER_VERSION_KEY, DEFAULT_VERSION); } - public static String getDefaultMetaContent() { - return String.format(""" - ## SERIAL = 0 - ## VERSION = 1 - """); - } + } diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java index ff277725..fb495a87 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java @@ -11,14 +11,8 @@ */ public class LinkFileV0 extends LinkFile { - /** - * Load from a source, if it exists, otherwise create an empty link file. - * - * @param source the source to load from - */ - public LinkFileV0(StreamSource source) throws IOException { - super(source, loadContentFromSource(source)); - } + public static final String VERSION = "0"; + protected LinkFileV0(StreamSource source, LinkFileMeta meta, String content) { super(source, meta, content); @@ -40,4 +34,10 @@ public LinkFile appendEntry(String link, String md5, String info, FileReader rea entries.add(new LinkFileEntryV0(link)); return this; } + + public static String getDefaultMetaContent() { + return String.format(""" + ## VERSION = 0 + """); + } } diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java index 271a83ec..7bc162e6 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java @@ -12,19 +12,12 @@ */ public class LinkFileV1 extends LinkFile { + public static final String VERSION = "1"; + public static final String DEFAULT_TABLE_HEADER = "#File\tTimestamp\tMD5\tSerial\tInfo"; + private static boolean allowOverwriteOfTargets = Boolean.parseBoolean(System.getProperty("gor.link.versioned.allow.overwrite", "false")); - /** - * Load from a source, if it exists, otherwise create an empty link file. - * - * @param source the source to load from - */ - public LinkFileV1(StreamSource source) throws IOException { - super(source, loadContentFromSource(source)); - checkDefaultMeta(); - } - protected LinkFileV1(StreamSource source, LinkFileMeta meta, String content) { super(source, meta, content); checkDefaultMeta(); @@ -79,9 +72,16 @@ private boolean canReuseEntryWithSameUrl(LinkFileEntry oldEntry, LinkFileEntry n } private void checkDefaultMeta() { - if (!meta.containsProperty(BaseMeta.HEADER_VERSION_KEY)) { - getMeta().loadAndMergeMeta(LinkFileMeta.getDefaultMetaContent()); - meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1"); + if (!meta.getVersion().equals(VERSION)) { + meta.loadAndMergeMeta(getDefaultMetaContent()); } } + + public static String getDefaultMetaContent() { + return String.format(""" + ## SERIAL = 0 + ## VERSION = %s + %s + """, VERSION, DEFAULT_TABLE_HEADER); + } } diff --git a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java index 9156417e..56da8203 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java +++ b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java @@ -5,9 +5,11 @@ import org.gorpipe.gor.driver.providers.stream.sources.StreamSource; import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.contrib.java.lang.system.EnvironmentVariables; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.rules.TemporaryFolder; import java.io.ByteArrayInputStream; @@ -30,6 +32,10 @@ public class LinkFileTest { public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); + @Rule + public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); + + private StreamSource mockSource; private final String v1LinkFileContent = """ ## SERIAL = 1 @@ -55,10 +61,34 @@ public void setUp() { public void testCreateLinkFile() { LinkFile linkFile = LinkFile.create(mockSource, v1LinkFileContent); assertNotNull(linkFile); + assertTrue(linkFile instanceof LinkFileV1); + assertEquals("1", linkFile.getMeta().getVersion()); assertEquals(2, linkFile.getEntries().size()); assertEquals(100, linkFile.getEntriesCountMax()); } + @Test + public void testCreateLinkFileSimple() { + LinkFile linkFile = LinkFile.create(mockSource, "test.gorz"); + assertNotNull(linkFile); + assertTrue(linkFile instanceof LinkFileV1); + assertEquals("1", linkFile.getMeta().getVersion()); + assertEquals(1, linkFile.getEntries().size()); + assertEquals(100, linkFile.getEntriesCountMax()); + } + + @Ignore("Fiddly test depending on system properties, ignore for now. Can run in isolation to verify.") + @Test + public void testCreateLinkFileSimpleWithDefault0() { + System.setProperty("gor.driver.link.default.version", "0"); + LinkFile linkFile = LinkFile.create(mockSource, "test.gorz"); + assertNotNull(linkFile); + assertTrue(linkFile instanceof LinkFileV0); + assertEquals("0", linkFile.getMeta().getVersion()); + assertEquals(1, linkFile.getEntries().size()); + assertEquals(100, linkFile.getEntriesCountMax()); + } + @Test public void testLoadLinkFile() throws IOException { when(mockSource.exists()).thenReturn(true); @@ -101,7 +131,7 @@ public void testGetTimedPath() { @Test public void testSaveNewV1LinkFile() throws IOException { var linkPath = workPath.resolve("test.link"); - LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString())); + LinkFile linkFile = LinkFile.createV1(new FileSource(linkPath.toString()), ""); linkFile.appendEntry(simpleFile, "NEWMD5SUM"); linkFile.save(); String savedContent = Files.readString(linkPath); @@ -112,7 +142,7 @@ public void testSaveNewV1LinkFile() throws IOException { @Test public void testSaveNewV0LinkFile() throws IOException { var linkPath = workPath.resolve("test.link"); - LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString())); + LinkFile linkFile = LinkFile.createV0(new FileSource(linkPath.toString()), ""); linkFile.appendEntry(simpleFile, "NEWMD5SUM"); linkFile.save(); String savedContent = Files.readString(linkPath); @@ -135,7 +165,7 @@ public void testSaveLinkFileV1ToV1() throws IOException { public void testSaveLinkFileV0ToV0() throws IOException { var linkPath = workPath.resolve("test.link"); Files.writeString(linkPath, "a/b/c.gorz"); - LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString())); + LinkFile linkFile = LinkFile.load(new FileSource(linkPath.toString())); linkFile.appendEntry(simpleFile, "NEWMD5SUM"); linkFile.save(); String savedContent = Files.readString(linkPath); @@ -146,11 +176,12 @@ public void testSaveLinkFileV0ToV0() throws IOException { public void testSaveLinkFileV0ToV1() throws IOException { var linkPath = workPath.resolve("test.link"); Files.writeString(linkPath, "a/b/c.gorz"); - LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString())); + LinkFile linkFile = LinkFile.loadV1(new FileSource(linkPath.toString())); linkFile.appendEntry(simpleFile, "NEWMD5SUM"); linkFile.save(); String savedContent = Files.readString(linkPath); assertTrue(savedContent.contains("## VERSION = 1")); + assertEquals(2, linkFile.getEntries().size()); assertTrue(savedContent.contains(simpleFile)); } @@ -158,14 +189,13 @@ public void testSaveLinkFileV0ToV1() throws IOException { public void testSaveLinkFileV1ToV0() throws IOException { var linkPath = workPath.resolve("test.link"); Files.writeString(linkPath, v1LinkFileContent); - LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString())); + LinkFile linkFile = LinkFile.loadV0(new FileSource(linkPath.toString())); linkFile.appendEntry(simpleFile, "NEWMD5SUM"); linkFile.save(); String savedContent = Files.readString(linkPath); assertEquals(simpleFile, savedContent.trim()); } - @Test(expected = IllegalArgumentException.class) public void testInferDataFileNameFromLinkFile_NullOrEmptyPath() throws Exception { LinkFile.inferDataFileNameFromLinkFile(new FileSource(""), null); From 82ba6495f6f67a6a3b77a502bb36b4001388a412 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Thu, 11 Dec 2025 12:18:00 +0000 Subject: [PATCH 2/2] fix(ENGKNOW-2961): Feature flag, and refactor. --- .../java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java index 85408753..193e1e8e 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java +++ b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java @@ -35,10 +35,6 @@ public class LinkFileTest { public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); - @Rule - public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); - - private StreamSource mockSource; private final String v1LinkFileContent = """ ## SERIAL = 1