Update Yaci Store version, fix yaci-store jar download handling, rollback enhancements

.github/workflows/dev-release-cli.yml

@@ -12,7 +12,7 @@ jobs:
   buildAndPush:
     strategy:
       matrix:
-        os: ["ubuntu-20.04", "macos-14"]
+        os: ["ubuntu-22.04", "macos-14"]
     runs-on: ${{ matrix.os }}
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

.github/workflows/release-cli.yml

@@ -10,7 +10,7 @@ jobs:
   buildAndPush:
     strategy:
       matrix:
-        os: ["ubuntu-20.04-self-hosted", "macos-14"]
+        os: ["ubuntu-22.04", "macos-14"]
     runs-on: ${{ matrix.os }}
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -73,7 +73,7 @@ jobs:
     needs: buildAndPush
     strategy:
       matrix:
-        os: [ "ubuntu-20.04-self-hosted", "macos-14" ]
+        os: [ "ubuntu-22.04", "macos-14" ]
     runs-on: ${{ matrix.os }}
     env:
       FORCE_COLOR: 1
@@ -97,7 +97,7 @@ jobs:
       - name: Determine Platform-Specific Folder
         id: platform-folder
         run: |
-          if [[ "${{ matrix.os }}" == "ubuntu-20.04-self-hosted" ]]; then
+          if [[ "${{ matrix.os }}" == "ubuntu-22.04" ]]; then
             echo "npm_folder=yaci-devkit-linux-x64" >> $GITHUB_ENV
           elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
             echo "npm_folder=yaci-devkit-macos-arm64" >> $GITHUB_ENV

.github/workflows/release-viewer.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   npm-publish:
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       FORCE_COLOR: 1
     steps:

applications/cli/Earthfile

@@ -50,7 +50,7 @@ docker-build:
     FROM ubuntu:22.04
     ENV JAVA_HOME=/opt/java/openjdk
     ENV STORE_VERSION=0.1.0
-    ENV STORE_NATIVE_BRANCH=release/2.0.0-beta2
+    ENV STORE_NATIVE_BRANCH=release/2.0.0-beta3
 
     ARG TARGETOS
     ARG TARGETARCH

applications/cli/config/download.properties

@@ -3,9 +3,9 @@ node.version=10.5.0
 ogmios.version=6.13.0
 kupo.version=2.11.0
 
-yaci.store.tag=rel-native-2.0.0-beta2
-yaci.store.version=2.0.0-beta2
-yaci.store.jar.version=2.0.0-beta2
+yaci.store.tag=rel-native-2.0.0-beta3
+yaci.store.version=2.0.0-beta3
+yaci.store.jar.version=2.0.0-beta3
 
 #node.url=
 #ogmios.url=

applications/cli/docker/store-application.properties

@@ -31,6 +31,11 @@ store.account.api-enabled=true
 store.account.balance-aggregation-enabled=true
 store.account.history-cleanup-enabled=false
 
+store.adapot.enabled=true
+store.adapot.api-enabled=true
+store.governance-aggr.enabled=true
+store.governance-aggr.api-enabled=true
+
 store.live.enabled=true
 
 store.epoch.endpoints.epoch.local.enabled=true

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java

@@ -19,6 +19,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.Objects;
 
 import static com.bloxbean.cardano.yacicli.util.ConsoleWriter.*;
@@ -168,6 +169,7 @@ public boolean downloadYaciStoreNative(boolean overwrite) {
         return false;
     }
 
+    //YaciStore Jar is bundled in a zip file.
     public boolean downloadYaciStoreJar(boolean overwrite) {
         downloadJre(overwrite); //Download JRE first (if not already downloaded
 
@@ -190,11 +192,43 @@ public boolean downloadYaciStoreJar(boolean overwrite) {
             }
         }
 
-        var downloadedFile = download("yaci-store", downloadPath, clusterConfig.getYaciStoreBinPath(), "yaci-store.jar");
+        var downloadedFile = download("yaci-store", downloadPath, clusterConfig.getYaciStoreBinPath(), "yaci-store-jar.zip");
         if (downloadedFile != null) {
-            return true;
+            try {
+                var tmpFolder = Paths.get(clusterConfig.getYaciStoreBinPath(), "tmp");
+                extractZip(downloadedFile.toFile().getAbsolutePath(), tmpFolder.toFile().getAbsolutePath());
+
+                File[] files = tmpFolder.toFile().listFiles();
+                if(files != null && files.length > 0) {
+                    File extractedFolder = files[0];
+                    if (extractedFolder.getName().startsWith("yaci-store")) {
+                        File targetFolder = Paths.get(tmpFolder.toFile().getAbsolutePath(), "yaci-store-files").toFile();
+                        if (!extractedFolder.renameTo(targetFolder)) {
+                            writeLn(error("Failed to rename folder " + extractedFolder.getAbsolutePath() + " to " + targetFolder.getAbsolutePath()));
+                            return false; // Abort further operations
+                        }
+                    }
+                }
+
+                //Move the file yaci-store.jar inside yaci-store folder to yaciStoreBinPath. Then remove the tmpFolder
+                Path downloadedYaciStoreJar = Paths.get(tmpFolder.toFile().getAbsolutePath(), "yaci-store-files", "yaci-store.jar");
+
+                if(downloadedYaciStoreJar.toFile().exists()) {
+                    writeLn(info("Copying yaci-store.jar to " + yaciStoreJar.toFile().getAbsolutePath()));
+                    Files.copy(downloadedYaciStoreJar, yaciStoreJar.toFile().toPath(), StandardCopyOption.REPLACE_EXISTING);
+                    writeLn(success("Copied"));
+                } else {
+                    writeLn(error("yaci-store.jar not found in the extracted folder : " + downloadedYaciStoreJar.toFile().getAbsolutePath()));
+                }
+
+                FileUtils.deleteDirectory(tmpFolder.toFile());
+                return true;
+            } catch (IOException e) {
+                e.printStackTrace();
+                writeLn(error("Error extracting yaci-store jar zip: " + e.getMessage()));
+            }
         } else {
-            writeLn(error("Download failed for yaci-store"));
+            writeLn(error("Download failed for yaci-store jar zip"));
         }
 
         return false;
@@ -503,7 +537,7 @@ private String resolveYaciStoreJarDownloadPath() {
             return null;
         }
 
-        String url = YACI_STORE_DOWNLOAD_URL + "/v" + yaciStoreJarVersion + "/yaci-store-all-" + yaciStoreJarVersion +".jar";
+        String url = YACI_STORE_DOWNLOAD_URL + "/v" + yaciStoreJarVersion + "/yaci-store-" + yaciStoreJarVersion +".zip";
         return url;
     }
 

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/general/DownloadCommand.java

@@ -48,7 +48,7 @@ public boolean download(
                 validComponent = true;
             }
 
-            if (componentList.contains("all") || componentList.contains("yaci-store-jar")) {
+            if (componentList.contains("yaci-store-jar")) {
                 downloadService.downloadYaciStoreJar(overwrite);
                 validComponent = true;
             }
@@ -93,7 +93,8 @@ public void downloadAndStart(
             @ShellOption(value = {"--enable-kupomios"}, defaultValue = "false", help= "Enable Ogmios and Kupo") boolean enableKupomios,
             @ShellOption(value = {"--interactive"}, defaultValue="false", help="To start in interactive mode when 'up' command is passed as an arg to yaci-cli") boolean interactive,
             @ShellOption(value = {"--tail"}, defaultValue="false", help="To tail the network when 'up' command is passed as an arg to yaci-cli. Only works in non-interactive mode.") boolean tail,
-            @ShellOption(value = {"--enable-multi-node"}, defaultValue = "false", help="Create multiple local block producing nodes") boolean enableMultiNode
+            @ShellOption(value = {"--enable-multi-node"}, defaultValue = "false", help="Create multiple local block producing nodes") boolean enableMultiNode,
+            @ShellOption(value = {"--stake-ratio-factor"}, defaultValue = "5", help="The stake ratio between the primary node and two peers is only used when multi-node is enabled for rollback testing.") int stakeRatioFactor
     ) {
 
         if (components == null)
@@ -125,7 +126,7 @@ public void downloadAndStart(
         var status = download(componentList.toArray(new String[0]), overwrite);
         if (status) {
             clusterCommands.createCluster(clusterName, port, submitApiPort, slotLength, blockTime, epochLength,
-                    true, true, null, genesisProfile, false, enableMultiNode);
+                    true, true, null, genesisProfile, false, enableMultiNode, stakeRatioFactor);
 
             if (!interactive && tail)
                 clusterCommands.ltail(true, true, true, true, true, true, null, null);

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java

@@ -106,7 +106,8 @@ public void createCluster(@ShellOption(value = {"-n", "--name"}, defaultValue =
                                       help = "Use a pre-defined genesis profile (Options: zero_fee, zero_min_utxo_value, zero_fee_and_min_utxo_value)")
                                   GenesisProfile genesisProfile,
                               @ShellOption(value = {"--generate-new-keys"}, defaultValue = "false", help = "Generate new genesis keys, pool keys instead of default keys") boolean generateNewKeys,
-                              @ShellOption(value = {"--enable-multi-node"}, defaultValue = "false", help="Create multiple local block producing nodes") boolean enableMultiNode
+                              @ShellOption(value = {"--enable-multi-node"}, defaultValue = "false", help="Create multiple local block producing nodes") boolean enableMultiNode,
+                              @ShellOption(value = {"--stake-ratio-factor"}, defaultValue = "5", help="The stake ratio between the primary node and two peers is only used when multi-node is enabled for rollback testing") int stakeRatioFactor
     ) {
 
         try {
@@ -168,6 +169,7 @@ else if (era.equalsIgnoreCase("conway"))
                     .socatPort(socatPort)
                     .prometheusPort(prometheusPort)
                     .localMultiNodeEnabled(enableMultiNode)
+                    .localMultiNodeStakeRatioFactor(stakeRatioFactor)
                     .build();
 
             boolean success = localClusterService.createNodeClusterFolder(clusterName, clusterInfo, overwrite, generateNewKeys, enableMultiNode, (msg) -> writeLn(msg));

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterInfo.java

@@ -44,4 +44,5 @@ public class ClusterInfo {
     private int prometheusPort=12798;
 
     private boolean localMultiNodeEnabled;
+    private int localMultiNodeStakeRatioFactor;
 }

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java

@@ -361,7 +361,7 @@ private void updateGenesis(Path clusterFolder, String clusterName, ClusterInfo c
 
         //Local BP peers for rollback testing
         if (enableMultiNode) {
-            localPeerService.setupNewPoolInfos(genesisConfigCopy, writer);
+            localPeerService.setupNewPoolInfos(genesisConfigCopy, clusterInfo, writer);
         }
 
         String slotLengthStr;

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java

@@ -1,9 +1,13 @@
 package com.bloxbean.cardano.yacicli.localcluster;
 
 import com.bloxbean.cardano.yaci.core.util.OSUtil;
+import com.bloxbean.cardano.yacicli.common.CommandContext;
 import com.bloxbean.cardano.yacicli.localcluster.config.CustomGenesisConfig;
 import com.bloxbean.cardano.yacicli.localcluster.config.GenesisConfig;
+import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStarted;
+import com.bloxbean.cardano.yacicli.localcluster.events.FirstRunDone;
 import com.bloxbean.cardano.yacicli.localcluster.model.RunStatus;
+import com.bloxbean.cardano.yacicli.localcluster.peer.LocalPeerService;
 import com.bloxbean.cardano.yacicli.util.PortUtil;
 import com.bloxbean.cardano.yacicli.util.ProcessUtil;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
@@ -42,6 +46,7 @@ public class ClusterStartService {
     private final ProcessUtil processUtil;
     private final GenesisConfig genesisConfig;
     private final CustomGenesisConfig customGenesisConfig;
+    private final LocalPeerService localPeerService;
 
     private ObjectMapper objectMapper = new ObjectMapper();
     private List<Process> processes = new ArrayList<>();
@@ -67,6 +72,15 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu
             if (clusterInfo.isMasterNode() && firstRun)
                 setupFirstRun(clusterInfo, clusterFolder, writer);
 
+            if (clusterInfo.isLocalMultiNodeEnabled()) {
+                String clusterName = CommandContext.INSTANCE.getProperty(ClusterConfig.CLUSTER_NAME);
+                if (firstRun) {
+                    localPeerService.handleFirstRun(new FirstRunDone(clusterName));
+                }
+
+                localPeerService.handleClusterStarted(new ClusterStarted(clusterName));
+            }
+
             Process nodeProcess = startNode(clusterFolder, clusterInfo, writer);
             if (nodeProcess == null) {
                 writer.accept(error("Node process could not be started."));

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java

@@ -1,6 +1,7 @@
 package com.bloxbean.cardano.yacicli.localcluster.api;
 
 import com.bloxbean.cardano.yacicli.localcluster.ClusterCommands;
+import com.bloxbean.cardano.yacicli.localcluster.ClusterConfig;
 import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo;
 import com.bloxbean.cardano.yacicli.localcluster.ClusterService;
 import com.bloxbean.cardano.yacicli.localcluster.config.ApplicationConfig;
@@ -29,6 +30,10 @@
 import java.util.Map;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
+import com.bloxbean.cardano.yaci.core.protocol.chainsync.messages.Point;
+import com.bloxbean.cardano.yacicli.common.Tuple;
+import java.util.ArrayList;
+import java.util.List;
 
 @RestController
 @RequestMapping(path = "/local-cluster/api/admin")
@@ -246,7 +251,7 @@ public boolean create(@RequestBody DevNetCreateRequest request) {
 
         try {
             clusterCommands.createCluster(DEFAULT_CLUSTER_NAME, 3001, 8090, slotLength, blockTime, epochLength, true,
-                    true, "conway", null, false, request.enableMultiNode());
+                    true, "conway", null, false, request.enableMultiNode(), request.multiNodeStakeRatioFactor);
             return true;
         } catch (Exception e) {
             return false;
@@ -258,14 +263,71 @@ public boolean create(@RequestBody DevNetCreateRequest request) {
         }
     }
 
+    @Operation(summary = "Retrieve the current tip for all nodes (multi-node aware)")
+    @GetMapping("/devnet/tip")
+    public MultiNodeTipResponse getTip() {
+        CommandContext.INSTANCE.setProperty(ClusterConfig.CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+
+        boolean isMultiNodeEnabled = false;
+        try {
+            var clusterInfo = clusterService.getClusterInfo(DEFAULT_CLUSTER_NAME);
+            isMultiNodeEnabled = clusterInfo.isLocalMultiNodeEnabled();
+        } catch (Exception e) {
+            log.error("Error getting cluster info", e);
+        }
+
+        List<NodeTip> tips = new ArrayList<>();
+
+        // Get tip for primary node
+        Tuple<Long, Point> tip1 = clusterUtilService.getTip(msg -> log.debug(msg));
+        if (tip1 != null) {
+            tips.add(new NodeTip("node-1", tip1._1, tip1._2.getSlot(), tip1._2.getHash()));
+        }
+
+        // Get tips for additional nodes if multi-node is enabled
+        if (isMultiNodeEnabled) {
+            Tuple<Long, Point> tip2 = clusterUtilService.getTip(msg -> log.debug(msg), "node-2");
+            if (tip2 != null) {
+                tips.add(new NodeTip("node-2", tip2._1, tip2._2.getSlot(), tip2._2.getHash()));
+            }
+
+            Tuple<Long, Point> tip3 = clusterUtilService.getTip(msg -> log.debug(msg), "node-3");
+            if (tip3 != null) {
+                tips.add(new NodeTip("node-3", tip3._1, tip3._2.getSlot(), tip3._2.getHash()));
+            }
+        }
+
+        return new MultiNodeTipResponse(isMultiNodeEnabled, tips);
+    }
+
     record DevNetCreateRequest(Map<String, String> genesisProperties,
                                @Schema(description = "Create multiple local block producing nodes", defaultValue = "false")
                                boolean enableMultiNode,
+                               @Schema(description = "The stake ratio between the primary node and two peers is only used when multi-node is enabled for rollback testing", defaultValue = "5")
+                               int multiNodeStakeRatioFactor,
                                @Schema(description = "Enable Yaci Store", defaultValue = "false")
                                boolean enableYaciStore,
                                @Schema(description = "Enable Ogmios", defaultValue = "false")
                                boolean enableOgmios,
                                @Schema(description = "Enable Ogmios and Kupo", defaultValue = "false")
                                boolean enableKupomios) {
     }
+
+    record NodeTip(
+            @Schema(description = "Node name")
+            String nodeName,
+            @Schema(description = "Current block number")
+            long blockNumber,
+            @Schema(description = "Current slot number")
+            long slot,
+            @Schema(description = "Current block hash")
+            String blockHash) {
+    }
+
+    record MultiNodeTipResponse(
+            @Schema(description = "Whether multi-node mode is enabled")
+            boolean multiNodeEnabled,
+            @Schema(description = "List of tips for all nodes")
+            List<NodeTip> tips) {
+    }
 }