Integration test using template

ice-rest-catalog/pom.xml

@@ -16,7 +16,7 @@
     <sqlite.version>3.46.1.0</sqlite.version>
     <jetcd.version>0.8.5</jetcd.version>
     <grpc.version>1.70.0</grpc.version>
-    <testcontainers.version>1.20.2</testcontainers.version>
+    <testcontainers.version>1.21.4</testcontainers.version>
   </properties>
 
   <dependencies>
@@ -310,6 +310,12 @@
       <artifactId>jackson-annotations</artifactId>
       <version>${jackson.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <version>${jackson.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.etcd</groupId>
       <artifactId>jetcd-core</artifactId>

ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java

@@ -201,7 +201,7 @@ void performMaintenance(
     }
   }
 
-  private static Server createServer(
+  static Server createServer(
       String host, int port, Catalog catalog, Config config, Map<String, String> icebergConfig) {
     var s = createBaseServer(catalog, config, icebergConfig, true);
     ServerConnector connector = new ServerConnector(s);

ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.altinity.ice.rest.catalog;
+
+import static com.altinity.ice.rest.catalog.Main.createServer;
+
+import com.altinity.ice.rest.catalog.internal.config.Config;
+import java.io.File;
+import java.net.URI;
+import java.nio.file.Files;
+import java.util.Map;
+import org.apache.iceberg.catalog.Catalog;
+import org.eclipse.jetty.server.Server;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+
+/**
+ * Base class for REST catalog integration tests. Provides common setup and teardown for minio, REST
+ * catalog server, and REST client.
+ */
+public abstract class RESTCatalogTestBase {
+
+  protected static final Logger logger = LoggerFactory.getLogger(RESTCatalogTestBase.class);
+  protected Server server;
+
+  @SuppressWarnings("rawtypes")
+  protected final GenericContainer minio =
+      new GenericContainer("minio/minio:latest")
+          .withExposedPorts(9000)
+          .withEnv("MINIO_ACCESS_KEY", "minioadmin")
+          .withEnv("MINIO_SECRET_KEY", "minioadmin")
+          .withCommand("server", "/data");
+
+  @BeforeClass
+  public void setUp() throws Exception {
+    // Start minio container
+    minio.start();
+
+    // Configure S3 properties for minio
+    String minioEndpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000);
+
+    // Create S3 client to create bucket
+    S3Client s3Client =
+        S3Client.builder()
+            .endpointOverride(URI.create(minioEndpoint))
+            .region(Region.US_EAST_1)
+            .credentialsProvider(
+                StaticCredentialsProvider.create(
+                    AwsBasicCredentials.create("minioadmin", "minioadmin")))
+            .forcePathStyle(true)
+            .build();
+
+    // Create the test bucket
+    try {
+      s3Client.createBucket(CreateBucketRequest.builder().bucket("test-bucket").build());
+      logger.info("Created test-bucket in minio");
+    } catch (Exception e) {
+      logger.warn("Bucket may already exist: {}", e.getMessage());
+    } finally {
+      s3Client.close();
+    }
+
+    // Create ICE REST catalog server configuration
+    Config config =
+        new Config(
+            "localhost:8080", // addr
+            "localhost:8081", // debugAddr
+            null, // adminAddr
+            "test-catalog", // name
+            "jdbc:sqlite::memory:", // uri
+            "s3://test-bucket/warehouse", // warehouse
+            null, // localFileIOBaseDir
+            new Config.S3(minioEndpoint, true, "minioadmin", "minioadmin", "us-east-1"), // s3
+            null, // bearerTokens
+            new Config.AnonymousAccess(
+                true,
+                new Config.AccessConfig(
+                    false, null)), // anonymousAccess - enable with read-write for testing
+            null, // maintenanceSchedule
+            null, // maintenance
+            null, // loadTableProperties
+            null // icebergProperties
+            );
+
+    // Create backend catalog from config
+    Map<String, String> icebergConfig = config.toIcebergConfig();
+    Catalog backendCatalog =
+        org.apache.iceberg.CatalogUtil.buildIcebergCatalog("backend", icebergConfig, null);
+
+    // Start ICE REST catalog server
+    server = createServer("localhost", 8080, backendCatalog, config, icebergConfig);
+    server.start();
+
+    // Wait for server to be ready
+    while (!server.isStarted()) {
+      Thread.sleep(100);
+    }
+
+    // Server is ready for CLI commands
+  }
+
+  @AfterClass
+  public void tearDown() {
+
+    // Stop the REST catalog server
+    if (server != null) {
+      try {
+        server.stop();
+      } catch (Exception e) {
+        logger.error("Error stopping server: {}", e.getMessage(), e);
+      }
+    }
+
+    // Stop minio container
+    if (minio != null && minio.isRunning()) {
+      minio.stop();
+    }
+  }
+
+  /** Helper method to create a temporary CLI config file */
+  protected File createTempCliConfig() throws Exception {
+    File tempConfigFile = File.createTempFile("ice-rest-cli-", ".yaml");
+    tempConfigFile.deleteOnExit();
+
+    String configContent = "uri: http://localhost:8080\n";
+    Files.write(tempConfigFile.toPath(), configContent.getBytes());
+
+    return tempConfigFile;
+  }
+
+  /** Get the MinIO endpoint URL */
+  protected String getMinioEndpoint() {
+    return "http://" + minio.getHost() + ":" + minio.getMappedPort(9000);
+  }
+
+  /** Get the REST catalog URI */
+  protected String getCatalogUri() {
+    return "http://localhost:8080";
+  }
+}

ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.altinity.ice.rest.catalog;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/**
+ * Scenario-based integration tests for ICE REST Catalog.
+ *
+ * <p>This test class automatically discovers and executes all test scenarios from the
+ * test/resources/scenarios directory. Each scenario is run as a separate test case.
+ */
+public class ScenarioBasedIT extends RESTCatalogTestBase {
+
+  /**
+   * Data provider that discovers all test scenarios.
+   *
+   * @return Array of scenario names to be used as test parameters
+   * @throws Exception If there's an error discovering scenarios
+   */
+  @DataProvider(name = "scenarios")
+  public Object[][] scenarioProvider() throws Exception {
+    Path scenariosDir = getScenariosDirectory();
+    ScenarioTestRunner runner = createScenarioRunner();
+
+    List<String> scenarios = runner.discoverScenarios();
+
+    if (scenarios.isEmpty()) {
+      logger.warn("No test scenarios found in: {}", scenariosDir);
+      return new Object[0][0];
+    }
+
+    logger.info("Discovered {} test scenario(s): {}", scenarios.size(), scenarios);
+
+    // Convert to Object[][] for TestNG data provider
+    Object[][] data = new Object[scenarios.size()][1];
+    for (int i = 0; i < scenarios.size(); i++) {
+      data[i][0] = scenarios.get(i);
+    }
+    return data;
+  }
+
+  /**
+   * Parameterized test that executes a single scenario.
+   *
+   * @param scenarioName Name of the scenario to execute
+   * @throws Exception If the scenario execution fails
+   */
+  @Test(dataProvider = "scenarios")
+  public void testScenario(String scenarioName) throws Exception {
+    logger.info("====== Starting scenario test: {} ======", scenarioName);
+
+    ScenarioTestRunner runner = createScenarioRunner();
+    ScenarioTestRunner.ScenarioResult result = runner.executeScenario(scenarioName);
+
+    // Log results
+    if (result.getRunScriptResult() != null) {
+      logger.info("Run script exit code: {}", result.getRunScriptResult().exitCode());
+    }
+
+    if (result.getVerifyScriptResult() != null) {
+      logger.info("Verify script exit code: {}", result.getVerifyScriptResult().exitCode());
+    }
+
+    // Assert success
+    if (!result.isSuccess()) {
+      StringBuilder errorMessage = new StringBuilder();
+      errorMessage.append("Scenario '").append(scenarioName).append("' failed:\n");
+
+      if (result.getRunScriptResult() != null && result.getRunScriptResult().exitCode() != 0) {
+        errorMessage.append("\nRun script failed with exit code: ");
+        errorMessage.append(result.getRunScriptResult().exitCode());
+        errorMessage.append("\nStdout:\n");
+        errorMessage.append(result.getRunScriptResult().stdout());
+        errorMessage.append("\nStderr:\n");
+        errorMessage.append(result.getRunScriptResult().stderr());
+      }
+
+      if (result.getVerifyScriptResult() != null
+          && result.getVerifyScriptResult().exitCode() != 0) {
+        errorMessage.append("\nVerify script failed with exit code: ");
+        errorMessage.append(result.getVerifyScriptResult().exitCode());
+        errorMessage.append("\nStdout:\n");
+        errorMessage.append(result.getVerifyScriptResult().stdout());
+        errorMessage.append("\nStderr:\n");
+        errorMessage.append(result.getVerifyScriptResult().stderr());
+      }
+
+      throw new AssertionError(errorMessage.toString());
+    }
+
+    logger.info("====== Scenario test passed: {} ======", scenarioName);
+  }
+
+  /**
+   * Create a ScenarioTestRunner with the appropriate template variables.
+   *
+   * @return Configured ScenarioTestRunner
+   * @throws Exception If there's an error creating the runner
+   */
+  private ScenarioTestRunner createScenarioRunner() throws Exception {
+    Path scenariosDir = getScenariosDirectory();
+
+    // Create CLI config file
+    File cliConfig = createTempCliConfig();
+
+    // Build template variables
+    Map<String, String> templateVars = new HashMap<>();
+    templateVars.put("CLI_CONFIG", cliConfig.getAbsolutePath());
+    templateVars.put("MINIO_ENDPOINT", getMinioEndpoint());
+    templateVars.put("CATALOG_URI", getCatalogUri());
+
+    // Try to find ice-jar in the build
+    String projectRoot = Paths.get("").toAbsolutePath().getParent().toString();
+    String iceJar = projectRoot + "/ice/target/ice-jar";
+    File iceJarFile = new File(iceJar);
+
+    if (iceJarFile.exists() && iceJarFile.canExecute()) {
+      // Use pre-built ice-jar if available
+      templateVars.put("ICE_CLI", iceJar);
+      logger.info("Using ice-jar from: {}", iceJar);
+    } else {
+      // Fall back to using local-ice wrapper script
+      String localIce = projectRoot + "/.bin/local-ice";
+      templateVars.put("ICE_CLI", localIce);
+      logger.info("Using local-ice script from: {}", localIce);
+    }
+
+    return new ScenarioTestRunner(scenariosDir, templateVars);
+  }
+
+  /**
+   * Get the path to the scenarios directory.
+   *
+   * @return Path to scenarios directory
+   * @throws URISyntaxException If the resource URL cannot be converted to a path
+   */
+  private Path getScenariosDirectory() throws URISyntaxException {
+    // Get the scenarios directory from test resources
+    URL scenariosUrl = getClass().getClassLoader().getResource("scenarios");
+
+    if (scenariosUrl == null) {
+      // If not found in resources, try relative to project
+      return Paths.get("src/test/resources/scenarios");
+    }
+
+    return Paths.get(scenariosUrl.toURI());
+  }
+}