From e22049f12abe4354601ffbdde49b9c129bc52a5a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Dec 2025 17:39:59 +0100 Subject: [PATCH 1/5] Start service in SetUp process to update placeholders in config --- internal/benchrunner/runners/system/runner.go | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 7606cce48..7517d1b1e 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -58,6 +58,8 @@ type runner struct { mcollector *collector corporaFile string + service servicedeployer.DeployedService + // Execution order of following handlers is defined in runner.TearDown() method. deletePolicyHandler func(context.Context) error resetAgentPolicyHandler func(context.Context) error @@ -152,12 +154,32 @@ func (r *runner) setUp(ctx context.Context) error { } r.svcInfo.OutputDir = outputDir + // It is required to read once the configuration to know if a service deployer is needed. scenario, err := readConfig(r.options.BenchPath, r.options.BenchName, r.svcInfo) if err != nil { return err } r.scenario = scenario + if r.scenario.Corpora.InputService != nil { + // Just in the case service deployer is needed (input_service field), setup the service now. + // and re-read the configuration to have the final one with any possible service-related variable applied. + s, err := r.setupService(ctx) + if errors.Is(err, os.ErrNotExist) { + logger.Debugf("No service deployer defined for this benchmark") + } else if err != nil { + return err + } + r.service = s + r.svcInfo = s.Info() + + scenario, err := readConfig(r.options.BenchPath, r.options.BenchName, r.svcInfo) + if err != nil { + return err + } + r.scenario = scenario + } + if r.scenario.Corpora.Generator != nil { var err error r.generator, err = r.initializeGenerator(ctx) @@ -243,17 +265,6 @@ func (r *runner) setUp(ctx context.Context) error { } func (r *runner) run(ctx context.Context) (report reporters.Reportable, err error) { - var service servicedeployer.DeployedService - if r.scenario.Corpora.InputService != nil { - s, err := r.setupService(ctx) - if errors.Is(err, os.ErrNotExist) { - logger.Debugf("No service deployer defined for this benchmark") - } else if err != nil { - return nil, err - } - service = s - } - r.startMetricsColletion(ctx) defer r.mcollector.stop() @@ -271,8 +282,8 @@ func (r *runner) run(ctx context.Context) (report reporters.Reportable, err erro } // Signal to the service that the agent is ready (policy is assigned). - if service != nil && r.scenario.Corpora.InputService != nil && r.scenario.Corpora.InputService.Signal != "" { - if err = service.Signal(ctx, r.scenario.Corpora.InputService.Signal); err != nil { + if r.service != nil && r.scenario.Corpora.InputService != nil && r.scenario.Corpora.InputService.Signal != "" { + if err = r.service.Signal(ctx, r.scenario.Corpora.InputService.Signal); err != nil { return nil, fmt.Errorf("failed to notify benchmark service: %w", err) } } From 1a27d21a20f920a186f2261be38f96e7d4d679f9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Dec 2025 12:14:16 +0100 Subject: [PATCH 2/5] Allow to read config without rendering handlebars --- internal/benchrunner/runners/system/runner.go | 17 +++++++++-------- internal/benchrunner/runners/system/scenario.go | 15 +++++++++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 7517d1b1e..307ca4eaf 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -154,8 +154,9 @@ func (r *runner) setUp(ctx context.Context) error { } r.svcInfo.OutputDir = outputDir - // It is required to read once the configuration to know if a service deployer is needed. - scenario, err := readConfig(r.options.BenchPath, r.options.BenchName, r.svcInfo) + // First read of the configuration to know if a service deployer is needed. + // No need to render any template at this point. + scenario, err := readRawConfig(r.options.BenchPath, r.options.BenchName) if err != nil { return err } @@ -171,14 +172,14 @@ func (r *runner) setUp(ctx context.Context) error { return err } r.service = s - r.svcInfo = s.Info() + } - scenario, err := readConfig(r.options.BenchPath, r.options.BenchName, r.svcInfo) - if err != nil { - return err - } - r.scenario = scenario + // Read the configuration again to have any possible service-related variable applied. + scenario, err = readConfig(r.options.BenchPath, r.options.BenchName, &r.svcInfo) + if err != nil { + return err } + r.scenario = scenario if r.scenario.Corpora.Generator != nil { var err error diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go index 3cb9cceb4..b919c6943 100644 --- a/internal/benchrunner/runners/system/scenario.go +++ b/internal/benchrunner/runners/system/scenario.go @@ -77,7 +77,12 @@ func defaultConfig() *scenario { } } -func readConfig(benchPath string, scenario string, svcInfo servicedeployer.ServiceInfo) (*scenario, error) { +// readRawConfig reads the configuration without applying any template +func readRawConfig(benchPath string, scenario string) (*scenario, error) { + return readConfig(benchPath, scenario, nil) +} + +func readConfig(benchPath string, scenario string, svcInfo *servicedeployer.ServiceInfo) (*scenario, error) { configPath := filepath.Clean(filepath.Join(benchPath, fmt.Sprintf("%s.yml", scenario))) data, err := os.ReadFile(configPath) if err != nil { @@ -87,9 +92,11 @@ func readConfig(benchPath string, scenario string, svcInfo servicedeployer.Servi return nil, fmt.Errorf("could not load system benchmark configuration file: %s: %w", configPath, err) } - data, err = applyServiceInfo(data, svcInfo) - if err != nil { - return nil, fmt.Errorf("could not apply context to benchmark configuration file: %s: %w", configPath, err) + if svcInfo != nil { + data, err = applyServiceInfo(data, *svcInfo) + if err != nil { + return nil, fmt.Errorf("could not apply context to benchmark configuration file: %s: %w", configPath, err) + } } cfg, err := yaml.NewConfig(data, ucfg.PathSep(".")) From c0973e4956aca876d18008caf6ea59541b8687db Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Dec 2025 19:30:09 +0100 Subject: [PATCH 3/5] Add service name parameter to serviceSetup --- internal/benchrunner/runners/system/runner.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 307ca4eaf..2b6229d67 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -160,12 +160,11 @@ func (r *runner) setUp(ctx context.Context) error { if err != nil { return err } - r.scenario = scenario - if r.scenario.Corpora.InputService != nil { + if scenario.Corpora.InputService != nil { // Just in the case service deployer is needed (input_service field), setup the service now. // and re-read the configuration to have the final one with any possible service-related variable applied. - s, err := r.setupService(ctx) + s, err := r.setupService(ctx, scenario.Corpora.InputService.Name) if errors.Is(err, os.ErrNotExist) { logger.Debugf("No service deployer defined for this benchmark") } else if err != nil { @@ -309,7 +308,7 @@ func (r *runner) run(ctx context.Context) (report reporters.Reportable, err erro return createReport(r.options.BenchName, r.corporaFile, r.scenario, msum) } -func (r *runner) setupService(ctx context.Context) (servicedeployer.DeployedService, error) { +func (r *runner) setupService(ctx context.Context, serviceName string) (servicedeployer.DeployedService, error) { stackVersion, err := r.options.KibanaClient.Version() if err != nil { return nil, fmt.Errorf("cannot request Kibana version: %w", err) @@ -332,7 +331,7 @@ func (r *runner) setupService(ctx context.Context) (servicedeployer.DeployedServ return nil, fmt.Errorf("could not create service runner: %w", err) } - r.svcInfo.Name = r.scenario.Corpora.InputService.Name + r.svcInfo.Name = serviceName service, err := serviceDeployer.SetUp(ctx, r.svcInfo) if err != nil { return nil, fmt.Errorf("could not setup service: %w", err) From f97b7a9ff3571566761cba34c49c541c6369cbc2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 10 Dec 2025 17:31:54 +0100 Subject: [PATCH 4/5] Create function to get service name from config --- internal/benchrunner/runners/system/runner.go | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 2b6229d67..ff351caaf 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -154,17 +154,15 @@ func (r *runner) setUp(ctx context.Context) error { } r.svcInfo.OutputDir = outputDir - // First read of the configuration to know if a service deployer is needed. - // No need to render any template at this point. - scenario, err := readRawConfig(r.options.BenchPath, r.options.BenchName) + serviceName, err := r.serviceDefinedInConfig() if err != nil { - return err + return fmt.Errorf("failed to determine if service is defined in config: %w", err) } - if scenario.Corpora.InputService != nil { + if serviceName != "" { // Just in the case service deployer is needed (input_service field), setup the service now. // and re-read the configuration to have the final one with any possible service-related variable applied. - s, err := r.setupService(ctx, scenario.Corpora.InputService.Name) + s, err := r.setupService(ctx, serviceName) if errors.Is(err, os.ErrNotExist) { logger.Debugf("No service deployer defined for this benchmark") } else if err != nil { @@ -174,7 +172,7 @@ func (r *runner) setUp(ctx context.Context) error { } // Read the configuration again to have any possible service-related variable applied. - scenario, err = readConfig(r.options.BenchPath, r.options.BenchName, &r.svcInfo) + scenario, err := readConfig(r.options.BenchPath, r.options.BenchName, &r.svcInfo) if err != nil { return err } @@ -264,6 +262,21 @@ func (r *runner) setUp(ctx context.Context) error { return nil } +func (r *runner) serviceDefinedInConfig() (string, error) { + // Read of the configuration to know if a service deployer is needed. + // No need to render any template at this point. + scenario, err := readRawConfig(r.options.BenchPath, r.options.BenchName) + if err != nil { + return "", err + } + + if scenario.Corpora.InputService == nil { + return "", nil + } + + return scenario.Corpora.InputService.Name, nil +} + func (r *runner) run(ctx context.Context) (report reporters.Reportable, err error) { r.startMetricsColletion(ctx) defer r.mcollector.stop() From e6ea4bc5b8d13d68a1b09352a85b6b7172e98241 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 10 Dec 2025 18:26:02 +0100 Subject: [PATCH 5/5] Update comments --- internal/benchrunner/runners/system/runner.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index ff351caaf..a5f44ac9f 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -160,8 +160,9 @@ func (r *runner) setUp(ctx context.Context) error { } if serviceName != "" { - // Just in the case service deployer is needed (input_service field), setup the service now. - // and re-read the configuration to have the final one with any possible service-related variable applied. + // Just in the case service deployer is needed (input_service field), setup the service now so all the + // required information is available in r.svcInfo (e.g. hostname, port, etc). + // This info may be needed to render the variables in the configuration. s, err := r.setupService(ctx, serviceName) if errors.Is(err, os.ErrNotExist) { logger.Debugf("No service deployer defined for this benchmark") @@ -171,7 +172,7 @@ func (r *runner) setUp(ctx context.Context) error { r.service = s } - // Read the configuration again to have any possible service-related variable applied. + // Read the configuration again to have any possible service-related variable rendered. scenario, err := readConfig(r.options.BenchPath, r.options.BenchName, &r.svcInfo) if err != nil { return err