[WIP] Allow to use custom URLs to retrieve ECS schemas

cmd/build.go

@@ -13,6 +13,7 @@ import (
 	"github.com/elastic/elastic-package/internal/builder"
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/files"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
 )
@@ -78,6 +79,11 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 	}
 	logger.Debugf("Use build directory: %s", buildDir)
 
+	config, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
 	target, err := builder.BuildPackage(builder.BuildOptions{
 		PackageRoot:    packageRoot,
 		BuildDir:       buildDir,
@@ -86,6 +92,7 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 		SkipValidation: skipValidation,
 		RepositoryRoot: repositoryRoot,
 		UpdateReadmes:  true,
+		SchemaURLs:     config.SchemaURLs(),
 	})
 	if err != nil {
 		return fmt.Errorf("building package failed: %w", err)

cmd/lint.go

@@ -12,6 +12,7 @@ import (
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/docs"
 	"github.com/elastic/elastic-package/internal/files"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/validation"
@@ -57,7 +58,12 @@ func lintCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("package root not found: %w", err)
 	}
 
-	readmeFiles, err := docs.AreReadmesUpToDate(repositoryRoot, packageRoot)
+	config, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	readmeFiles, err := docs.AreReadmesUpToDate(repositoryRoot, packageRoot, config.SchemaURLs())
 	if err != nil {
 		for _, f := range readmeFiles {
 			if !f.UpToDate {

internal/builder/external_fields.go

@@ -22,7 +22,7 @@ import (
 
 var semver3_0_0 = semver.MustParse("3.0.0")
 
-func resolveExternalFields(packageRoot, buildPackageRoot string) error {
+func resolveExternalFields(packageRoot, buildPackageRoot string, schemaURLs fields.SchemaURLs) error {
 	bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot)
 	if err != nil {
 		return fmt.Errorf("can't read build manifest: %w", err)
@@ -37,7 +37,7 @@ func resolveExternalFields(packageRoot, buildPackageRoot string) error {
 	}
 
 	logger.Debugf("Package has external dependencies defined")
-	fdm, err := fields.CreateFieldDependencyManager(bm.Dependencies)
+	fdm, err := fields.CreateFieldDependencyManager(bm.Dependencies, schemaURLs)
 	if err != nil {
 		return fmt.Errorf("can't create field dependency manager: %w", err)
 	}

internal/builder/packages.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/elastic/elastic-package/internal/docs"
 	"github.com/elastic/elastic-package/internal/environment"
+	"github.com/elastic/elastic-package/internal/fields"
 	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
@@ -34,6 +35,7 @@ type BuildOptions struct {
 	SignPackage    bool
 	SkipValidation bool
 	UpdateReadmes  bool
+	SchemaURLs     fields.SchemaURLs
 }
 
 // BuildDirectory function locates the target build directory. If the directory doesn't exist, it will create it.
@@ -215,7 +217,7 @@ func BuildPackage(options BuildOptions) (string, error) {
 	}
 
 	logger.Debug("Resolve external fields")
-	err = resolveExternalFields(options.PackageRoot, buildPackageRoot)
+	err = resolveExternalFields(options.PackageRoot, buildPackageRoot, options.SchemaURLs)
 	if err != nil {
 		return "", fmt.Errorf("resolving external fields failed: %w", err)
 	}
@@ -231,7 +233,7 @@ func BuildPackage(options BuildOptions) (string, error) {
 	}
 
 	if options.UpdateReadmes {
-		err = docs.UpdateReadmes(options.RepositoryRoot, options.PackageRoot, buildPackageRoot)
+		err = docs.UpdateReadmes(options.RepositoryRoot, options.PackageRoot, buildPackageRoot, options.SchemaURLs)
 		if err != nil {
 			return "", fmt.Errorf("updating readme files failed: %w", err)
 		}

internal/docs/exported_fields.go

@@ -25,7 +25,7 @@ var escaper = strings.NewReplacer("*", "\\*", "{", "\\{", "}", "\\}", "<", "\\<"
 
 // renderExportedFields renders the fields for a package or data stream, fieldsParentRoot must be
 // the path to the root directory of the package or data stream.
-func renderExportedFields(fieldsParentRoot string) (string, error) {
+func renderExportedFields(fieldsParentRoot string, schemaURLs fields.SchemaURLs) (string, error) {
 	injectOptions := fields.InjectFieldsOptions{
 		// Keep External parameter when rendering fields, so we can render
 		// documentation for empty groups imported from ECS, for backwards compatibility.
@@ -35,7 +35,10 @@ func renderExportedFields(fieldsParentRoot string) (string, error) {
 		// keep them to accept them for validation.
 		SkipEmptyFields: true,
 	}
-	validator, err := fields.CreateValidatorForDirectory(fieldsParentRoot, fields.WithInjectFieldsOptions(injectOptions))
+	validator, err := fields.CreateValidatorForDirectory(fieldsParentRoot,
+		fields.WithInjectFieldsOptions(injectOptions),
+		fields.WithSchemaURLs(schemaURLs),
+	)
 	if err != nil {
 		return "", fmt.Errorf("can't create fields validator instance (path: %s): %w", fieldsParentRoot, err)
 	}

internal/docs/readme.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/pmezard/go-difflib/difflib"
 
+	"github.com/elastic/elastic-package/internal/fields"
 	"github.com/elastic/elastic-package/internal/logger"
 )
 
@@ -31,7 +32,7 @@ const (
 )
 
 // AreReadmesUpToDate function checks if all the .md readme files are up-to-date.
-func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFile, error) {
+func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string, schemaURLs fields.SchemaURLs) ([]ReadmeFile, error) {
 	linksFilePath, err := linksDefinitionsFilePath(repositoryRoot)
 	if err != nil {
 		return nil, fmt.Errorf("locating links file failed: %w", err)
@@ -45,7 +46,7 @@ func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFi
 	var readmeFiles []ReadmeFile
 	for _, filePath := range files {
 		fileName := filepath.Base(filePath)
-		ok, diff, err := isReadmeUpToDate(fileName, linksFilePath, packageRoot)
+		ok, diff, err := isReadmeUpToDate(fileName, linksFilePath, packageRoot, schemaURLs)
 		if !ok || err != nil {
 			readmeFile := ReadmeFile{
 				FileName: fileName,
@@ -64,11 +65,11 @@ func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFi
 }
 
 // isReadmeUpToDate function checks if a single readme file is up-to-date.
-func isReadmeUpToDate(fileName, linksFilePath, packageRoot string) (bool, string, error) {
+func isReadmeUpToDate(fileName, linksFilePath, packageRoot string, schemaURLs fields.SchemaURLs) (bool, string, error) {
 	logger.Debugf("Check if %s is up-to-date", fileName)
 
 	// the readme is generated within the package root, so source should be the packageRoot files too
-	rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot)
+	rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot, schemaURLs)
 	if err != nil {
 		return false, "", fmt.Errorf("generating readme file failed: %w", err)
 	}
@@ -99,7 +100,7 @@ func isReadmeUpToDate(fileName, linksFilePath, packageRoot string) (bool, string
 
 // UpdateReadmes function updates all .md readme files using a defined template
 // files. The function doesn't perform any action if the template file is not present.
-func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string) error {
+func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string, schemaURLs fields.SchemaURLs) error {
 	linksFilePath, err := linksDefinitionsFilePath(repositoryRoot)
 	if err != nil {
 		return fmt.Errorf("locating links file failed: %w", err)
@@ -112,7 +113,7 @@ func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string
 
 	for _, filePath := range readmeFiles {
 		fileName := filepath.Base(filePath)
-		target, err := updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot)
+		target, err := updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot, schemaURLs)
 		if err != nil {
 			return fmt.Errorf("updating readme file %s failed: %w", fileName, err)
 		}
@@ -128,10 +129,10 @@ func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string
 
 // updateReadme function updates a single readme file using a defined template file.
 // It writes the rendered file to both the package directory and the package build directory.
-func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string) (string, error) {
+func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string, schemaURLs fields.SchemaURLs) (string, error) {
 	logger.Debugf("Update the %s file", fileName)
 
-	rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot)
+	rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot, schemaURLs)
 	if err != nil {
 		return "", err
 	}
@@ -154,7 +155,7 @@ func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string)
 // generateReadme function generates the readme file content
 // the readme takes a template that lives under the _dev/build/docs directory at the packageRoot.
 // the readme template reads data from the packageRoot directory.
-func generateReadme(fileName, linksFilePath, packageRoot string) ([]byte, bool, error) {
+func generateReadme(fileName, linksFilePath, packageRoot string, schemaURLs fields.SchemaURLs) ([]byte, bool, error) {
 	logger.Debugf("Generate %s file (package: %s)", fileName, packageRoot)
 	templatePath, found, err := findReadmeTemplatePath(fileName, packageRoot)
 	if err != nil {
@@ -173,7 +174,7 @@ func generateReadme(fileName, linksFilePath, packageRoot string) ([]byte, bool,
 
 	// templatePath lives under the _dev/build/docs directory at the package root.
 	// builtPackageRoot is the root directory of the built package.
-	rendered, err := renderReadme(fileName, packageRoot, templatePath, linksMap)
+	rendered, err := renderReadme(fileName, packageRoot, templatePath, linksMap, schemaURLs)
 	if err != nil {
 		return nil, true, fmt.Errorf("rendering Readme failed: %w", err)
 	}
@@ -194,7 +195,7 @@ func findReadmeTemplatePath(fileName, packageRoot string) (string, bool, error)
 }
 
 // renderReadme function renders the readme file reading from
-func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) ([]byte, error) {
+func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap, schemaURLs fields.SchemaURLs) ([]byte, error) {
 	logger.Debugf("Render %s file (package: %s, templatePath: %s)", fileName, packageRoot, templatePath)
 
 	t := template.New(fileName)
@@ -208,9 +209,9 @@ func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap)
 		"fields": func(args ...string) (string, error) {
 			if len(args) > 0 {
 				dataStreamRoot := filepath.Join(packageRoot, "data_stream", args[0])
-				return renderExportedFields(dataStreamRoot)
+				return renderExportedFields(dataStreamRoot, schemaURLs)
 			}
-			return renderExportedFields(packageRoot)
+			return renderExportedFields(packageRoot, schemaURLs)
 		},
 		"url": func(args ...string) (string, error) {
 			options := linkOptions{}

internal/fields/dependency_manager.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -27,17 +28,20 @@ const (
 	localFilePrefix    = "file://"
 
 	ecsSchemaFile = "ecs_nested.yml"
-	ecsSchemaURL  = "https://raw.githubusercontent.com/elastic/ecs/%s/generated/ecs/%s"
 )
 
 // DependencyManager is responsible for resolving external field dependencies.
 type DependencyManager struct {
 	schema map[string][]FieldDefinition
 }
 
+type SchemaURLs struct {
+	ECSBase string `yaml:"ecs_base,omitempty"`
+}
+
 // CreateFieldDependencyManager function creates a new instance of the DependencyManager.
-func CreateFieldDependencyManager(deps buildmanifest.Dependencies) (*DependencyManager, error) {
-	schema, err := buildFieldsSchema(deps)
+func CreateFieldDependencyManager(deps buildmanifest.Dependencies, urls SchemaURLs) (*DependencyManager, error) {
+	schema, err := buildFieldsSchema(deps, urls)
 	if err != nil {
 		return nil, fmt.Errorf("can't build fields schema: %w", err)
 	}
@@ -46,31 +50,31 @@ func CreateFieldDependencyManager(deps buildmanifest.Dependencies) (*DependencyM
 	}, nil
 }
 
-func buildFieldsSchema(deps buildmanifest.Dependencies) (map[string][]FieldDefinition, error) {
+func buildFieldsSchema(deps buildmanifest.Dependencies, urls SchemaURLs) (map[string][]FieldDefinition, error) {
 	schema := map[string][]FieldDefinition{}
-	ecsSchema, err := loadECSFieldsSchema(deps.ECS)
+	ecsSchema, err := loadECSFieldsSchema(deps.ECS, urls.ECSBase)
 	if err != nil {
 		return nil, fmt.Errorf("can't load fields: %w", err)
 	}
 	schema[ecsSchemaName] = ecsSchema
 	return schema, nil
 }
 
-func loadECSFieldsSchema(dep buildmanifest.ECSDependency) ([]FieldDefinition, error) {
+func loadECSFieldsSchema(dep buildmanifest.ECSDependency, baseURL string) ([]FieldDefinition, error) {
 	if dep.Reference == "" {
 		logger.Debugf("ECS dependency isn't defined")
 		return nil, nil
 	}
 
-	content, err := readECSFieldsSchemaFile(dep)
+	content, err := readECSFieldsSchemaFile(dep, baseURL)
 	if err != nil {
 		return nil, fmt.Errorf("error reading ECS fields schema file: %w", err)
 	}
 
 	return parseECSFieldsSchema(content)
 }
 
-func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency) ([]byte, error) {
+func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency, baseURL string) ([]byte, error) {
 	if strings.HasPrefix(dep.Reference, localFilePrefix) {
 		path := strings.TrimPrefix(dep.Reference, localFilePrefix)
 		return os.ReadFile(path)
@@ -90,7 +94,10 @@ func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency) ([]byte, error) {
 	if errors.Is(err, os.ErrNotExist) {
 		logger.Debugf("Pulling ECS dependency using reference: %s", dep.Reference)
 
-		url := fmt.Sprintf(ecsSchemaURL, gitReference, ecsSchemaFile)
+		url, err := ecsSchemaURL(baseURL, gitReference, ecsSchemaFile)
+		if err != nil {
+			return nil, fmt.Errorf("can't generate ECS schema URL: %w", err)
+		}
 		logger.Debugf("Schema URL: %s", url)
 		resp, err := http.Get(url)
 		if err != nil {
@@ -166,6 +173,20 @@ func asGitReference(reference string) (string, error) {
 	return reference[len(gitReferencePrefix):], nil
 }
 
+func ecsSchemaURL(baseURL string, gitReference string, schemaFile string) (string, error) {
+	if baseURL == "" {
+		return "", errors.New("no base URL configured")
+	}
+	parsedBaseURL, err := url.Parse(baseURL)
+	switch {
+	case err != nil:
+		return "", fmt.Errorf("invalid base URL (%s) for ECS schema: %w", baseURL, err)
+	case parsedBaseURL.Scheme != "http" && parsedBaseURL.Scheme != "https":
+		return "", fmt.Errorf("invalid scheme in base URL, found %s, expected http or https", parsedBaseURL.Scheme)
+	}
+	return parsedBaseURL.JoinPath(gitReference, "generated", "ecs", schemaFile).String(), nil
+}
+
 // InjectFieldsOptions allow to configure fields injection.
 type InjectFieldsOptions struct {
 	// KeepExternal can be set to true to avoid deleting the `external` parameter