shdsbilling/on-premise/main/rancher_v2.9.1/source/rancher-2.9.1/pkg/image/charts.go

package image

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/Masterminds/semver/v3"
	"github.com/pkg/errors"
	libhelm "github.com/rancher/rancher/pkg/helm"
	"github.com/sirupsen/logrus"
	"gopkg.in/yaml.v2"
	"helm.sh/helm/v3/pkg/repo"
)

const RancherVersionAnnotationKey = "catalog.cattle.io/rancher-version"

// chartsToCheckConstraints and systemChartsToCheckConstraints define which charts and system charts should
// be checked for images and added to imageSet based on whether the given Rancher version/tag satisfies the chart's
// Rancher version constraints to allow support for multiple version lines of a chart in airgap setups. If a chart is
// not defined here, only the latest version of it will be checked for images and added if it passes the constraint.
// Note: CRD charts need to be added as well.
var chartsToCheckConstraints = map[string]struct{}{
	"rancher-istio": {},
}
var systemChartsToCheckConstraints = map[string]struct{}{
	"rancher-monitoring": {},
}

// chartsToIgnoreTags and systemChartsToIgnoreTags defines the charts and system charts in which a specified
// image tag should be ignored.
var chartsToIgnoreTags = map[string]string{
	"rancher-vsphere-csi": "latest",
	"rancher-vsphere-cpi": "latest",
}
var systemChartsToIgnoreTags = map[string]string{}

type Charts struct {
	Config ExportConfig
}

// FetchImages finds all the images used by all the charts in a Rancher charts repository and adds them to imageSet.
// The images from the latest version of each chart are always added to the images set, whereas the remaining versions
// are added only if the given Rancher version/tag satisfies the chart's Rancher version constraint annotation.
func (c Charts) FetchImages(imagesSet map[string]map[string]struct{}) error {
	if c.Config.ChartsPath == "" || c.Config.RancherVersion == "" {
		return nil
	}
	index, err := repo.LoadIndexFile(filepath.Join(c.Config.ChartsPath, "index.yaml"))
	if err != nil {
		return err
	}
	// Filter index entries based on their Rancher version constraint
	var filteredVersions repo.ChartVersions
	for _, versions := range index.Entries {
		if len(versions) == 0 {
			continue
		}
		// Always append the latest version of the chart if it passes the constraint check
		// Note: Selecting the correct latest version relies on the charts-build-scripts `make standardize` command
		// sorting the versions in the index file in descending order correctly.
		latestVersion := versions[0]
		if isConstraintSatisfied, err := c.checkChartVersionConstraint(*latestVersion); err != nil {
			return errors.Wrapf(err, "failed to check constraint of chart")
		} else if isConstraintSatisfied {
			filteredVersions = append(filteredVersions, latestVersion)
		}
		// Append the remaining versions of the chart if the chart exists in the chartsToCheckConstraints map
		// and the given Rancher version satisfies the chart's Rancher version constraint annotation.
		chartName := versions[0].Metadata.Name
		if _, ok := chartsToCheckConstraints[chartName]; ok {
			for _, version := range versions[1:] {
				if isConstraintSatisfied, err := c.checkChartVersionConstraint(*version); err != nil {
					return errors.Wrapf(err, "failed to check constraint of chart")
				} else if isConstraintSatisfied {
					filteredVersions = append(filteredVersions, version)
				}
			}
		}
	}
	// Find values.yaml files in the tgz files of each chart, and check for images to add to imageSet
	for _, version := range filteredVersions {
		tgzPath := filepath.Join(c.Config.ChartsPath, version.URLs[0])
		versionValues, err := decodeValuesFilesInTgz(tgzPath)
		if err != nil {
			logrus.Info(err)
			continue
		}
		tag, _ := chartsToIgnoreTags[version.Name]
		chartNameAndVersion := fmt.Sprintf("%s:%s", version.Name, version.Version)
		for _, values := range versionValues {
			if err = pickImagesFromValuesMap(imagesSet, values, chartNameAndVersion, c.Config.OsType, tag); err != nil {
				return err
			}
		}
	}
	return nil
}

// checkChartVersionConstraint retrieves the value of a chart's Rancher version constraint annotation, and
// returns true if the Rancher version in the export configuration satisfies the chart's constraint, false otherwise.
// If a chart does not have a Rancher version annotation defined, this function returns false.
func (c Charts) checkChartVersionConstraint(version repo.ChartVersion) (bool, error) {
	if constraintStr, ok := version.Annotations[RancherVersionAnnotationKey]; ok {
		return compareRancherVersionToConstraint(c.Config.RancherVersion, constraintStr)
	}
	return false, nil
}

type SystemCharts struct {
	Config ExportConfig
}

type Questions struct {
	RancherMinVersion string `yaml:"rancher_min_version"`
	RancherMaxVersion string `yaml:"rancher_max_version"`
}

// FetchImages finds all the images used by all the charts in a Rancher system charts repository and adds them to imageSet.
// The images from the latest version of each chart are always added to the images set, whereas the remaining versions
// are added only if the given Rancher version/tag satisfies the chart's Rancher version constraint defined in its questions file.
func (sc SystemCharts) FetchImages(imagesSet map[string]map[string]struct{}) error {
	if sc.Config.SystemChartsPath == "" || sc.Config.RancherVersion == "" {
		return nil
	}
	// Load system charts virtual index
	helm := libhelm.Helm{
		LocalPath: sc.Config.SystemChartsPath,
		IconPath:  sc.Config.SystemChartsPath,
		Hash:      "",
	}
	virtualIndex, err := helm.LoadIndex()
	if err != nil {
		return errors.Wrapf(err, "failed to load system charts index")
	}
	// Filter index entries based on their Rancher version constraint
	var filteredVersions libhelm.ChartVersions
	for _, versions := range virtualIndex.IndexFile.Entries {
		if len(versions) == 0 {
			continue
		}
		// Always append the latest version of the chart unless it has been intentionally hidden with constraints
		latestVersion := versions[0]
		if isConstraintSatisfied, err := sc.checkChartVersionConstraint(*latestVersion); err != nil {
			return errors.Wrapf(err, "failed to filter chart versions")
		} else if isConstraintSatisfied {
			filteredVersions = append(filteredVersions, latestVersion)
		}
		// Append the remaining versions of the chart if the chart exists in the systemChartsToCheckConstraints map
		// and the given Rancher version satisfies the chart's Rancher version constraint defined in its questions file
		chartName := versions[0].ChartMetadata.Name
		if _, ok := systemChartsToCheckConstraints[chartName]; ok {
			for _, version := range versions[1:] {
				if isConstraintSatisfied, err := sc.checkChartVersionConstraint(*version); err != nil {
					return errors.Wrapf(err, "failed to filter chart versions")
				} else if isConstraintSatisfied {
					filteredVersions = append(filteredVersions, version)
				}
			}
		}
	}
	// Find values.yaml files in each chart's local files, and check for images to add to imageSet
	for _, version := range filteredVersions {
		for _, file := range version.LocalFiles {
			if !isValuesFile(file) {
				continue
			}
			values, err := decodeValuesFile(file)
			if err != nil {
				return err
			}
			tag, _ := systemChartsToIgnoreTags[version.Name]
			chartNameAndVersion := fmt.Sprintf("%s:%s", version.Name, version.Version)
			if err = pickImagesFromValuesMap(imagesSet, values, chartNameAndVersion, sc.Config.OsType, tag); err != nil {
				return err
			}
		}
	}
	return nil
}

// checkChartVersionConstraint retrieves the value of a chart's Rancher version defined in its questions file, and
// returns true if the Rancher version in the export configuration satisfies the chart's constraint, false otherwise.
// If a chart does not have a Rancher version constraint defined, this function returns false.
func (sc SystemCharts) checkChartVersionConstraint(version libhelm.ChartVersion) (bool, error) {
	questionsPath := filepath.Join(sc.Config.SystemChartsPath, version.Dir, "questions.yaml")
	questions, err := decodeQuestionsFile(questionsPath)
	if os.IsNotExist(err) {
		questionsPath = filepath.Join(sc.Config.SystemChartsPath, version.Dir, "questions.yml")
		questions, err = decodeQuestionsFile(questionsPath)
	}
	if err != nil {
		logrus.Warnf("skipping system chart, %s:%s does not have a questions file", version.ChartMetadata.Name, version.ChartMetadata.Version)
		return false, nil
	}
	constraintStr := minMaxToConstraintStr(questions.RancherMinVersion, questions.RancherMaxVersion)
	if constraintStr == "" {
		return false, nil
	}
	return compareRancherVersionToConstraint(sc.Config.RancherVersion, constraintStr)
}

// compareRancherVersionToConstraint returns true if the Rancher version satisfies constraintStr, false otherwise.
func compareRancherVersionToConstraint(rancherVersion, constraintStr string) (bool, error) {
	if constraintStr == "" {
		return false, errors.Errorf("Invalid constraint string: \"%s\"", constraintStr)
	}
	constraint, err := semver.NewConstraint(constraintStr)
	if err != nil {
		return false, err
	}
	rancherSemVer, err := semver.NewVersion(rancherVersion)
	if err != nil {
		return false, err
	}
	// When the exporter is ran in a dev environment, we replace the rancher version with a dev version (e.g 2.X.99).
	// This breaks the semver compare logic for exporting because we use the Rancher version constraint < 2.X.99-0 in
	// many of our charts and since 2.X.99 > 2.X.99-0 the comparison returns false which is not the desired behavior.
	patch := rancherSemVer.Patch()
	if patch == 99 {
		patch = 98
	}
	// All pre-release versions are removed because the semver comparison will not yield the desired behavior unless
	// the constraint has a pre-release too. Since the exporter for charts can treat pre-releases and releases equally,
	// is cleaner to remove it. E.g. comparing rancherVersion 2.6.4-rc1 and constraint 2.6.3 - 2.6.5 yields false because
	// the versions in the contraint do not have a pre-release. This behavior comes from the semver module and is intentional.
	rSemVer, err := semver.NewVersion(fmt.Sprintf("%d.%d.%d", rancherSemVer.Major(), rancherSemVer.Minor(), patch))
	if err != nil {
		return false, err
	}
	return constraint.Check(rSemVer), nil
}

// minMaxToConstraintStr converts min and max Rancher version strings into a constraint string
// E.g min "2.6.3" max "2.6.4" -> constraintStr "2.6.3 - 2.6.4".
func minMaxToConstraintStr(min, max string) string {
	if min != "" && max != "" {
		return fmt.Sprintf("%s - %s", min, max)
	}
	if min != "" {
		return fmt.Sprintf(">= %s", min)
	}
	if max != "" {
		return fmt.Sprintf("<= %s", max)
	}
	return ""
}

// pickImagesFromValuesMap walks a values map to find images, and add them to imagesSet.
func pickImagesFromValuesMap(imagesSet map[string]map[string]struct{}, values map[interface{}]interface{}, chartNameAndVersion string, osType OSType, tagToIgnore string) error {
	walkMap(values, func(inputMap map[interface{}]interface{}) {
		repository, ok := inputMap["repository"].(string)
		if !ok {
			return
		}
		// No string type assertion because some charts have float typed image tags
		tag, ok := inputMap["tag"]
		if !ok {
			return
		}
		if fmt.Sprintf("%v", tag) == tagToIgnore {
			return
		}
		imageName := fmt.Sprintf("%s:%v", repository, tag)
		// By default, images are added to the generic images list ("linux"). For Windows and multi-OS
		// images to be considered, they must use a comma-delineated list (e.g. "os: windows",
		// "os: windows,linux", and "os: linux,windows").
		osList, ok := inputMap["os"].(string)
		if !ok {
			if inputMap["os"] != nil {
				errors.Errorf("field 'os:' for image %s contains neither a string nor nil", imageName)
			}
			if osType == Linux {
				addSourceToImage(imagesSet, imageName, chartNameAndVersion)
				return
			}
		}
		for _, os := range strings.Split(osList, ",") {
			os = strings.TrimSpace(os)
			if strings.EqualFold("windows", os) && osType == Windows {
				addSourceToImage(imagesSet, imageName, chartNameAndVersion)
				return
			}
			if strings.EqualFold("linux", os) && osType == Linux {
				addSourceToImage(imagesSet, imageName, chartNameAndVersion)
				return
			}
		}
	})
	return nil
}

// decodeValueFilesInTgz reads tarball in tgzPath and returns a slice of values corresponding to values.yaml files found inside of it.
func decodeValuesFilesInTgz(tgzPath string) ([]map[interface{}]interface{}, error) {
	tgz, err := os.Open(tgzPath)
	if err != nil {
		return nil, err
	}
	defer tgz.Close()
	gzr, err := gzip.NewReader(tgz)
	if err != nil {
		return nil, err
	}
	defer gzr.Close()
	tr := tar.NewReader(gzr)
	var valuesSlice []map[interface{}]interface{}
	for {
		header, err := tr.Next()
		switch {
		case err == io.EOF:
			return valuesSlice, nil
		case err != nil:
			return nil, err
		case header.Typeflag == tar.TypeReg && isValuesFile(header.Name):
			var values map[interface{}]interface{}
			if err := decodeYAMLFile(tr, &values); err != nil {
				return nil, err
			}
			valuesSlice = append(valuesSlice, values)
		default:
			continue
		}
	}
}

// walkMap walks inputMap and calls the callback function on all map type nodes including the root node.
func walkMap(inputMap interface{}, callback func(map[interface{}]interface{})) {
	switch data := inputMap.(type) {
	case map[interface{}]interface{}:
		callback(data)
		for _, value := range data {
			walkMap(value, callback)
		}
	case []interface{}:
		for _, elem := range data {
			walkMap(elem, callback)
		}
	}
}

func decodeQuestionsFile(path string) (Questions, error) {
	var questions Questions
	file, err := os.Open(path)
	if err != nil {
		return Questions{}, err
	}
	defer file.Close()
	if err := decodeYAMLFile(file, &questions); err != nil {
		return Questions{}, err
	}
	return questions, nil
}

func decodeValuesFile(path string) (map[interface{}]interface{}, error) {
	var values map[interface{}]interface{}
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	if err := decodeYAMLFile(file, &values); err != nil {
		return nil, err
	}
	return values, nil
}

func decodeYAMLFile(r io.Reader, target interface{}) error {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	return yaml.Unmarshal(data, target)
}

func isValuesFile(path string) bool {
	basename := filepath.Base(path)
	return basename == "values.yaml" || basename == "values.yml"
}