#!/bin/bash set -e cleanup() { EXIT=$? set +ex echo Stopping rancher server kill $RANCHER_RUN_PID wait $RANCHER_RUN_PID if [ $PID != -1 ]; then kill $PID wait $PID fi return $EXIT } # skipping tests for s390x since k3s doesn't support s390x yet if [ ${ARCH} == s390x ]; then exit 0 fi cd $(dirname $0)/.. echo Running unit tests CGO_ENABLED=0 go test -cover -tags=test ./pkg/... if [ ${ARCH} == arm64 ] || [ ${ARCH} == s390x ]; then export ETCD_UNSUPPORTED_ARCH=${ARCH} fi TB_ORG=rancher if [ -z "${TEST_DIST}" ] || [ "${TEST_DIST}" = "k3s" ]; then TEST_DIST=k3s AIRGAP=-airgap TB_ORG=k3s-io else LINUX=.linux fi export DIST=${TEST_DIST} export SOME_K8S_VERSION=${SOME_K8S_VERSION} export TB_ORG=${TB_ORG} export CATTLE_CHART_DEFAULT_URL=${CATTLE_CHART_DEFAULT_URL} # Tell Rancher to use the recently-built Rancher cluster agent image. This image is built as part of CI and will be # copied to the in-cluster registry during test setup below. source ./scripts/version export CATTLE_AGENT_IMAGE="rancher/rancher-agent:${AGENT_TAG}" echo "Using Rancher agent image $CATTLE_AGENT_IMAGE" eval "$(grep '^ENV CATTLE_SYSTEM_AGENT' package/Dockerfile | awk '{print "export " $2 "=" $3}')" eval "$(grep '^ENV CATTLE_WINS_AGENT' package/Dockerfile | awk '{print "export " $2 "=" $3}')" eval "$(grep '^ENV CATTLE_CSI_PROXY_AGENT' package/Dockerfile | awk '{print "export " $2 "=" $3}')" eval "$(grep '^ENV CATTLE_KDM_BRANCH' package/Dockerfile | awk '{print "export " $2 "=" $3}')" if [ -z "${SOME_K8S_VERSION}" ]; then # Get the last release for $DIST, which is usually the latest version or an experimental version. # Previously this would use channels, but channels no longer reflect the latest version since # https://github.com/rancher/rancher/issues/36827 has added appDefaults. We do not use appDefaults # here for simplicity's sake, as it requires semver parsing & matching. The last release should # be good enough for our needs. export SOME_K8S_VERSION=$(curl -sS https://raw.githubusercontent.com/rancher/kontainer-driver-metadata/release-v2.9/data/data.json | jq -r ".$DIST.releases[-1].version") fi if [ -z "${CATTLE_CHART_DEFAULT_URL}" ]; then # If `CATTLE_CHART_DEFAULT_URL` is not set, use the `https://github.com/rancher/charts` so GitHub is used instead of # the default `https://git.rancher.io/charts` to reduce the reliance and load on our Git mirror export CATTLE_CHART_DEFAULT_URL=https://github.com/rancher/charts fi echo Starting rancher server for test touch /tmp/rancher.log mkdir -p /var/lib/rancher/$DIST/agent/images grep PodTestImage ./tests/v2prov/defaults/defaults.go | cut -f2 -d'"' > /var/lib/rancher/$DIST/agent/images/pull.txt grep MachineProvisionImage ./pkg/settings/setting.go | cut -f4 -d'"' >> /var/lib/rancher/$DIST/agent/images/pull.txt mkdir -p /usr/share/rancher/ui/assets curl -sLf https://github.com/rancher/system-agent/releases/download/${CATTLE_SYSTEM_AGENT_VERSION}/rancher-system-agent-amd64 -o /usr/share/rancher/ui/assets/rancher-system-agent-amd64 curl -sLf https://github.com/rancher/system-agent/releases/download/${CATTLE_SYSTEM_AGENT_VERSION}/rancher-system-agent-arm64 -o /usr/share/rancher/ui/assets/rancher-system-agent-arm64 curl -sLf https://github.com/rancher/system-agent/releases/download/${CATTLE_SYSTEM_AGENT_VERSION}/system-agent-uninstall.sh -o /usr/share/rancher/ui/assets/system-agent-uninstall.sh build_and_run_rancher() { RESTART_COUNT=0 while sleep 2; do if [ "$PID" != "-1" ] && [ ! -e /proc/$PID ]; then echo Rancher died dump_rancher_logs echo K3s logs were: echo -e "-----K3S-LOG-DUMP-START-----" cat build/testdata/k3s.log | gzip | base64 -w 0 echo -e "\n-----K3S-LOG-DUMP-END-----" set +e echo Attempting to kill K3s pkill -e k3s set -e PID=-1 if [ "$RESTART_COUNT" = "2" ]; then echo Rancher died 3 times, aborting kill -42 $PWRAPPROC fi RESTART_COUNT=$((RESTART_COUNT + 1)) sleep 5 fi if [ "$PID" = "-1" ]; then echo Starting rancher server using run ./scripts/run >/tmp/rancher.log 2>&1 & PID=$! fi sleep 2 done } dump_rancher_logs() { echo Rancher logs were echo -e "-----RANCHER-LOG-DUMP-START-----" cat /tmp/rancher.log | gzip | base64 -w 0 echo -e "\n-----RANCHER-LOG-DUMP-END-----" } # Compile Rancher # This needs to happen before build_and_run_rancher is executed in the background. # Otherwise, build_and_run_rancher will also compile Rancher, overly elongating the # time before Rancher is ready and therefore causing flakiness of the health check below. ./scripts/build-server # uncomment to get startup logs. Don't leave them on because it slows drone down too # much #tail -F /tmp/rancher.log & #TPID=$! trap "exit 1" 42 PWRAPPROC="$$" PID=-1 build_and_run_rancher & RANCHER_RUN_PID=$! trap cleanup exit echo "Waiting for Rancher to be healthy" ./scripts/retry --sleep 2 "curl -sf -o /dev/null http://localhost:8080/ping" echo "Waiting up to 5 minutes for rancher-webhook deployment" ./scripts/retry \ --timeout 300 `# Time out after 300 seconds (5 min)` \ --sleep 2 `# Sleep for 2 seconds in between attempts` \ --message-interval 30 `# Print the progress message below every 30 attempts (roughly every minute)` \ --message "rancher-webhook was not available after {{elapsed}} seconds" `# Print this progress message` \ "kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml rollout status -w -n cattle-system deploy/rancher-webhook &>/dev/null" echo "Waiting up to 5 minutes for rancher-provisioning-capi deployment" ./scripts/retry \ --timeout 300 `# Time out after 300 seconds (5 min)` \ --sleep 2 `# Sleep for 2 seconds in between attempts` \ --message-interval 30 `# Print the progress message below every 30 attempts (roughly every minute)` \ --message "rancher-provisioning-capi was not available after {{elapsed}} seconds" `# Print this progress message` \ "kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml rollout status -w -n cattle-provisioning-capi-system deploy/capi-controller-manager &>/dev/null" #kill $TPID echo Running build-integration-setup ./tests/v2/integration/scripts/build-integration-setup echo Running integrationsetup export CATTLE_TEST_CONFIG=$(pwd)/config.yaml # used by integration tests and test setup ./tests/v2/integration/bin/integrationsetup || { dump_rancher_logs exit 1 } echo Running go integration tests CGO_ENABLED=0 go test -v -failfast -p 1 ./tests/v2/integration/... || { dump_rancher_logs exit 1 } echo Running tox tests INT_TESTS_STARTED=true cd ./tests/integration tox -e rancher -- -m "not nonparallel" -n $(nproc) tox -e rancher -- -m nonparallel tail -f /tmp/rancher-test.log &