run-pelorus-e2e-tests

#!/usr/bin/env bash
#
# Copyright Red Hat
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#

SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
# Match the .venv created by the Makefile
DEFAULT_VENV="${SCRIPT_DIR}/../.venv"
TMP_DIR_PREFIX="pelorus_tmp_"
PELORUS_NAMESPACE="pelorus"
# Used in CI
PROW_SECRETS_DIR="/var/run/konveyor/pelorus/pelorus-github/"
PROW_S3_SECRETS_DIR="/var/run/konveyor/pelorus/pelorus-s3amazon/"

# Binary build script
BINARY_BUILD_SCRIPT="${SCRIPT_DIR}/e2e-tests-templates/build_binary_app"


# Used to download required files prior to running the job
# Arguments:
#    $1 - URL from which download the file
#    $2 - File name of the output file
#    $3 - (Optional) - destination directory, defaults to SCRIPT_DIR
function download_file_from_url() {
    local url=$1
    local file_name=$2
    local dest_folder="${3:-$SCRIPT_DIR}" # Use ./ as default dest_folder

    pushd "${dest_folder}" || exit
      echo "Downloading file: ${url}"
      echo "To: ${dest_folder}/${file_name}"
      if curl --fail-with-body --help >/dev/null 2>&1; then
          curl --fail-with-body -Lo "${file_name}" "${url}" || exit
      elif type curl; then
          curl -Lo "${file_name}" "${url}" || exit
      else
          wget -O "${file_name}" "${url}" || exit
      fi
    popd || exit
}

# Function to safely remove temporary files and temporary download dir
# Argument is optional exit value to propagate it after cleanup
function cleanup_and_exit() {
    local exit_val=$1
    if [ -z "${DWN_DIR}" ]; then
        echo "cleanup_and_exit(): Temp download dir not provided !" >&2
    else
      # Ensure dir exists and starts with prefix
      if [ -d "${DWN_DIR}" ]; then
          PELORUS_TMP_DIR=$(basename "${DWN_DIR}")
          if [[ "${PELORUS_TMP_DIR}" =~ "${TMP_DIR_PREFIX}"* ]]; then
              echo "Cleaning up temporary files"
              eval rm -f "${DWN_DIR}/*"
              rmdir "${DWN_DIR}"
          fi
      fi
    fi
    # Show logs from pods which are in CrashLoopBackOff state
    for failing_pod in $(oc get pods -n "${PELORUS_NAMESPACE}" | grep -i crash | cut -d ' ' -f1);
    do
        echo "Logs from crashed pod: ${failing_pod}"
        oc logs -n "${PELORUS_NAMESPACE}" "${failing_pod}"
    done

    # Cleanup binary builds
    for binary_build_ns in "${BINARY_BUILDS_NAMESPACES[@]}"; do
        "${BINARY_BUILD_SCRIPT}" -c -n "${binary_build_ns}"
    done

    # Propagate exit value if was provided
    [ -n "${exit_val}" ] && echo "Exit code: ${exit_val}" && exit "$exit_val"
    exit 0
}

function retry() {
    local timeout="$1"; shift
    local sleep_time="$1"; shift
    local cmd="$*"
    # Let's print what is happening in the subshell
    set -x
    timeout "$timeout" bash -c "until ${cmd}; do sleep ${sleep_time}; done" || exit 2
    set +x
}

function print_help() {
    printf "\nUsage: %s [OPTION]... -d [DIR]\n\n" % "$0"
    printf "\tStartup:\n"
    printf "\t  -h\tprint this help\n"
    printf "\n\tOptions:\n"
    printf "\t  -b\tbranch of the mig-demo-apps\n"
    printf "\t  -f\tvalues filename passed to the Pelorus deployment from the todolist-mongo-go project\n"
    printf "\t  -o\tgithub organization of the mig-demo-apps\n"
    printf "\t  -d\tpath to virtualenv DIR\n"
    printf "\t  -s\tpath to DIR with secrets files (used in prow CI)\n"
    printf "\t  -e\tenable exporter <comma_list>. e.g. failure\n"    
    printf "\t  -t\tenable Thanos with s3 bucket\n"
    printf "\t  -c\tpath to DIR with secrets files for s3 bucket\n"

    exit 0
}

# We use exported functions instead of aliases, so they are available
# in subshell. This is required for timeout.
set -a
# shellcheck disable=SC2269
PELORUS_NAMESPACE="${PELORUS_NAMESPACE}"
function ogn() { printf "oc get --namespace %s $*\n" "${PELORUS_NAMESPACE}"; oc get --namespace "${PELORUS_NAMESPACE}" "$@"; }
function ogns() { printf "oc get --namespace %s svc $*\n" "${PELORUS_NAMESPACE}"; oc get --namespace "${PELORUS_NAMESPACE}" svc "$@"; }
function ornds() { printf "oc rollout status --namespace %s deployments $*\n" "${PELORUS_NAMESPACE}"; oc rollout status --namespace ${PELORUS_NAMESPACE} deployments "$@"; }
function owpr() { printf "oc wait pod --for=condition=Ready -n %s -l pelorus.konveyor.io/exporter-type=$*\n" "${PELORUS_NAMESPACE}"; oc wait pod --for=condition=Ready -n ${PELORUS_NAMESPACE} -l pelorus.konveyor.io/exporter-type="$*"; }
set +a

### Options
OPTIND=1
ENABLE_FAIL_EXP=false
ENABLE_GITLAB_COM_EXP=false
ENABLE_GITEA_COM_EXP=false
ENABLE_BITBUCKET_COM_EXP=false
ENABLE_JIRA_FAIL_EXP=false
ENABLE_JIRA_CUSTOM_FAIL_EXP=false
ENABLE_THANOS=false

# Used for cleanup to ensure created binary builds can be safely deleted
BINARY_BUILDS_NAMESPACES=()

while getopts "h?b:d:s:o:f:e:tc:" option; do
    case "$option" in
    h|\?) print_help;;
    b)    demo_branch=$OPTARG;;
    f)    ci_filename=$OPTARG;;
    o)    demo_org=$OPTARG;;
    d)    venv_dir=$OPTARG;;
    s)    secrets_dir=$OPTARG;;
    e)    enable_exporters=$OPTARG;;
    t)    ENABLE_THANOS=true;;
    c)    s3_secrets_dir=$OPTARG;;
    esac
done

if [ -z "${venv_dir}" ]; then
    VENV="${DEFAULT_VENV}"
else
    VENV="${venv_dir}"
fi

if [ -z "${secrets_dir}" ]; then
    SECRETS_DIR="${PROW_SECRETS_DIR}"
else
    SECRETS_DIR="${secrets_dir}"
fi

if [ -z "${s3_secrets_dir}" ]; then
    S3_SECRETS_DIR="${PROW_S3_SECRETS_DIR}"
else
    S3_SECRETS_DIR="${s3_secrets_dir}"
fi

if [ -n "${enable_exporters}" ]; then
    echo ",$enable_exporters," | grep -q ",failure," && ENABLE_FAIL_EXP=true && echo "Enabling Failure exporter"
    echo ",$enable_exporters," | grep -q ",gitlab_committime," && ENABLE_GITLAB_COM_EXP=true && echo "Enabling Gitlab committime exporter"
    echo ",$enable_exporters," | grep -q ",gitea_committime," && ENABLE_GITEA_COM_EXP=true && echo "Enabling Gitea committime exporter"
    echo ",$enable_exporters," | grep -q ",bitbucket_committime," && ENABLE_BITBUCKET_COM_EXP=true && echo "Enabling Bitbucket committime exporter"
    echo ",$enable_exporters," | grep -q ",jira_committime," && ENABLE_JIRA_FAIL_EXP=true && echo "Enabling JIRA failure exporter"
    echo ",$enable_exporters," | grep -q ",jira_custom_committime," && ENABLE_JIRA_CUSTOM_FAIL_EXP=true && echo "Enabling JIRA custom failure exporter"
fi

if [ -z "${demo_branch}" ]; then
    demo_branch="master"
fi

if [ -z "${demo_org}" ]; then
    demo_org="konveyor"
fi

if [ -z "${ci_filename}" ]; then
    ci_filename="values.yaml"
fi


### MAIN
# Create download directory
DWN_DIR=$(TMPDIR="${VENV}" mktemp -d -t "${TMP_DIR_PREFIX}XXXXX") || exit 2

echo "Temporary directory created: ${DWN_DIR}"

# Cleanup download directory on exit
trap 'cleanup_and_exit $?' INT TERM EXIT

download_file_from_url "https://raw.githubusercontent.com/$demo_org/mig-demo-apps/$demo_branch/apps/todolist-mongo-go/pelorus/$ci_filename" "ci_values.yaml" "${DWN_DIR}"
download_file_from_url "https://raw.githubusercontent.com/$demo_org/mig-demo-apps/$demo_branch/apps/todolist-mongo-go/mongo-persistent.yaml" "mongo-persistent.yaml" "${DWN_DIR}"

# Create namespace where pelorus and grafana, prometheus operators will get deployed
oc create namespace "${PELORUS_NAMESPACE}"

# Modify downloaded files
sed -i.bak "s/your_org/$demo_org/g" "${DWN_DIR}/mongo-persistent.yaml"

# Show what has been modified:
diff -uNr "${DWN_DIR}/mongo-persistent.yaml" "${DWN_DIR}/mongo-persistent.yaml.bak"

# Used to create Values Helm file for use with deployment
# that is passed together with other values files as additional
# --values <path_to_thanos_config> parameter
#
# Arguments:
#    $1 - S3 compatible bucket access point
#    $2 - S3 compatible bucket name
#    $3 - S3 compatible bucket access key
#    $4 - S3 compatible bucket secret access key
#
# Return:
#    path to the temporary Helm file that allows to enable Thanos
#
function s3_thanos_tmp() {
    TMP_FILE=$(TMPDIR="${DWN_DIR}" mktemp -t "XXXXX.thanos.pelorus.yaml")
    local bucket_access_point=$1
    local thanos_bucket_name=$2
    local bucket_access_key=$3
    local bucket_secret_access_key=$4

cat <<EOF >> "${TMP_FILE}"
thanos_bucket_name: $thanos_bucket_name
bucket_access_point: $bucket_access_point
bucket_access_key: $bucket_access_key
bucket_secret_access_key: $bucket_secret_access_key
EOF

    echo "${TMP_FILE}"
}

function create_s3_thanos_config() {
    local env_s3_host=$1
    local env_s3_bucket=$2
    local env_s3_access_key=$3
    local env_s3_secret_key=$4

    thanos_config_filepath=""
    # Either pass everything as env variables or as files inside s3 secret folder
    if [[ -n ${!env_s3_host} ]] && [[ -n ${!env_s3_bucket} ]] && \
       [[ -n ${!env_s3_access_key} ]] && [[ -n ${!env_s3_secret_key} ]]; then
         echo "Getting S3 bucket credentials and access data from the env variables" 1>&2
         thanos_config_filepath=$( s3_thanos_tmp "${!env_s3_host}" "${!env_s3_bucket}" "${!env_s3_access_key}" "${!env_s3_secret_key}" )
    elif [ -r "${S3_SECRETS_DIR}/${env_s3_host}" ] && [ -r "${S3_SECRETS_DIR}/${env_s3_bucket}" ] && \
         [ -r "${S3_SECRETS_DIR}/${env_s3_access_key}" ] && [ -r "${S3_SECRETS_DIR}/${env_s3_secret_key}" ]; then
         echo "Getting S3 bucket credentials and access data from the ${S3_SECRETS_DIR} folder" 1>&2
         thanos_config_filepath=$( s3_thanos_tmp "$( cat "${S3_SECRETS_DIR}/${env_s3_host}" )" "$( cat "${S3_SECRETS_DIR}/${env_s3_bucket}" )" \
                               "$( cat "${S3_SECRETS_DIR}/${env_s3_access_key}" )" "$( cat "${S3_SECRETS_DIR}/${env_s3_secret_key}" )" )
    else
        echo "ERROR: Thanos could not be used, due to missing secret values, exiting..." 1>&2
        echo "${thanos_config_filepath}"
        exit 1
    fi

    # Ensure path is set and file is readable
    if [[ -n "${thanos_config_filepath}" ]] && [ -r "${thanos_config_filepath}" ]; then
        echo "${thanos_config_filepath}"
    else
        echo "ERROR: Thanos config file can not be read, exiting..." 1>&2
        exit 1
    fi
}

# Used to create secret temporary file that is used with oc apply
# command. This is to ensure API Tokens are hidden from the terminal
# output.
#
# Arguments:
#    $1 - namespace for which secret will be applied, e.g. pelorus
#    $2 - secret name, e.g. github-secret
#    $3 - api token value
#    $4 - (Optional) - api user name. Not all API tokens require corresponding user.
#
# Return:
#    path to the temporary secret file
#
function mksecret_temp() {
    TMP_FILE=$(TMPDIR="${DWN_DIR}" mktemp -t "XXXXX.pelorus.yaml")
    local secret_namespace=$1
    local secret_name=$2
    local api_token=$3
    local api_user=$4

# We store values under multiple value names to satisfy different exporter types.
# e.g. API_USER is used in the failure exporter while USER in commit time
cat <<EOF >> "${TMP_FILE}"
apiVersion: v1
kind: Secret
metadata:
  name: $secret_name
  namespace: $secret_namespace
type: Opaque
stringData:
  API_TOKEN: $api_token
  TOKEN: $api_token
EOF

    if [[ -n ${api_user} ]]; then
        echo "  USER: $api_user" >> "${TMP_FILE}"
        echo "  API_USER: $api_user" >> "${TMP_FILE}"
    fi

    echo "${TMP_FILE}"
}

function create_k8s_secret() {
    local secret_name=$1
    local env_token_name=$2
    local env_user_name="${3:-NO_USERNAME_DEFINED_VAR}"

    SECRET_CONFIGURED=false
    oc -n $PELORUS_NAMESPACE get secret "${secret_name}"
    secret_present=$?
    if [[ $secret_present = 0 ]]; then
        echo "Secret $secret_name was found"
        SECRET_CONFIGURED=true
    else
      echo "Secret $secret_name was not found, checking env variables to create one"
      secret_filepath=""
      # TOKEN must always be set, USER may be required depending on the backend type
      if [[ -n ${!env_token_name} ]] && [[ -n ${!env_user_name} ]]; then
          echo "API Token and user passed from env var"
          secret_filepath=$( mksecret_temp "$PELORUS_NAMESPACE" "$secret_name" "${!env_token_name}" "${!env_user_name}" )
      elif [[ -n ${!env_token_name} ]]; then
          echo "API Token passed from env var"
          secret_filepath=$( mksecret_temp "$PELORUS_NAMESPACE" "$secret_name" "${!env_token_name}" )
      elif [ -r "${SECRETS_DIR}/${env_token_name}" ] && [ -r "${SECRETS_DIR}/${env_user_name}" ]; then
          echo "Getting API TOKEN and API USER from secrets mounted directory"
          secret_filepath=$( mksecret_temp "$PELORUS_NAMESPACE" "$secret_name" "$( cat "${SECRETS_DIR}/${env_token_name}" )" "$( cat "${SECRETS_DIR}/${env_user_name}" )" )
      elif [ -r "${SECRETS_DIR}/${env_token_name}" ]; then
          echo "Getting API TOKEN from secrets mounted directory"
          secret_filepath=$( mksecret_temp "$PELORUS_NAMESPACE" "$secret_name" "$( cat "${SECRETS_DIR}/${env_token_name}" )" )
      else
          echo "ERROR: API Token for ${secret_name} not provided, exiting..."
          exit 1
      fi
      # Ensure path is set and file is readable
      if [[ -n "${secret_filepath}" ]] && [ -r "${secret_filepath}" ]; then
          oc apply -f "${secret_filepath}"
          oc_apply=$?
          if [[ $oc_apply = 0 ]]; then
              echo "Secret $secret_name was added"
              SECRET_CONFIGURED=true
          fi
          rm "${secret_filepath}"
      fi
    fi
    echo "${SECRET_CONFIGURED}"
}

# enable the pelorus failure exporter for github
if [ "${ENABLE_FAIL_EXP}" == true ]; then
    secret_configured=$(create_k8s_secret github-secret TOKEN)
    secret_exit=$?
    echo "${secret_configured}"
    if [[ $secret_exit != 0 ]]; then
        exit 1
    fi

    # uncomment the failure exporter in ci_values.yaml
    if [[ "$secret_configured" == *true ]]; then
        sed -i.bak "s/#@//g" "${DWN_DIR}/ci_values.yaml"
        echo "The pelorus failure exporter has been enabled"
    fi

    # if required update the failure issue github organization
    if [ "$demo_org" != "konveyor" ]; then
        sed -i.bak "s/konveyor\/mig-demo-apps/$demo_org\/mig-demo-apps/g" "${DWN_DIR}/ci_values.yaml"
    fi
fi

# enable gitlab committime exporter
if [ "${ENABLE_GITLAB_COM_EXP}" == true ]; then
    secret_configured=$(create_k8s_secret gitlab-secret GITLAB_API_TOKEN)
    secret_exit=$?
    echo "${secret_configured}"
    if [[ $secret_exit != 0 ]]; then
        exit 1
    fi
    # uncomment the gitlab committime exporter in ci_values.yaml
    if [[ "$secret_configured" == *true ]]; then
        sed -i.bak "s/#gitlab-committime@//g" "${DWN_DIR}/ci_values.yaml"
        echo "The pelorus gitlab committime exporter has been enabled"
    fi

    # namespace must much one from ci_values.yaml
    build_ns="gitlab-binary"
    BINARY_BUILDS_NAMESPACES+=("${build_ns}")
    build_uri="https://gitlab.com/mpryc/pelorus-gitlab"
    build_hash="b807fd8e1b2bd1755eca14960e5352fe89ff466e"
    "${BINARY_BUILD_SCRIPT}" -n "${build_ns}" -b "${build_ns}-todolist" -u "${build_uri}" -s "${build_hash}"
fi

# enable gitea committime exporter
if [ "${ENABLE_GITEA_COM_EXP}" == true ]; then
    secret_configured=$(create_k8s_secret gitea-secret GITEA_API_TOKEN)
    secret_exit=$?
    echo "${secret_configured}"
    if [[ $secret_exit != 0 ]]; then
        exit 1
    fi
    # uncomment the gitea committime exporter in ci_values.yaml
    if [[ "$secret_configured" == *true ]]; then
        sed -i.bak "s/#gitea-committime@//g" "${DWN_DIR}/ci_values.yaml"
        echo "The pelorus gitea committime exporter has been enabled"
    fi

    # namespace must much one from ci_values.yaml
    build_ns="gitea-binary"
    BINARY_BUILDS_NAMESPACES+=("${build_ns}")
    build_uri="https://try.gitea.io/mpryc/pelorus-gitea"
    build_hash="897f5c490442e88cc609c241da880128ad02e42b"
    "${BINARY_BUILD_SCRIPT}" -n "${build_ns}" -b "${build_ns}-todolist" -u "${build_uri}" -s "${build_hash}"
fi

# enable bitbucket committime exporter
if [ "${ENABLE_BITBUCKET_COM_EXP}" == true ]; then
    secret_configured=$(create_k8s_secret bitbucket-secret BITBUCKET_API_TOKEN BITBUCKET_API_USER)
    secret_exit=$?
    echo "${secret_configured}"    
    if [[ $secret_exit != 0 ]]; then
        exit 1
    fi
    # uncomment the bitbucket committime exporter in ci_values.yaml
    if [[ "$secret_configured" == *true ]]; then
        sed -i.bak "s/#bitbucket-committime@//g" "${DWN_DIR}/ci_values.yaml"
        echo "The pelorus bitbucket committime exporter has been enabled"
    fi

    # namespace must much one from ci_values.yaml
    build_ns="bitbucket-binary"
    BINARY_BUILDS_NAMESPACES+=("${build_ns}")
    build_uri="https://bitbucket.org/michalpryc/pelorus-bitbucket"
    build_hash="e2c4ef00468dfc10aad1bd2d4c9d470160a7f471"
    "${BINARY_BUILD_SCRIPT}" -n "${build_ns}" -b "${build_ns}-todolist" -u "${build_uri}" -s "${build_hash}"
fi

# enable JIRA failure exporter
if [ "${ENABLE_JIRA_FAIL_EXP}" == true ]; then
    secret_configured=$(create_k8s_secret jira-secret JIRA_TOKEN JIRA_USER)
    secret_exit=$?
    echo "${secret_configured}"
    if [[ $secret_exit != 0 ]]; then
        exit 1
    fi
    # uncomment the jira failure exporter in ci_values.yaml
    if [[ "$secret_configured" == *true ]]; then
        sed -i.bak "s/#jira-failure@//g" "${DWN_DIR}/ci_values.yaml"
        echo "The pelorus JIRA failure exporter has been enabled"
    fi
fi

# enable JIRA with custom query failure exporter
# Corresponding JIRA card to test against
# https://pelorustest.atlassian.net/jira/core/projects/FIRST/board?selectedIssue=FIRST-14
if [ "${ENABLE_JIRA_CUSTOM_FAIL_EXP}" == true ]; then
    secret_configured=$(create_k8s_secret jira-secret JIRA_TOKEN JIRA_USER)
    secret_exit=$?
    echo "${secret_configured}"
    if [[ $secret_exit != 0 ]]; then
        exit 1
    fi
    # uncomment the jira failure exporter in ci_values.yaml
    if [[ "$secret_configured" == *true ]]; then
        sed -i.bak "s/#jira-custom-failure@//g" "${DWN_DIR}/ci_values.yaml"
        echo "The pelorus JIRA custom failure exporter has been enabled"
    fi
fi

# We do check for the exit status, as we are not really interested in the
# current state, e.g. Active of that namespace before deleting resources.
if oc get namespace mongo-persistent 2>/dev/null; then
    oc delete -f "${DWN_DIR}/mongo-persistent.yaml"
fi

# Delete mongo-persistent-scc (securitycontextconstraints.security.openshift.io)
# which may be left from previous runs.
if oc get scc mongo-persistent-scc 2>/dev/null; then
    oc delete scc mongo-persistent-scc
fi

# From now on, exit if something goes wrong
set -e

# If this is a pull request from pelorus
if [ "${REPO_NAME}" == "pelorus" ]; then
    # Check if PULL_NUMBER exists and it's actual number
    if [ ${PULL_NUMBER+x} ] && [[ $PULL_NUMBER =~ ^[0-9]+$ ]]; then
        echo "Provided PULL_NUMBER: '$PULL_NUMBER'"
        sed -i "s/source_ref:.*/source_ref: refs\/pull\/${PULL_NUMBER}\/head/" "${DWN_DIR}/ci_values.yaml"
    fi
fi

# Ensure we are in the top-level directory of pelorus project
pushd "${SCRIPT_DIR}/../"

# Apply config maps for the exporters
# disable for now, as it's divergent from the install instructions.
# oc apply -f "charts/pelorus/configmaps"

helm install operators charts/operators --namespace pelorus --debug --wait --wait-for-jobs

# Wait for grafana and prometheus deployments to be rolled out
retry 5m 1s ornds prometheus-operator
retry 5m 1s ornds grafana-operator-controller-manager

helm install pelorus charts/pelorus --namespace pelorus --debug --wait --wait-for-jobs

# check final deployment
retry 5m 5s ogns grafana-service
retry 5m 5s ogns prometheus-operated
retry 5m 5s ogns prometheus-pelorus

# Print ci_values.yaml for easy debug
cat "${DWN_DIR}/ci_values.yaml"

# Thanos and s3 support
THANOS_HELM_FLAG=""
if [ "${ENABLE_THANOS}" == true ]; then
    echo "Enabling Thanos support"
    thanos_config_file=$(create_s3_thanos_config  s3_host s3_bucket s3_access_key s3_secret_key)
    THANOS_HELM_FLAG=" --values ${thanos_config_file}"
    echo "${THANOS_HELM_FLAG}"
fi

# update exporter values and helm upgrade
eval helm upgrade pelorus charts/pelorus --namespace pelorus --values "${DWN_DIR}/ci_values.yaml" "${THANOS_HELM_FLAG}"

retry 10m 5s owpr deploytime
retry 10m 5s owpr committime

oc create -f "${DWN_DIR}/mongo-persistent.yaml"

retry 2m 5s oc wait pod --for=condition=Ready -n mongo-persistent -l app=mongo
retry 10m 10s oc wait pod --for=condition=Ready -n mongo-persistent -l app=todolist

# Test all deployed exporters
exporters=$(oc get route -n "${PELORUS_NAMESPACE}"|grep "-exporter")
echo "$exporters"
for exporter_route in $(echo "$exporters" | awk '{print $2}');
do
    echo "$exporter_route"
    curl "$exporter_route"
    curl "$exporter_route" 2>&1 | grep todolist || exit 2
done

if oc get pods -n pelorus | grep -q Crash ; then
    echo "Some pods are not functioning properly"
    oc get pods -n pelorus
    exit 1
fi

# Validate Thanos, by querying thanos-query with the known data, 20 years from now should be sufficient and should
# not be greater then this CI setup ;)
if [ "${ENABLE_THANOS}" == true ]; then
    THANOS_QUERY_HOST=$(oc get route thanos-pelorus -n pelorus --no-headers -o custom-columns="HOST:spec.host")
    echo "Thanos query host: $THANOS_QUERY_HOST"
    DEPLOY_NO=$(curl -u internal:changeme -k --data-urlencode 'query=count(count_over_time(deploy_timestamp{app="todolist"}[20y]))' "https://${THANOS_QUERY_HOST}/api/v1/query" | jq -r '.data.result[] | [.value]' | grep \" | tr -dc '0-9')
    echo "Deploy count: $DEPLOY_NO"
    if [ "$DEPLOY_NO" -lt "3" ]; then
        echo "Thanos and s3 data is not as expected"
        exit 1
    fi
fi