This message was deleted.
# atlantis-community
s
This message was deleted.
t
Helm upgrade failed: values don't meet the specifications of the schema(s) in the following chart(s): atlantis: - (root): Additional property annotations is not allowed
g
@רועי טבת FYI
t
If i can get some time I can help investigate what the root is, but got other job stuff to knock out 🙂
just wanted at least to bring it up
u
Hi @Tyler Allen by any chance can you share with me your values? you are using helm install directly or a system like Flux or ArgoCD?
t
Flux
sure buddy, let me santize some of the things and i can post it here no problemo
you want it json formated?
u
Great 🙂 we are also deploying with flux and i checked its all good
t
sweet yeah, it'd been easier for me to figure out if the error was a bit more explicit
u
Json is good
t
Copy code
{
  "values": {
    "volumeClaim": {
      "enabled": true,
      "dataStorage": "100Gi"
    },
    "image": {
      "repository": "redacted",
      "tag": "latest",
      "pullPolicy": "Always"
    },
    "atlantisUrl": "<https://redacted>",
    "orgAllowlist": "redacted/engineering/*",
    "tolerations": [
      {
        "key": "onDemand",
        "operator": "Equal",
        "value": "true",
        "effect": "NoSchedule"
      }
    ],
    "nodeSelector": {
      "<http://karpenter.sh/capacity-type|karpenter.sh/capacity-type>": "on-demand"
    },
    "allowDraftPRs": true,
    "enableDiffMarkdownFormat": true,
    "ingress": {
      "enabled": true,
      "ingressClassName": "main-class",
      "annotations": {
        "<http://cert-manager.io/cluster-issuer|cert-manager.io/cluster-issuer>": "letsencrypt-prod",
        "<http://external-dns.alpha.kubernetes.io/hostname|external-dns.alpha.kubernetes.io/hostname>": "atlantis.redacted",
        "<http://alb.ingress.kubernetes.io/group.name|alb.ingress.kubernetes.io/group.name>": "main-devops-eks",
        "<http://alb.ingress.kubernetes.io/listen-ports|alb.ingress.kubernetes.io/listen-ports>": "[{\"HTTPS\":443}]",
        "<http://alb.ingress.kubernetes.io/target-type|alb.ingress.kubernetes.io/target-type>": "ip"
      },
      "path": "/",
      "pathType": "Prefix",
      "host": "atlantis.redacted"
    },
    "resources": {
      "requests": {
        "memory": "4Gi",
        "cpu": "8096m"
      },
      "limits": {
        "memory": "4Gi",
        "cpu": "8096m"
      }
    },
    "serviceAccount": {
      "create": true,
      "mount": true,
      "name": "atlantis",
      "annotations": {
        "<http://eks.amazonaws.com/role-arn|eks.amazonaws.com/role-arn>": "redacted"
      }
    },
    "environment": {
      "ATLANTIS_ALLOW_COMMANDS": "version,plan,apply,unlock,approve_policies,import",
      "ATLANTIS_HIDE_PREV_PLAN_COMMENTS": "true",
      "ATLANTIS_WRITE_GIT_CREDS": "true",
      "TF_INPUT": "false",
      "ATLANTIS_ENABLE_POLICY_CHECKS": "false",
      "ATLANTIS_GITLAB_USER": "atlantis",
      "ATLANTIS_LOG_LEVEL": "info",
      "ATLANTIS_CONFIG": "/home/atlantis/config.yaml",
      "ATLANTIS_PARALLEL_POOL_SIZE": "6",
      "ATLANTIS_ALLOW_DRAFT_PRS": "false",
      "TF_CLI_ARGS_plan": "-compact-warnings",
      "ATLANTIS_GITLAB_HOSTNAME": "redacted",
      "TF_CLI_ARGS_APPLY": "-compact-warnings",
      "ATLANTIS_DEFAULT_TF_VERSION": "1.4.7",
      "ATLANTIS_ENABLE_DIFF_MARKDOWN_FORMAT": "true",
      "ATLANTIS_PORT": "4141",
      "TG_STATE_ACCESS_ROLE": "redacted"
    },
    "environmentSecrets": [
      {
        "name": "ATLANTIS_API_SECRET",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "atlantis_api_secret"
        }
      },
      {
        "name": "ATLANTIS_SLACK_TOKEN",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "slack_token"
        }
      },
      {
        "name": "SSH_KEY",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "ssh_key"
        }
      },
      {
        "name": "TG_TF_REGISTRY_TOKEN",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "gitlab_token"
        }
      },
      {
        "name": "GITLAB_TOKEN",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "gitlab_token"
        }
      },
      {
        "name": "ATLANTIS_GITLAB_TOKEN",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "gitlab_token"
        }
      },
      {
        "name": "ATLANTIS_GITLAB_WEBHOOK_SECRET",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "webhook_secret"
        }
      },
      {
        "name": "SUMOLOGIC_ACCESSID",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "sumo_accessid"
        }
      },
      {
        "name": "SUMOLOGIC_ACCESSKEY",
        "secretKeyRef": {
          "name": "atlantis-secrets",
          "key": "sumo_accesskey"
        }
      }
    ],
    "config": "---\nwebhooks:\n  - event: apply\n    workspace-regex: .*\n    kind: slack\n    channel: tool-atlantis\n",
    "repoConfig": "---\nrepos:\n  - id: \"/.*/\"\n    allowed_overrides: [apply_requirements, workflow]\n    apply_requirements: []\n    workflow: terragrunt\n    pre_workflow_hooks:\n      - run: rm -rf /tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM\n      - run: mkdir -p /tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM            \n      - run: \"terragrunt-atlantis-config generate --output atlantis.yaml --create-workspace --ignore-parent-terragrunt --ignore-dependency-blocks --parallel --num-executors 6 --terraform-version 1.4.7 && python3 utilities/atlantis-config-filtering.py\"\n    post_workflow_hooks:\n      - run: |\n          infracost comment gitlab --repo $BASE_REPO_OWNER/$BASE_REPO_NAME \\\n                                  --merge-request $PULL_NUM \\\n                                  --path /tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM/'*'-infracost.json \\\n                                  --gitlab-token $GITLAB_TOKEN \\\n                                  --gitlab-server-url \"redacted\" \\\n                                  --behavior new || true\npolicies:\n  owners:\n    users:\n    - redacted\n  policy_sets:\n    - name: default\n      # Built into the container at this time likely will go to its own repo for dynamic pull/updates without rebuilding the container\n      path: /home/atlantis/conftest_policies/terraform\n      source: local\nworkflows:\n\n  terragrunt:\n    import:\n      steps:\n        - env:\n            name: TERRAGRUNT_TFPATH\n            command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n        - run: COMMENT_ARGS=${COMMENT_ARGS//\\\\} && terragrunt import $COMMENT_ARGS\n    plan:\n      steps:\n        - run: infracost configure set api_key $INFRACOST_TOKEN\n        - env:\n            name: INFRACOST_OUTPUT\n            command: 'echo \"/tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM/$WORKSPACE-${REPO_REL_DIR//\\//-}-infracost.json\"'\n        - env:\n            name: TERRAGRUNT_TFPATH\n            command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n        - run: echo \"Clearing caches\"\n        - run: find $(git rev-parse --show-toplevel) -type d -name '.terragrunt-cache' | xargs rm -rf\n        - run: COMMENT_ARGS=${COMMENT_ARGS//\\\\} && terragrunt plan $COMMENT_ARGS -out=$PLANFILE --terragrunt-source-update \n        - run: terragrunt show -json $PLANFILE > $SHOWFILE\n        - run: |\n            if [ ! -d \"/tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM\" ]; then\n              mkdir -p /tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM\n            fi\n\n            infracost breakdown --path=$SHOWFILE \\\n                                --format=json \\\n                                --log-level=info \\\n                                --out-file=$INFRACOST_OUTPUT \\\n                                --project-name=$REPO_REL_DIR\n    # policy_check:\n    #   steps:\n    #     - env:\n    #         name: TERRAGRUNT_TFPATH\n    #         command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n    #     - run: \"terragrunt show -json $PLANFILE > ${PLANFILE}.json\"\n    #     - run: \"conftest test ${PLANFILE}.json -p /home/atlantis/conftest_policies/terraform/ --update redacted\" \n    apply:\n      steps:\n        - env:\n            name: TERRAGRUNT_TFPATH\n            command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n        - run: \"terragrunt apply --terragrunt-source-update $PLANFILE\" \n\n  terragrunt-all:\n    import:\n      steps:\n        - env:\n            name: TERRAGRUNT_TFPATH\n            command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n        - run: COMMENT_ARGS=${COMMENT_ARGS//\\\\} && terragrunt import $COMMENT_ARGS\n    plan:\n      steps:\n        - run: infracost configure set api_key $INFRACOST_TOKEN\n        - env:\n            name: INFRACOST_OUTPUT\n            command: 'echo \"/tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM/$WORKSPACE-${REPO_REL_DIR//\\//-}-infracost.json\"'\n        - env:\n            name: TERRAGRUNT_TFPATH\n            command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n        - run: COMMENT_ARGS=${COMMENT_ARGS//\\\\} && terragrunt run-all plan -no-color --terragrunt-non-interactive $COMMENT_ARGS -out=$PLANFILE --terragrunt-source-update\n        - run: terragrunt show -json $PLANFILE > $SHOWFILE\n        - run: |\n            if [ ! -d \"/tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM\" ]; then\n              mkdir -p /tmp/$BASE_REPO_OWNER-$BASE_REPO_NAME-$PULL_NUM\n            fi\n            \n            infracost breakdown --path=$SHOWFILE \\\n                                --format=json \\\n                                --log-level=info \\\n                                --out-file=$INFRACOST_OUTPUT \\\n                                --project-name=$REPO_REL_DIR\n    apply:\n      steps:\n        - env:\n            name: TERRAGRUNT_TFPATH\n            command: 'echo \"terraform${ATLANTIS_TERRAFORM_VERSION}\"'\n        - run: \"terragrunt run-all apply -no-color --terragrunt-non-interactive --terragrunt-source-update $PLANFILE\"\n"
  }
}
This was the official helm error
Copy code
- lastTransitionTime: "2023-11-10T14:04:03Z"
    message: |-
      Helm upgrade failed: values don't meet the specifications of the schema(s) in the following chart(s):
      atlantis:
      - (root): Additional property annotations is not allowed


      Last Helm logs:

      preparing upgrade for atlantis
      resetting values to the chart's original version
    reason: UpgradeFailed
u
ok so the values are indeed going fine through schema validation which make me thing flux is injecting something even tough my helmrelease is working well can i please also get the helmrelease definition? @Tyler Allen
t
oh yeah
100%
Copy code
apiVersion: <http://helm.toolkit.fluxcd.io/v2beta1|helm.toolkit.fluxcd.io/v2beta1>
kind: HelmRelease
metadata:
  annotations:
    <http://reconcile.fluxcd.io/requestedAt|reconcile.fluxcd.io/requestedAt>: 2023-11-10T14:42:12.464577846Z
  creationTimestamp: 2023-10-03T17:45:32Z
  finalizers:
    - <http://finalizers.fluxcd.io|finalizers.fluxcd.io>
  generation: 19
  labels:
    <http://kustomize.toolkit.fluxcd.io/name|kustomize.toolkit.fluxcd.io/name>: apps
    <http://kustomize.toolkit.fluxcd.io/namespace|kustomize.toolkit.fluxcd.io/namespace>: flux-system
  managedFields:
    - apiVersion: <http://helm.toolkit.fluxcd.io/v2beta1|helm.toolkit.fluxcd.io/v2beta1>
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            f:<http://kustomize.toolkit.fluxcd.io/name|kustomize.toolkit.fluxcd.io/name>: {}
            f:<http://kustomize.toolkit.fluxcd.io/namespace|kustomize.toolkit.fluxcd.io/namespace>: {}
        f:spec:
          f:chart:
            f:spec:
              f:chart: {}
              f:sourceRef:
                f:kind: {}
                f:name: {}
                f:namespace: {}
              f:version: {}
          f:interval: {}
          f:values:
            f:allowDraftPRs: {}
            f:annotations:
              .: {}
              f:<http://cert-manager.io/cluster-issuer|cert-manager.io/cluster-issuer>: {}
              f:<http://external-dns.alpha.kubernetes.io/hostname|external-dns.alpha.kubernetes.io/hostname>: {}
            f:atlantisUrl: {}
            f:config: {}
            f:enableDiffMarkdownFormat: {}
            f:environment:
              .: {}
              f:ATLANTIS_ALLOW_COMMANDS: {}
              f:ATLANTIS_ALLOW_DRAFT_PRS: {}
              f:ATLANTIS_CONFIG: {}
              f:ATLANTIS_DEFAULT_TF_VERSION: {}
              f:ATLANTIS_ENABLE_DIFF_MARKDOWN_FORMAT: {}
              f:ATLANTIS_ENABLE_POLICY_CHECKS: {}
              f:ATLANTIS_GITLAB_HOSTNAME: {}
              f:ATLANTIS_GITLAB_USER: {}
              f:ATLANTIS_HIDE_PREV_PLAN_COMMENTS: {}
              f:ATLANTIS_LOG_LEVEL: {}
              f:ATLANTIS_PARALLEL_POOL_SIZE: {}
              f:ATLANTIS_PORT: {}
              f:ATLANTIS_WRITE_GIT_CREDS: {}
              f:TF_CLI_ARGS_APPLY: {}
              f:TF_CLI_ARGS_plan: {}
              f:TF_INPUT: {}
              f:TG_STATE_ACCESS_ROLE: {}
            f:environmentSecrets: {}
            f:image:
              .: {}
              f:pullPolicy: {}
              f:repository: {}
              f:tag: {}
            f:ingress:
              .: {}
              f:annotations:
                .: {}
                f:<http://alb.ingress.kubernetes.io/group.name|alb.ingress.kubernetes.io/group.name>: {}
                f:<http://alb.ingress.kubernetes.io/listen-ports|alb.ingress.kubernetes.io/listen-ports>: {}
                f:<http://alb.ingress.kubernetes.io/target-type|alb.ingress.kubernetes.io/target-type>: {}
                f:<http://cert-manager.io/cluster-issuer|cert-manager.io/cluster-issuer>: {}
                f:<http://external-dns.alpha.kubernetes.io/hostname|external-dns.alpha.kubernetes.io/hostname>: {}
              f:enabled: {}
              f:<http://external-dns.alpha.kubernetes.io/hostname|external-dns.alpha.kubernetes.io/hostname>: {}
              f:host: {}
              f:ingressClassName: {}
              f:path: {}
              f:pathType: {}
            f:nodeSelector:
              .: {}
              f:<http://karpenter.sh/capacity-type|karpenter.sh/capacity-type>: {}
            f:orgAllowlist: {}
            f:repoConfig: {}
            f:resources:
              .: {}
              f:limits:
                .: {}
                f:cpu: {}
                f:memory: {}
              f:requests:
                .: {}
                f:cpu: {}
                f:memory: {}
            f:serviceAccount:
              .: {}
              f:annotations:
                .: {}
                f:<http://eks.amazonaws.com/role-arn|eks.amazonaws.com/role-arn>: {}
              f:create: {}
              f:mount: {}
              f:name: {}
            f:tolerations: {}
            f:volumeClaim:
              .: {}
              f:dataStorage: {}
              f:enabled: {}
      manager: kustomize-controller
      operation: Apply
      time: 2023-11-10T14:45:50Z
    - apiVersion: <http://helm.toolkit.fluxcd.io/v2beta1|helm.toolkit.fluxcd.io/v2beta1>
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:finalizers:
            .: {}
            v:"<http://finalizers.fluxcd.io|finalizers.fluxcd.io>": {}
      manager: helm-controller
      operation: Update
      time: 2023-10-03T17:45:32Z
    - apiVersion: <http://helm.toolkit.fluxcd.io/v2beta1|helm.toolkit.fluxcd.io/v2beta1>
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:<http://reconcile.fluxcd.io/requestedAt|reconcile.fluxcd.io/requestedAt>: {}
      manager: gitops-server
      operation: Update
      time: 2023-11-10T14:42:12Z
    - apiVersion: <http://helm.toolkit.fluxcd.io/v2beta1|helm.toolkit.fluxcd.io/v2beta1>
      fieldsType: FieldsV1
      fieldsV1:
        f:status:
          f:conditions: {}
          f:helmChart: {}
          f:lastAppliedRevision: {}
          f:lastAttemptedRevision: {}
          f:lastAttemptedValuesChecksum: {}
          f:lastHandledReconcileAt: {}
          f:lastReleaseRevision: {}
          f:observedGeneration: {}
      manager: helm-controller
      operation: Update
      subresource: status
      time: 2023-11-10T14:47:54Z
  name: atlantis
  namespace: atlantis
  resourceVersion: "130212982"
  uid: c589877c-aa4b-48cd-8a42-b4aa3861083c
  spec:
  chart:
    spec:
      chart: atlantis
      reconcileStrategy: ChartVersion
      sourceRef:
        kind: HelmRepository
        name: atlantis
        namespace: atlantis
      version: 4.16.0
  interval: 1m0s
  values:
    <VALUES_FROM_ABOVE>
status:
  conditions:
    - lastTransitionTime: 2023-11-10T14:47:54Z
      message: Release reconciliation succeeded
      reason: ReconciliationSucceeded
      status: "True"
      type: Ready
    - lastTransitionTime: 2023-11-10T14:47:54Z
      message: Helm upgrade succeeded
      reason: UpgradeSucceeded
      status: "True"
      type: Released
  helmChart: atlantis/atlantis-atlantis
  lastAppliedRevision: 4.16.0
  lastAttemptedRevision: 4.16.0
  lastAttemptedValuesChecksum: 98ec38fa82108ba6bdd7691ae8e07484cbdf8814
  lastHandledReconcileAt: 2023-11-10T14:42:12.464577846Z
  lastReleaseRevision: 29
  observedGeneration: 19
I didn't see anything Jarring, when you said that, i did think that potentially it was doing a field merge into values
u
I found something weird here @Tyler Allen
somehow it got annotations field
t
ooo
u
you have another values file?
t
let me check could be a bad patch
via kustomize checking it out
u
might be got it
t
im embarassed
that was exactly what it was
You da bes
silly yaml indentation issue
however your json schema fixed another problem for us with bool/vs string value so thats $$$$
u
on that subject @Tyler Allen what do you thing about the schemaValidation with additionalProperties false my thinking here is if you have something “extra” i want you to know it and not let it pass through WDYT?
t
Yeah that'd be super nice
Now that I go back and look at the error it told me exactly what it was
u
And @Tyler Allen We are really much a like flux + atlantis + terragrunt + infracost 🙂 nice to see that
t
but i equated property annotations as a thing and not the actual property "annotations"
Yeah its awesome seeing how other people are using it for sure!
Im always on the lookout for sweet terragrunt handling items
u
Talked with my team last week that i cannot imagine going back
t
we are revamping our terragrunt structure as a whole, one thing i am gonna try to tackle is having a atlantis run-all plan
not sure how im gonna do it yet but i will do it, but to your point our Developer adoption of shared infra has skyrocketed
u
Nice!
im really afraid of terragrunt apply-all as its not asking and just applyingggg
t
we use atlantis-config-generate and another thing i'd like to tackle is not having gitlab going down because atlantis generates 100s of pipeline jobs due to dependency finds
but luckily we pipe that to a python script that cleans/controls the atlantis.yaml file it outputs
u
mmm nice also using the terragrunt-atlantis-config but i not seeing the issue you are facing
t
so lets say, we have a VPC module and like 90 other things depend on that, it will cause atlantis to say Im going to plan those other 90 things, which i mean in theory you want but it has caused us issues
u
got it we are probably much smaller
t
def something to think about however the atlantis extra_dependencies lets us be explicit with dependencies on things that are a MUST to go so it is nice
Copy code
locals {
  extra_atlantis_dependencies = [
    "../../../vpc/terragrunt.hcl"
  ]
}
as an update im now using 4.17 helm chart post changes, I really appreciate the extra set of eyes
u
Great to hear! and also i’ll be happy to keep on touch about atlantis and infra 🙂 its nice to connect with someone with the same stack
t
Yes please, i was about to say the same thing
ill DM you on the side that way we can keep in touch
p
What happened here? what is the TLDR:
is there something I need to do ?
revert the PR?
t
Hey PePe! Nothing is wrong we had a kustomize patch that added an invalid field against the schema
Roi helped me find it all is good and we are now using that version
p
perfect, thanks!