ASulaiman
05/22/2022, 3:09 PM"Cannot end publish with com.van.logging.gcp.CloudStoragePublishHelper@13049a9a due to error: Cannot end publishing: Invalid bucket name: '${env:GCS_LOG_BUCKET}'"
i'm not using GCS bucket and config pointing to GCS is off. this happens only on "pods airbyte-worker"
apiVersion: apps/v1
kind: Deployment
metadata:
name: airbyte-worker
spec:
replicas: 1
selector:
matchLabels:
airbyte: worker
template:
metadata:
labels:
airbyte: worker
spec:
serviceAccountName: airbyte-admin
automountServiceAccountToken: true
containers:
- name: airbyte-worker-container
image: airbyte/worker
env:
- name: AIRBYTE_VERSION
valueFrom:
configMapKeyRef:
name: airbyte-env
key: AIRBYTE_VERSION
# - name: CONFIG_ROOT
# valueFrom:
# configMapKeyRef:
# name: airbyte-env
# key: CONFIG_ROOT
- name: DATABASE_HOST
valueFrom:
configMapKeyRef:
name: airbyte-env
key: DATABASE_HOST
- name: DATABASE_PORT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: DATABASE_PORT
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: airbyte-secrets
key: DATABASE_PASSWORD
- name: DATABASE_URL
valueFrom:
configMapKeyRef:
name: airbyte-env
key: DATABASE_URL
- name: DATABASE_USER
valueFrom:
secretKeyRef:
name: airbyte-secrets
key: DATABASE_USER
- name: TRACKING_STRATEGY
valueFrom:
configMapKeyRef:
name: airbyte-env
key: TRACKING_STRATEGY
- name: WORKSPACE_DOCKER_MOUNT
value: workspace
- name: WORKSPACE_ROOT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: WORKSPACE_ROOT
- name: WORKER_ENVIRONMENT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: WORKER_ENVIRONMENT
- name: LOCAL_ROOT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: LOCAL_ROOT
- name: WEBAPP_URL
valueFrom:
configMapKeyRef:
name: airbyte-env
key: WEBAPP_URL
- name: TEMPORAL_HOST
valueFrom:
configMapKeyRef:
name: airbyte-env
key: TEMPORAL_HOST
- name: TEMPORAL_WORKER_PORTS
valueFrom:
configMapKeyRef:
name: airbyte-env
key: TEMPORAL_WORKER_PORTS
- name: LOG_LEVEL
valueFrom:
configMapKeyRef:
name: airbyte-env
key: LOG_LEVEL
- name: JOB_KUBE_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SUBMITTER_NUM_THREADS
valueFrom:
configMapKeyRef:
name: airbyte-env
key: SUBMITTER_NUM_THREADS
- name: JOB_MAIN_CONTAINER_CPU_REQUEST
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_MAIN_CONTAINER_CPU_REQUEST
- name: JOB_MAIN_CONTAINER_CPU_LIMIT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_MAIN_CONTAINER_CPU_LIMIT
- name: JOB_MAIN_CONTAINER_MEMORY_REQUEST
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_MAIN_CONTAINER_MEMORY_REQUEST
- name: JOB_MAIN_CONTAINER_MEMORY_LIMIT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_MAIN_CONTAINER_MEMORY_LIMIT
- name: S3_LOG_BUCKET
valueFrom:
configMapKeyRef:
name: airbyte-env
key: S3_LOG_BUCKET
- name: S3_LOG_BUCKET_REGION
valueFrom:
configMapKeyRef:
name: airbyte-env
key: S3_LOG_BUCKET_REGION
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: airbyte-secrets
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: airbyte-secrets
key: AWS_SECRET_ACCESS_KEY
- name: S3_MINIO_ENDPOINT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: S3_MINIO_ENDPOINT
- name: S3_PATH_STYLE_ACCESS
valueFrom:
configMapKeyRef:
name: airbyte-env
key: S3_PATH_STYLE_ACCESS
# - name: GOOGLE_APPLICATION_CREDENTIALS
# valueFrom:
# secretKeyRef:
# name: airbyte-secrets
# key: GOOGLE_APPLICATION_CREDENTIALS
# - name: GCS_LOG_BUCKET
# valueFrom:
# configMapKeyRef:
# name: airbyte-env
# key: GCS_LOG_BUCKET
- name: INTERNAL_API_HOST
valueFrom:
configMapKeyRef:
name: airbyte-env
key: INTERNAL_API_HOST
- name: JOB_KUBE_TOLERATIONS
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_KUBE_TOLERATIONS
- name: JOB_KUBE_ANNOTATIONS
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_KUBE_ANNOTATIONS
- name: JOB_KUBE_NODE_SELECTORS
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_KUBE_NODE_SELECTORS
- name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY
valueFrom:
configMapKeyRef:
name: airbyte-env
key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY
# todo: add other state storage keys
- name: STATE_STORAGE_MINIO_BUCKET_NAME
valueFrom:
configMapKeyRef:
name: airbyte-env
key: STATE_STORAGE_MINIO_BUCKET_NAME
- name: STATE_STORAGE_MINIO_ENDPOINT
valueFrom:
configMapKeyRef:
name: airbyte-env
key: STATE_STORAGE_MINIO_ENDPOINT
# - name: STATE_STORAGE_MINIO_ACCESS_KEY
# valueFrom:
# secretKeyRef:
# name: airbyte-secrets
# key: STATE_STORAGE_MINIO_ACCESS_KEY
# - name: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
# valueFrom:
# secretKeyRef:
# name: airbyte-secrets
# key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
- name: CONTAINER_ORCHESTRATOR_ENABLED
valueFrom:
configMapKeyRef:
name: airbyte-env
key: CONTAINER_ORCHESTRATOR_ENABLED
ports:
- containerPort: 9000 # for heartbeat server
- containerPort: 9001 # start temporal worker port pool
- containerPort: 9002
- containerPort: 9003
- containerPort: 9004
- containerPort: 9005
- containerPort: 9006
- containerPort: 9007
- containerPort: 9008
- containerPort: 9009
- containerPort: 9010
- containerPort: 9011
- containerPort: 9012
- containerPort: 9013
- containerPort: 9014
- containerPort: 9015
- containerPort: 9016
- containerPort: 9017
- containerPort: 9018
- containerPort: 9019
- containerPort: 9020
- containerPort: 9021
- containerPort: 9022
- containerPort: 9023
- containerPort: 9024
- containerPort: 9025
- containerPort: 9026
- containerPort: 9027
- containerPort: 9028
- containerPort: 9029
- containerPort: 9030 # end temporal worker port pool
# volumeMounts:
# - name: gcs-log-creds-volume
# mountPath: /secrets/gcs-log-creds
# readOnly: true
# volumes:
# - name: gcs-log-creds-volume
# secret:
# secretName: gcs-log-creds
Augustin Lafanechere (Airbyte)
05/24/2022, 5:56 PM