需要通过CD变量更改AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER和AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER的值.上述值存在于airflow_template.yaml文件中。我试着替换CI-CD变量,但它不起作用。如果有更好的方法参数化。请让我知道。
#My project folder structure looks like below:
dataops
-- docker
-- base
-- airflow.cfg
-- **airflow_template.yaml**
-- Dockerfile
-- dag-image
--Dockerfile
-- helm
--Chart.yaml
--values.yaml
--templates
--deployment.yaml
--svc.yaml
**airflow_template.yaml**
apiVersion: v1
kind: Pod
metadata:
labels: {}
spec:
containers:
- args: []
command: []
env:
- name: AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY
value: $DEV_AIRFLOW_CONTAINER_REPO
- name: AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER
value: $DEV_AIRFLOW_LOG_FOLDER
envFrom: []
imagePullPolicy: Always
name: base
ports: []
volumeMounts:
- mountPath: /usr/local/airflow/logs
name: airflow-logs
hostNetwork: false
imagePullSecrets: []
initContainers: []
nodeSelector: {}
restartPolicy: Never
securityContext:
runAsUser: 1000
serviceAccountName: default
volumes:
- emptyDir: {}
name: airflow-logs
gitlab-ci.yml
stages:
- build_and_upload
- deploy_to_dev
- tag_prod
- deploy_to_prod
build_and_upload:
stage: build_and_upload
image: docker:latest
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "/certs"
services:
- docker:19.03.14-dind
script:
- echo $DEV_CREDENTIALS > service_account.json && cat service_account.json | docker login -u _json_key --password-stdin https://gcr.io
- echo "as- $DEV_AIRFLOW_LOG_FOLDER"
- export DEV_AIRFLOW_LOG_FOLDER="${DEV_AIRFLOW_LOG_FOLDER}"
- mkdir -p edfi/operation
- cp -r airflow_dags/ dataops/docker/dag-image/airflow_dags/
- cd dataops/docker/dag-image/
- docker build -t "$DEV_DAGS_IMAGE:$CI_COMMIT_SHORT_SHA" --build-arg COMMIT_HASH=$CI_COMMIT_SHORT_SHA .
- docker tag $DEV_DAGS_IMAGE:$CI_COMMIT_SHORT_SHA $DEV_DAGS_IMAGE:latest
- docker push $DEV_DAGS_IMAGE:$CI_COMMIT_SHORT_SHA
- docker push $DEV_DAGS_IMAGE:latest
only:
refs:
- develop
# variables:
# - $CI_COMMIT_MESSAGE =~ /penguin/
deploy_to_dev:
stage: deploy_to_dev
image: $CI_REGISTRY_IMAGE:kube-image
script:
- echo $DEV_CREDENTIALS > service_account.json && cat service_account.json | docker login -u _json_key --password-stdin https://gcr.io
- echo "as- $DEV_AIRFLOW_LOG_FOLDER"
- export DEV_AIRFLOW_CONTAINER_REPO="${DEV_AIRFLOW_CONTAINER_REPO}"
- export DEV_AIRFLOW_LOG_FOLDER="${DEV_AIRFLOW_LOG_FOLDER}"
- gcloud auth activate-service-account $DEV_SERVICE_ACCOUNT --key-file=./service_account.json --project=$DEV_PROJECT_NAME
- gcloud container clusters get-credentials $DEV_GKE_CLUSTER --region $REGION
- echo $DEV_DB_CONN > dataops/helm/airflow-loadbalancer/files/secrets/airflow/AIRFLOW__CORE__SQL_ALCHEMY_CONN
- cd dataops/helm/
- helm upgrade airflow-dev airflow-loadbalancer/ --install --atomic --set dags_image.tag=$CI_COMMIT_SHORT_SHA
only:
refs:
- develop
发布于 2021-07-02 11:25:27
您可以将它变成一个jinja2模板,并使用一个小的Python程序将这些值插入到模板中。然后,您还拥有使用环境变量或其他东西的所有灵活性。
https://stackoverflow.com/questions/68179629
复制相似问题