diff --git a/jetty/deploy-nomulus-for-env.sh b/jetty/deploy-nomulus-for-env.sh index 43e1cf407..ea98c5dc2 100755 --- a/jetty/deploy-nomulus-for-env.sh +++ b/jetty/deploy-nomulus-for-env.sh @@ -37,15 +37,19 @@ for service in frontend backend pubapi console do sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \ sed s/ENVIRONMENT/"${environment}"/g | \ + sed s/PROXY_ENV/"${environment}"/g | \ + sed s/EPP/"epp"/g | \ kubectl apply -f - + kubectl rollout restart deployment/${service} # canary sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \ sed s/ENVIRONMENT/"${environment}"/g | \ + sed s/PROXY_ENV/"${environment}_canary"/g | \ + sed s/EPP/"epp-canary"/g | \ sed s/"${service}"/"${service}-canary"/g | \ kubectl apply -f - + kubectl rollout restart deployment/${service}-canary done -# Kills all running pods, new pods created will be pulling the new image. -kubectl delete pods --all kubectl apply -f "./kubernetes/gateway/nomulus-gateway.yaml" kubectl apply -f "./kubernetes/gateway/nomulus-iap-${environment}.yaml" for service in frontend backend console pubapi @@ -57,4 +61,16 @@ do sed s/SERVICE/"${service}-canary"/g "./kubernetes/gateway/nomulus-backend-policy-${environment}.yaml" | \ kubectl apply -f - done + +# Restart proxies +while read line +do + parts=(${line}) + echo "Updating cluster ${parts[0]} in location ${parts[1]}..." + gcloud container clusters get-credentials ${parts[0]} \ + --project ${project} --location ${parts[1]} + kubectl rollout restart deployment/proxy-deployment + kubectl rollout restart deployment/proxy-deployment-canary +done < <(gcloud container clusters list --project ${project} | grep proxy-cluster) + kubectl config use-context "$current_context" diff --git a/jetty/kubernetes/nomulus-backend.yaml b/jetty/kubernetes/nomulus-backend.yaml index a50458b3e..aad8a1969 100644 --- a/jetty/kubernetes/nomulus-backend.yaml +++ b/jetty/kubernetes/nomulus-backend.yaml @@ -12,6 +12,8 @@ spec: service: backend spec: serviceAccountName: nomulus + nodeSelector: + cloud.google.com/compute-class: "Performance" containers: - name: backend image: gcr.io/GCP_PROJECT/nomulus @@ -20,7 +22,8 @@ spec: name: http resources: requests: - cpu: "500m" + cpu: "100m" + memory: "512Mi" args: [ENVIRONMENT] env: - name: POD_ID diff --git a/jetty/kubernetes/nomulus-console.yaml b/jetty/kubernetes/nomulus-console.yaml index 926b3b967..e8b8fd82c 100644 --- a/jetty/kubernetes/nomulus-console.yaml +++ b/jetty/kubernetes/nomulus-console.yaml @@ -12,6 +12,8 @@ spec: service: console spec: serviceAccountName: nomulus + nodeSelector: + cloud.google.com/compute-class: "Performance" containers: - name: console image: gcr.io/GCP_PROJECT/nomulus @@ -20,7 +22,8 @@ spec: name: http resources: requests: - cpu: "500m" + cpu: "100m" + memory: "512Mi" args: [ENVIRONMENT] env: - name: POD_ID diff --git a/jetty/kubernetes/nomulus-frontend.yaml b/jetty/kubernetes/nomulus-frontend.yaml index 7b96e635f..a286a58b2 100644 --- a/jetty/kubernetes/nomulus-frontend.yaml +++ b/jetty/kubernetes/nomulus-frontend.yaml @@ -12,6 +12,8 @@ spec: service: frontend spec: serviceAccountName: nomulus + nodeSelector: + cloud.google.com/compute-class: "Performance" containers: - name: frontend image: gcr.io/GCP_PROJECT/nomulus @@ -20,7 +22,8 @@ spec: name: http resources: requests: - cpu: "500m" + cpu: "100m" + memory: "512Mi" args: [ENVIRONMENT] env: - name: POD_ID @@ -37,6 +40,27 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: frontend + - name: EPP + image: gcr.io/GCP_PROJECT/proxy + ports: + - containerPort: 30002 + name: epp + resources: + requests: + cpu: "100m" + memory: "512Mi" + args: [--env, PROXY_ENV, --log, --local] + env: + - name: POD_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_ID + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: EPP --- # Only need to define the service account once per cluster. apiVersion: v1 @@ -55,7 +79,7 @@ spec: apiVersion: apps/v1 kind: Deployment name: frontend - minReplicas: 5 + minReplicas: 15 maxReplicas: 15 metrics: - type: Resource @@ -77,6 +101,26 @@ spec: targetPort: http name: http --- +apiVersion: v1 +kind: Service +metadata: + name: EPP + annotations: + cloud.google.com/l4-rbs: enabled + networking.gke.io/weighted-load-balancing: pods-per-node +spec: + type: LoadBalancer + # Traffic is directly delivered to a node, preserving the original source IP. + externalTrafficPolicy: Local + ipFamilies: [IPv4, IPv6] + ipFamilyPolicy: RequireDualStack + selector: + service: frontend + ports: + - port: 700 + targetPort: epp + name: epp +--- apiVersion: net.gke.io/v1 kind: ServiceExport metadata: diff --git a/jetty/kubernetes/nomulus-pubapi.yaml b/jetty/kubernetes/nomulus-pubapi.yaml index ce8cb333c..402e9ce68 100644 --- a/jetty/kubernetes/nomulus-pubapi.yaml +++ b/jetty/kubernetes/nomulus-pubapi.yaml @@ -12,6 +12,8 @@ spec: service: pubapi spec: serviceAccountName: nomulus + nodeSelector: + cloud.google.com/compute-class: "Performance" containers: - name: pubapi image: gcr.io/GCP_PROJECT/nomulus @@ -20,7 +22,8 @@ spec: name: http resources: requests: - cpu: "500m" + cpu: "100m" + memory: "512Mi" args: [ENVIRONMENT] env: - name: POD_ID diff --git a/release/builder/build.sh b/release/builder/build.sh index 93010db4f..a646e98a7 100755 --- a/release/builder/build.sh +++ b/release/builder/build.sh @@ -58,6 +58,8 @@ apt-get install postgresql-client-17 procps -y # Install gcloud apt-get install google-cloud-cli -y apt-get install google-cloud-sdk-app-engine-java -y +apt-get install kubectl -y +apt-get install google-cloud-cli-gke-gcloud-auth-plugin -y # Install git apt-get install git -y diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index a16121836..c8c27cbfc 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -198,6 +198,7 @@ artifacts: - 'release/cloudbuild-delete-*.yaml' - 'release/cloudbuild-schema-deploy-*.yaml' - 'release/cloudbuild-schema-verify-*.yaml' + - 'release/cloudbuild-restart-proxies-*.yaml' - 'jetty/kubernetes/*.yaml' - 'jetty/kubernetes/gateway/*.yaml' # The images are already uploaded, but we still need to include them there so that diff --git a/release/cloudbuild-release.yaml b/release/cloudbuild-release.yaml index 6b33b8acb..9c6d4f4e0 100644 --- a/release/cloudbuild-release.yaml +++ b/release/cloudbuild-release.yaml @@ -88,6 +88,7 @@ steps: sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-restart-proxies.yaml sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy.yaml @@ -99,6 +100,11 @@ steps: > release/cloudbuild-deploy-gke-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \ > release/cloudbuild-delete-${environment}.yaml + sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml \ + > release/cloudbuild-restart-proxies-${environment}.yaml + sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml | \ + sed s/proxy-deployment/proxy-deployment-canary/g \ + > release/cloudbuild-restart-proxies-${environment}-canary.yaml done # Build and upload the schema_deployer image. - name: 'gcr.io/cloud-builders/docker' @@ -178,11 +184,13 @@ steps: base_domain=$(grep baseDomain \ ./core/src/main/java/google/registry/config/files/nomulus-config-${env}.yaml | \ awk '{print $2}') - for service in frontend backend pubapi console + for service in frontend backend pubapi console do # non-canary sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \ - sed s/ENVIRONMENT/${env}/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml + sed s/ENVIRONMENT/${env}/g | \ + sed s/PROXY_ENV/"${env}"/g | \ + sed s/EPP/"epp"/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml # Proxy '--log' flag does not work on production. if [ ${env} == production ] then @@ -196,6 +204,8 @@ steps: # canary sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \ sed s/ENVIRONMENT/${env}/g | \ + sed s/PROXY_ENV/"${env}_canary"/g | \ + sed s/EPP/"epp-canary"/g | \ sed s/${service}/${service}-canary/g \ > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml # Proxy '--log' flag does not work on production. @@ -264,7 +274,7 @@ steps: $(gcloud auth list --format='get(account)' --filter=active) git add . git commit -m "Release commit for tag ${TAG_NAME}" - git push -o nokeycheck origin master + git push -o nokeycheck origin master git tag ${TAG_NAME} git push -o nokeycheck origin ${TAG_NAME} timeout: 3600s diff --git a/release/cloudbuild-restart-proxies.yaml b/release/cloudbuild-restart-proxies.yaml new file mode 100644 index 000000000..5db4695b0 --- /dev/null +++ b/release/cloudbuild-restart-proxies.yaml @@ -0,0 +1,54 @@ +# This will do rolling restarts of all proxies. This forces the client to reconnect +# and resets the sessions. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config=cloudbuild-restart-proxies.yaml \ +# --substitutions=_ENV=[ENV] .. +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +# +# Note: to work around the issue in Spinnaker's 'Deployment Manifest' stage, +# variable references must avoid the ${var} format. Valid formats include +# $var or ${"${var}"}. This file uses the former. Since TAG_NAME and _ENV are +# expanded in the copies sent to Spinnaker, we preserve the brackets around +# them for safe pattern matching during release. +# See https://github.com/spinnaker/spinnaker/issues/3028 for more information. +steps: +# Pull the credential for nomulus tool. +- name: 'gcr.io/$PROJECT_ID/builder:latest' + entrypoint: /bin/bash + args: + - -c + - | + set -e + gcloud secrets versions access latest \ + --secret nomulus-tool-cloudbuild-credential > tool-credential.json +# Do rolling restarts of all proxies in all environments. +- name: 'gcr.io/$PROJECT_ID/builder:latest' + entrypoint: /bin/bash + args: + - -c + - | + if [ ${_ENV} == production ] + then + project_id="domain-registry" + else + project_id="domain-registry-${_ENV}" + fi + + gcloud auth activate-service-account --key-file=tool-credential.json + + while read line + do + name=$(echo $line | awk '{print $1}') + location=$(echo $line | awk '{print $2}') + echo $name $region + echo "Updating cluster $name in location $location..." + gcloud container clusters get-credentials $name \ + --project $project_id --location $location + kubectl rollout restart deployment/proxy-deployment + done < <(gcloud container clusters list --project $project_id | grep proxy-cluster) +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_8'