Skip to content

Commit fc66311

Browse files
onewlandonewland
authored
[gocd] split snuba deploys into snuba-deploy-py and snuba-deploy-rs (#7507)
Split out the current deploy pipelines into `deploy-snuba-py` and `deploy-snuba-rs`. The goal of this is to set us up for a future where we can only deploy rust consumers when changes are made in the Rust path (reducing the amount of SLO breaches on high-traffic consumers for rebalances). At this stage, we'll still deploy both with any code change to test. Next, we'll want to change the triggering logic to ignore code changes from irrelevant sources
1 parent 138f052 commit fc66311

File tree

9 files changed

+240
-31
lines changed

9 files changed

+240
-31
lines changed

‎gocd/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ These are a set of jsonnet and libsonnet files which are used
3434
to generate the GoCD pipelines. This avoids duplication across
3535
our pipelines as we deploy to multiple regions.
3636

37-
The `gocd/templates/snuba.jsonnet` file is the entry point for the
37+
The `gocd/templates/snuba-{py,rs}.jsonnet` file is the entry point for the
3838
generated pipelines.
3939

40-
`gocd/templates/pipelines/snuba.libsonnet` define the pipeline behaviors.
40+
`gocd/templates/pipelines/snuba-{py,rs}.libsonnet` define the pipeline behaviors.
4141
This library defines the GoCD pipeline, following the same naming
4242
as the [GoCD yaml pipelines](https://github.com/tomzo/gocd-yaml-config-plugin#readme).
4343

‎gocd/templates/bash/deploy.sh‎ renamed to ‎gocd/templates/bash/deploy-py.sh‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,47 +7,32 @@ eval $(regions-project-env-vars --region="${SENTRY_REGION}")
77
--label-selector="${LABEL_SELECTOR}" \
88
--image="us-docker.pkg.dev/sentryio/snuba-mr/image:${GO_REVISION_SNUBA_REPO}" \
99
--container-name="api" \
10-
--container-name="consumer" \
1110
--container-name="dlq-consumer" \
12-
--container-name="eap-items-consumer" \
1311
--container-name="eap-items-subscriptions-executor" \
1412
--container-name="eap-items-subscriptions-scheduler" \
1513
--container-name="errors-replacer" \
1614
--container-name="events-subscriptions-executor" \
1715
--container-name="events-subscriptions-scheduler" \
18-
--container-name="generic-metrics-counters-consumer" \
19-
--container-name="generic-metrics-distributions-consumer" \
20-
--container-name="generic-metrics-sets-consumer" \
2116
--container-name="genmetrics-counters-subs-scheduler" \
2217
--container-name="genmetrics-counts-subs-executor" \
2318
--container-name="genmetrics-distributions-subs-executor" \
2419
--container-name="genmetrics-distributions-subs-scheduler" \
2520
--container-name="genmetrics-sets-subs-executor" \
2621
--container-name="genmetrics-sets-subs-scheduler" \
2722
--container-name="group-attributes-consumer" \
28-
--container-name="loadbalancer-outcomes-consumer" \
2923
--container-name="lw-deletions-search-issues-consumer" \
30-
--container-name="metrics-consumer" \
3124
--container-name="metrics-counters-subscriptions-scheduler" \
3225
--container-name="metrics-sets-subscriptions-scheduler" \
3326
--container-name="metrics-subscriptions-executor" \
3427
--container-name="outcomes-billing-consumer" \
35-
--container-name="outcomes-consumer" \
36-
--container-name="profile-chunks-consumer" \
37-
--container-name="profiles-consumer" \
38-
--container-name="profiling-functions-consumer" \
39-
--container-name="querylog-consumer" \
40-
--container-name="replays-consumer" \
4128
--container-name="search-issues-consumer" \
4229
--container-name="snuba-admin" \
43-
--container-name="transactions-consumer-new" \
4430
--container-name="transactions-subscriptions-executor" \
4531
--container-name="transactions-subscriptions-scheduler" \
46-
--container-name="snuba" \
4732
&& k8s-deploy \
4833
--label-selector="${LABEL_SELECTOR}" \
4934
--image="us-docker.pkg.dev/sentryio/snuba-mr/image:${GO_REVISION_SNUBA_REPO}" \
5035
--type="cronjob" \
51-
--container-name="cleanup" \
5236
--container-name="optimize" \
37+
--container-name="cleanup" \
5338
--container-name="cardinality-report"

‎gocd/templates/bash/deploy-rs.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
eval $(regions-project-env-vars --region="${SENTRY_REGION}")
4+
5+
/devinfra/scripts/get-cluster-credentials \
6+
&& k8s-deploy \
7+
--label-selector="${LABEL_SELECTOR}" \
8+
--image="us-docker.pkg.dev/sentryio/snuba-mr/image:${GO_REVISION_SNUBA_REPO}" \
9+
--container-name="consumer" \
10+
--container-name="eap-items-consumer" \
11+
--container-name="errors-replacer" \
12+
--container-name="generic-metrics-counters-consumer" \
13+
--container-name="generic-metrics-distributions-consumer" \
14+
--container-name="generic-metrics-sets-consumer" \
15+
--container-name="loadbalancer-outcomes-consumer" \
16+
--container-name="metrics-consumer" \
17+
--container-name="outcomes-billing-consumer" \
18+
--container-name="outcomes-consumer" \
19+
--container-name="profile-chunks-consumer" \
20+
--container-name="profiles-consumer" \
21+
--container-name="profiling-functions-consumer" \
22+
--container-name="querylog-consumer" \
23+
--container-name="replays-consumer" \
24+
--container-name="transactions-consumer-new"

‎gocd/templates/bash/deploy-st.sh‎ renamed to ‎gocd/templates/bash/deploy-st-py.sh‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,3 @@ k8s-deploy \
99
--image="us-docker.pkg.dev/sentryio/snuba-mr/image:${GO_REVISION_SNUBA_REPO}" \
1010
--container-name="snuba" \
1111
--container-name="snuba-admin"
12-
13-
k8s-deploy \
14-
--label-selector="${LABEL_SELECTOR}" \
15-
--image="us-docker.pkg.dev/sentryio/snuba-mr/image:${GO_REVISION_SNUBA_REPO}" \
16-
--type="cronjob" \
17-
--container-name="cleanup"

‎gocd/templates/bash/deploy-st-rs.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
echo "TODO fix"
4+
echo "This is a no-op for single-tenants. deploy-snuba-py handles everything"
5+
echo "(and traffic is low enough that we don't have rebalance issues)"

‎gocd/templates/pipelines/snuba.libsonnet‎ renamed to ‎gocd/templates/pipelines/snuba-py.libsonnet‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ local deploy_canary_stage(region) =
136136
LABEL_SELECTOR: 'service=snuba,is_canary=true',
137137
},
138138
tasks: [
139-
gocdtasks.script(importstr '../bash/deploy.sh'),
139+
gocdtasks.script(importstr '../bash/deploy-py.sh'),
140140
gocdtasks.script(importstr '../bash/canary-ddog-health-check.sh'),
141141
],
142142
},
@@ -208,9 +208,9 @@ function(region) {
208208
},
209209
tasks: [
210210
if getsentry.is_st(region) then
211-
gocdtasks.script(importstr '../bash/deploy-st.sh')
211+
gocdtasks.script(importstr '../bash/deploy-st-py.sh')
212212
else
213-
gocdtasks.script(importstr '../bash/deploy.sh'),
213+
gocdtasks.script(importstr '../bash/deploy-py.sh'),
214214
],
215215
},
216216
},

‎gocd/templates/pipelines/snuba-rs.libsonnet‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
local getsentry = import 'github.com/getsentry/gocd-jsonnet/libs/getsentry.libsonnet';
2+
local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet';
3+
4+
// The return value of this function is the body of a GoCD pipeline.
5+
// More information on gocd-flavor YAML this is producing can be found here:
6+
// - https://github.com/tomzo/gocd-yaml-config-plugin#pipeline
7+
// - https://www.notion.so/sentry/GoCD-New-Service-Quickstart-6d8db7a6964049b3b0e78b8a4b52e25d
8+
9+
10+
// Snuba deploy to SaaS is blocked till S4S deploy is healthy
11+
local s4s_health_check(region) =
12+
if region == 's4s' then
13+
[
14+
{
15+
health_check: {
16+
jobs: {
17+
health_check: {
18+
environment_variables: {
19+
SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}',
20+
DATADOG_API_KEY: '{{SECRET:[devinfra][st_datadog_api_key]}}',
21+
DATADOG_APP_KEY: '{{SECRET:[devinfra][st_datadog_app_key]}}',
22+
LABEL_SELECTOR: 'service=snuba',
23+
},
24+
elastic_profile_id: 'snuba',
25+
tasks: [
26+
gocdtasks.script(importstr '../bash/s4s-sentry-health-check.sh'),
27+
gocdtasks.script(importstr '../bash/s4s-ddog-health-check.sh'),
28+
],
29+
},
30+
},
31+
},
32+
},
33+
]
34+
else
35+
[];
36+
37+
// Snuba deploy to ST is blocked till SaaS deploy is healthy
38+
local saas_health_check(region) =
39+
if region == 'us' || region == 'de' then
40+
[
41+
{
42+
health_check: {
43+
jobs: {
44+
health_check: {
45+
environment_variables: {
46+
SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}',
47+
DATADOG_API_KEY: '{{SECRET:[devinfra][sentry_datadog_api_key]}}',
48+
DATADOG_APP_KEY: '{{SECRET:[devinfra][sentry_datadog_app_key]}}',
49+
LABEL_SELECTOR: 'service=snuba',
50+
SENTRY_ENVIRONMENT: region,
51+
},
52+
elastic_profile_id: 'snuba',
53+
tasks: [
54+
gocdtasks.script(importstr '../bash/saas-sentry-health-check.sh'),
55+
gocdtasks.script(importstr '../bash/saas-sentry-error-check.sh'),
56+
gocdtasks.script(importstr '../bash/saas-ddog-health-check.sh'),
57+
],
58+
},
59+
},
60+
},
61+
},
62+
]
63+
else
64+
[];
65+
66+
67+
local deploy_canary_stage(region) =
68+
if region == 'us' then
69+
[
70+
{
71+
'deploy-canary': {
72+
fetch_materials: true,
73+
jobs: {
74+
'create-sentry-release': {
75+
environment_variables: {
76+
SENTRY_ORG: 'sentry',
77+
SENTRY_PROJECT: 'snuba',
78+
SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}',
79+
},
80+
timeout: 300,
81+
elastic_profile_id: 'snuba',
82+
tasks: [
83+
gocdtasks.script(importstr '../bash/sentry-release-canary.sh'),
84+
],
85+
},
86+
'deploy-canary': {
87+
timeout: 1200,
88+
elastic_profile_id: 'snuba',
89+
environment_variables: {
90+
SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}',
91+
DATADOG_API_KEY: '{{SECRET:[devinfra][sentry_datadog_api_key]}}',
92+
DATADOG_APP_KEY: '{{SECRET:[devinfra][sentry_datadog_app_key]}}',
93+
LABEL_SELECTOR: 'service=snuba,is_canary=true',
94+
},
95+
tasks: [
96+
gocdtasks.script(importstr '../bash/deploy-rs.sh'),
97+
gocdtasks.script(importstr '../bash/canary-ddog-health-check.sh'),
98+
],
99+
},
100+
},
101+
},
102+
},
103+
]
104+
else
105+
[];
106+
107+
function(region) {
108+
environment_variables: {
109+
SENTRY_REGION: region,
110+
// Required for checkruns.
111+
GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}',
112+
GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}',
113+
},
114+
lock_behavior: 'unlockWhenFinished',
115+
materials: {
116+
snuba_repo: {
117+
git: '[email protected]:getsentry/snuba.git',
118+
shallow_clone: false,
119+
branch: 'master',
120+
destination: 'snuba',
121+
},
122+
},
123+
stages: [
124+
{
125+
checks: {
126+
jobs: {
127+
checks: {
128+
elastic_profile_id: 'snuba',
129+
tasks: [
130+
gocdtasks.script(importstr '../bash/check-github.sh'),
131+
gocdtasks.script(importstr '../bash/check-migrations.sh'),
132+
],
133+
},
134+
},
135+
},
136+
},
137+
138+
] + deploy_canary_stage(region) + [
139+
140+
{
141+
'deploy-primary': {
142+
fetch_materials: true,
143+
jobs: {
144+
// NOTE: sentry-release-primary relies on the sentry-release-canary
145+
// script being run first. So any changes here should account for
146+
// this and update deploy_canary_stage accordingly
147+
[if region == 'us' then 'create-sentry-release' else null]: {
148+
environment_variables: {
149+
SENTRY_ORG: 'sentry',
150+
SENTRY_PROJECT: 'snuba',
151+
SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}',
152+
},
153+
timeout: 300,
154+
elastic_profile_id: 'snuba',
155+
tasks: [
156+
gocdtasks.script(importstr '../bash/sentry-release-primary.sh'),
157+
],
158+
},
159+
'deploy-primary': {
160+
timeout: 1200,
161+
elastic_profile_id: 'snuba',
162+
environment_variables: {
163+
LABEL_SELECTOR: 'service=snuba',
164+
},
165+
tasks: [
166+
if getsentry.is_st(region) then
167+
gocdtasks.script(importstr '../bash/deploy-st-rs.sh')
168+
else
169+
gocdtasks.script(importstr '../bash/deploy-rs.sh'),
170+
],
171+
},
172+
},
173+
},
174+
},
175+
176+
] + s4s_health_check(region) + saas_health_check(region),
177+
}

‎gocd/templates/snuba.jsonnet‎ renamed to ‎gocd/templates/snuba-py.jsonnet‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
local snuba = import './pipelines/snuba.libsonnet';
1+
local snuba = import './pipelines/snuba-py.libsonnet';
22
local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet';
33

4-
local pipedream_config = {
5-
name: 'snuba',
4+
local py_pipedream_config = {
5+
name: 'snuba-py',
66
materials: {
77
snuba_repo: {
88
git: '[email protected]:getsentry/snuba.git',
@@ -21,4 +21,4 @@ local pipedream_config = {
2121
auto_deploy: true,
2222
};
2323

24-
pipedream.render(pipedream_config, snuba)
24+
pipedream.render(py_pipedream_config, snuba)

‎gocd/templates/snuba-rs.jsonnet‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
local snuba = import './pipelines/snuba-rs.libsonnet';
2+
local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet';
3+
4+
local rs_pipedream_config = {
5+
name: 'snuba-rs',
6+
materials: {
7+
snuba_repo: {
8+
git: '[email protected]:getsentry/snuba.git',
9+
shallow_clone: true,
10+
branch: 'master',
11+
destination: 'snuba',
12+
},
13+
},
14+
rollback: {
15+
material_name: 'snuba_repo',
16+
stage: 'deploy-primary',
17+
elastic_profile_id: 'snuba',
18+
},
19+
20+
// Set to true to auto-deploy changes (defaults to true)
21+
auto_deploy: true,
22+
};
23+
24+
pipedream.render(rs_pipedream_config, snuba)

0 commit comments

Comments
 (0)