Skip to content

Commit 84cf82e

Browse files
leifmadsenleifmadsen
authored
Merge pull request #120 from infrawatch/import/stf153
Import master to stable-1.5
2 parents c823d1f + f1c2bf4 commit 84cf82e

38 files changed

+1023
-186
lines changed

‎.github/workflows/integration.yml‎

Lines changed: 189 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,16 @@ env:
1616
on: [push, pull_request]
1717

1818
jobs:
19-
metrics:
20-
name: "[metrics] handler: ceilometer-metrics, collectd-metrics; application: prometheus"
21-
runs-on: ubuntu-20.04
19+
collectd-metrics-bridge:
20+
name: "[metrics] transport: socket(sg-bridge); handler: collectd-metrics; application: prometheus"
21+
runs-on: ubuntu-22.04
2222
env:
23-
QDR_CHANNEL_CEILOMTR: ceilometer/metering.sample
24-
QDR_CHANNEL_COLLECTD: collectd/metrics
23+
QDR_CHANNEL: collectd/metrics
2524
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
2625
PROMETHEUS_IMAGE: prom/prometheus:latest
2726
steps:
2827
- name: Checkout code
29-
uses: actions/checkout@v2
28+
uses: actions/checkout@v3
3029
- name: Prepare environment
3130
run: |
3231
mkdir -p /opt/stack/
@@ -39,6 +38,13 @@ jobs:
3938
sudo rm -rf /etc/mysql
4039
sudo dpkg -l | grep -i mysql
4140
sudo apt-get clean
41+
- name: Prepare environment for postgres-server installation
42+
run: |
43+
sudo apt remove postgresql-client-common
44+
sudo apt install postgresql-client-common=238
45+
sudo apt install postgresql-common
46+
sudo python -m pip install --upgrade pip
47+
sudo python -m pip install --upgrade virtualenv
4248
- name: Install devstack
4349
run: |
4450
SOURCE=$(pwd)
@@ -58,17 +64,111 @@ jobs:
5864
run: |
5965
echo "${GITHUB_REF#refs/heads/}"
6066
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
61-
- name: Start sg-bridge for collectd from container image
67+
- name: Start sg-bridge from container image
6268
if: steps.bridge_branch.outcome != 'success'
6369
run: |
6470
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
65-
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL_COLLECTD \
71+
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \
6672
--gw_unix=$BRIDGE_SOCKET
67-
- name: Start sg-bridge for collectd with same branch
73+
- name: Start sg-bridge from same branch
6874
if: steps.bridge_branch.outcome == 'success'
6975
run: |
7076
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
71-
-e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL_COLLECTD -e OPSTOOLS_REPO \
77+
-e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
78+
--workdir=$(dirname $BRIDGE_SOCKET) \
79+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
80+
- name: Install collectd
81+
run: |
82+
sudo apt-get install collectd
83+
sudo systemctl stop collectd && sudo systemctl disable collectd
84+
sudo cp ci/integration/metrics/collectd/collectd.conf /etc/collectd/collectd.conf
85+
sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
86+
sudo collectd -C ci/integration/metrics/collectd/collectd.conf
87+
- name: Run sg-core to process metrics
88+
run: |
89+
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
90+
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
91+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_sg.sh
92+
- name: Run Prometheus to store metrics
93+
run: |
94+
docker run --name=prometheus -d --network host \
95+
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
96+
$PROMETHEUS_IMAGE
97+
- name: Debug output
98+
run: |
99+
sleep 360
100+
echo "=========================== qdr =========================" && \
101+
docker exec qdr qdstat -b 127.0.0.1:5666 -a
102+
docker logs qdr
103+
echo "========================= sg-core =======================" && \
104+
docker logs sgcore
105+
echo "======================== prometheus =====================" && \
106+
docker logs prometheus
107+
- name: Validate metrics processing
108+
run: |
109+
docker run --name=validate -uroot --network host \
110+
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
111+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_validation.sh
112+
#-------------------------------------------------------------------------------
113+
ceilometer-metrics-bridge:
114+
name: "[metrics] transport: socket(sg-bridge); handler: ceilometer-metrics; application: prometheus"
115+
runs-on: ubuntu-22.04
116+
env:
117+
QDR_CHANNEL: anycast/ceilometer/metering.sample
118+
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
119+
PROMETHEUS_IMAGE: prom/prometheus:latest
120+
steps:
121+
- name: Checkout code
122+
uses: actions/checkout@v3
123+
- name: Prepare environment
124+
run: |
125+
mkdir -p /opt/stack/
126+
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
127+
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
128+
run: |
129+
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
130+
sudo apt-get purge mysql\*
131+
sudo rm -rf /var/lib/mysql
132+
sudo rm -rf /etc/mysql
133+
sudo dpkg -l | grep -i mysql
134+
sudo apt-get clean
135+
- name: Prepare environment for postgres-server installation
136+
run: |
137+
sudo apt remove postgresql-client-common
138+
sudo apt install postgresql-client-common=238
139+
sudo apt install postgresql-common
140+
sudo python -m pip install --upgrade pip
141+
sudo python -m pip install --upgrade virtualenv
142+
- name: Install devstack
143+
run: |
144+
SOURCE=$(pwd)
145+
git clone http://github.com/openstack/devstack /opt/stack/devstack
146+
pushd /opt/stack/devstack
147+
cp $SOURCE/ci/integration/metrics/local.conf .
148+
sudo apt-get update
149+
./stack.sh
150+
popd
151+
# start message bus services
152+
- name: Start QDR service
153+
run: |
154+
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE
155+
- name: Check if sg-bridge repository has same topic branch
156+
id: bridge_branch
157+
continue-on-error: true
158+
run: |
159+
echo "${GITHUB_REF#refs/heads/}"
160+
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
161+
- name: Start sg-bridge from container image
162+
if: steps.bridge_branch.outcome != 'success'
163+
run: |
164+
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
165+
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \
166+
--gw_unix=$BRIDGE_SOCKET
167+
- name: Start sg-bridge from same branch
168+
if: steps.bridge_branch.outcome == 'success'
169+
run: |
170+
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
171+
-e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
72172
--workdir=$(dirname $BRIDGE_SOCKET) \
73173
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
74174
- name: Set Ceilometer pipelines to QDR output and restart notification agent
@@ -78,22 +178,15 @@ jobs:
78178
echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
79179
echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
80180
echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
81-
cp ci/integration/metrics/*pipeline.yaml /etc/ceilometer/.
181+
cp ci/integration/metrics/ceilometer/bridge/*pipeline.yaml /etc/ceilometer/.
82182
cat /etc/ceilometer/*
83183
sudo pip install pyngus
84184
sudo systemctl restart [email protected]
85-
- name: Install collectd
86-
run: |
87-
sudo apt-get install collectd
88-
sudo systemctl stop collectd && sudo systemctl disable collectd
89-
sudo cp ci/integration/metrics/collectd.conf /etc/collectd/collectd.conf
90-
sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
91-
sudo collectd -C ci/integration/metrics/collectd.conf
92185
- name: Run sg-core to process metrics
93186
run: |
94187
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
95188
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
96-
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_sg.sh
189+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/bridge/run_sg.sh
97190
- name: Run Prometheus to store metrics
98191
run: |
99192
docker run --name=prometheus -d --network host \
@@ -104,23 +197,94 @@ jobs:
104197
sleep 360
105198
echo "=========================== qdr =========================" && \
106199
docker exec qdr qdstat -b 127.0.0.1:5666 -a
200+
docker logs qdr
107201
echo "========================= sg-core =======================" && \
108202
docker logs sgcore
109-
echo "======================== collectd =======================" && \
110-
cat /var/log/collectd.log
111203
echo "========================= ceilometer ====================" && \
112204
sudo journalctl -xu [email protected]
205+
echo "======================== prometheus =====================" && \
206+
docker logs prometheus
207+
- name: Validate metrics processing
208+
run: |
209+
docker run --name=validate -uroot --network host \
210+
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
211+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh
212+
#-------------------------------------------------------------------------------
213+
ceilometer-metrics-tcp:
214+
name: "[metrics] transport: socket(tcp); handler: ceilometer-metrics; application: prometheus"
215+
runs-on: ubuntu-22.04
216+
env:
217+
PROMETHEUS_IMAGE: prom/prometheus:latest
218+
steps:
219+
- name: Checkout code
220+
uses: actions/checkout@v3
221+
- name: Prepare environment
222+
run: |
223+
mkdir -p /opt/stack/
224+
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
225+
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
226+
run: |
227+
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
228+
sudo apt-get purge mysql\*
229+
sudo rm -rf /var/lib/mysql
230+
sudo rm -rf /etc/mysql
231+
sudo dpkg -l | grep -i mysql
232+
sudo apt-get clean
233+
- name: Prepare environment for postgres-server installation
234+
run: |
235+
sudo apt remove postgresql-client-common
236+
sudo apt install postgresql-client-common=238
237+
sudo apt install postgresql-common
238+
sudo python -m pip install --upgrade pip
239+
sudo python -m pip install --upgrade virtualenv
240+
- name: Install devstack
241+
run: |
242+
SOURCE=$(pwd)
243+
git clone http://github.com/openstack/devstack /opt/stack/devstack
244+
pushd /opt/stack/devstack
245+
cp $SOURCE/ci/integration/metrics/local.conf .
246+
sudo apt-get update
247+
./stack.sh
248+
popd
249+
- name: Set Ceilometer pipelines to TCP output and restart notification agent
250+
run: |
251+
sudo apt-get install -y crudini
252+
echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
253+
echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
254+
echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
255+
echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
256+
cp ci/integration/metrics/ceilometer/tcp/*pipeline.yaml /etc/ceilometer/.
257+
cat /etc/ceilometer/*
258+
sudo pip install pyngus
259+
sudo systemctl restart [email protected]
260+
- name: Run sg-core to process metrics
261+
run: |
262+
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
263+
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
264+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/tcp/run_sg.sh
265+
- name: Run Prometheus to store metrics
266+
run: |
267+
docker run --name=prometheus -d --network host \
268+
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
269+
$PROMETHEUS_IMAGE
270+
- name: Debug output
271+
run: |
272+
sleep 360
113273
echo "========================= sg-core =======================" && \
274+
docker logs sgcore
275+
echo "========================= ceilometer ====================" && \
276+
sudo journalctl -xu [email protected]
277+
echo "======================== prometheus =====================" && \
114278
docker logs prometheus
115279
- name: Validate metrics processing
116280
run: |
117281
docker run --name=validate -uroot --network host \
118282
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
119-
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_validation.sh
283+
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh
120284
#-------------------------------------------------------------------------------
121285
logging:
122286
name: "[logging] handler: logs; application: elasticsearch, loki"
123-
runs-on: ubuntu-20.04
287+
runs-on: ubuntu-22.04
124288
env:
125289
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
126290

@@ -135,7 +299,7 @@ jobs:
135299
RSYSLOG_VOLUME: "--volume ${{ github.workspace }}/ci/service_configs/rsyslog/rsyslog_config.conf:/etc/rsyslog.d/integration.conf:z"
136300
steps:
137301
- name: Checkout code
138-
uses: actions/checkout@v2
302+
uses: actions/checkout@v3
139303
# start data store services
140304
- name: Start Elasticsearch service
141305
run: |
@@ -171,7 +335,7 @@ jobs:
171335
$RSYSLOG_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_rsyslog.sh
172336
- name: Wait for services to start successfuly
173337
run: |
174-
timeout=180
338+
timeout=240
175339
echo "======================= rsyslog ======================="
176340
rsyslog_wait=0
177341
while [[ $(docker exec qdr qdstat -b 127.0.0.1:5666 -a | grep rsyslog/logs | awk '{print $8}') -le 0 ]]

‎.github/workflows/tests.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ jobs:
2323
steps:
2424
- uses: actions/setup-go@v3
2525
with:
26-
go-version: '1.18'
26+
go-version: '1.19'
2727
- uses: actions/checkout@v3
2828
#- name: download libraries
2929
# run: go mod download
@@ -78,7 +78,7 @@ jobs:
7878
- uses: actions/checkout@v2
7979
- uses: actions/setup-go@v2
8080
with:
81-
go-version: '1.18'
81+
go-version: '1.19'
8282
- name: Verify image builds
8383
run: |
8484
docker build --tag infrawatch/sg-core:latest --file build/Dockerfile .

‎build.sh‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ base=$(pwd)
1414
GOCMD=${GOCMD:-"go"}
1515
PLUGIN_DIR=${PLUGIN_DIR:-"/tmp/plugins/"}
1616
CONTAINER_BUILD=${CONTAINER_BUILD:-false}
17+
BUILD_ARGS=${BUILD_ARGS:-''}
1718

1819
PRODUCTION_BUILD=${PRODUCTION_BUILD:-false}
1920
if $PRODUCTION_BUILD; then
@@ -60,7 +61,7 @@ build_plugins() {
6061
search_list "$(basename $i)" OMIT_TRANSPORTS
6162
if [ $? -ne 1 ]; then
6263
echo "building $(basename $i).so"
63-
$GOCMD build -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
64+
$GOCMD build $BUILD_ARGS -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
6465
fi
6566
done
6667

@@ -71,7 +72,7 @@ build_plugins() {
7172
search_list "$(basename $i)" OMIT_HANDLERS
7273
if [ $? -ne 1 ]; then
7374
echo "building $(basename $i).so"
74-
$GOCMD build -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
75+
$GOCMD build $BUILD_ARGS -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
7576
fi
7677
done
7778

@@ -82,7 +83,7 @@ build_plugins() {
8283
search_list "$(basename $i)" OMIT_APPLICATIONS
8384
if [ $? -ne 1 ]; then
8485
echo "building $(basename $i).so"
85-
$GOCMD build -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
86+
$GOCMD build $BUILD_ARGS -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
8687
fi
8788
done
8889
}
@@ -92,9 +93,9 @@ build_core() {
9293
cd "$base"
9394
if $CONTAINER_BUILD; then
9495
echo "building sg-core for container"
95-
$GOCMD build -o /tmp/sg-core cmd/*.go
96+
$GOCMD build $BUILD_ARGS -o /tmp/sg-core cmd/*.go
9697
else
97-
$GOCMD build -o sg-core cmd/*.go
98+
$GOCMD build $BUILD_ARGS -o sg-core cmd/*.go
9899
fi
99100
}
100101

‎build/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ COPY . $D/
1010
COPY build/repos/opstools.repo /etc/yum.repos.d/opstools.repo
1111

1212
RUN dnf install golang git qpid-proton-c-devel -y --setopt=tsflags=nodocs
13-
RUN go install golang.org/dl/go1.18@latest && /go/bin/go1.18 download && PRODUCTION_BUILD=false CONTAINER_BUILD=true GOCMD=/go/bin/go1.18 ./build.sh
13+
RUN go install golang.org/dl/go1.19@latest && /go/bin/go1.19 download && PRODUCTION_BUILD=false CONTAINER_BUILD=true GOCMD=/go/bin/go1.19 ./build.sh
1414

1515
# --- end build, create smart gateway layer ---
1616
FROM registry.access.redhat.com/ubi8-minimal:latest

‎ci/integration/logging/run_sg.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ dnf install -y git golang gcc make qpid-proton-c-devel
1313
export GOBIN=$GOPATH/bin
1414
export PATH=$PATH:$GOBIN
1515

16-
go install golang.org/dl/go1.18@latest
17-
go1.18 download
16+
go install golang.org/dl/go1.19@latest
17+
go1.19 download
1818

1919
# install sg-core and start sg-core
2020
mkdir -p /usr/lib64/sg-core
21-
PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.18 ./build.sh
21+
PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh
2222

2323
./sg-core -config ./ci/integration/logging/sg_config.yaml

‎ci/integration/metrics/event_pipeline.yaml‎ renamed to ‎ci/integration/metrics/ceilometer/bridge/event_pipeline.yaml‎

File renamed without changes.

‎ci/integration/metrics/pipeline.yaml‎ renamed to ‎ci/integration/metrics/ceilometer/bridge/pipeline.yaml‎

File renamed without changes.

0 commit comments

Comments
 (0)