Skip to content

Commit e8659e9

Browse files
committed
Merge branch 'master' of github.com:brave-intl/challenge-bypass-server into add_kafka_tls_version
2 parents 88f40f6 + b381f74 commit e8659e9

15 files changed

+1278
-31
lines changed
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Integration Tests
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- master
7+
- production
8+
types:
9+
- opened
10+
- synchronize
11+
- reopened
12+
13+
jobs:
14+
integration-tests:
15+
name: Run Integration Tests
16+
runs-on: ubuntu-latest
17+
timeout-minutes: 15
18+
19+
steps:
20+
- name: Checkout code
21+
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
22+
23+
- name: Run integration tests
24+
run: make integration-test
25+
26+
- name: Cleanup
27+
if: always()
28+
run: make integration-test-clean

Dockerfile.test

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM rust:1.69 AS rust_builder
2+
RUN rustup target add x86_64-unknown-linux-musl
3+
RUN apt-get update && apt-get install -y musl-tools
4+
RUN git clone https://github.com/brave-intl/challenge-bypass-ristretto-ffi /src
5+
WORKDIR /src
6+
RUN git checkout 1.0.1
7+
RUN CARGO_PROFILE_RELEASE_LTO=true cargo rustc --target=x86_64-unknown-linux-musl --release --crate-type staticlib
8+
9+
FROM golang:1.24
10+
WORKDIR /app
11+
COPY . .
12+
13+
# Copy the Rust FFI library from the rust builder stage
14+
COPY --from=rust_builder /src/target/x86_64-unknown-linux-musl/release/libchallenge_bypass_ristretto_ffi.a /usr/lib/libchallenge_bypass_ristretto_ffi.a
15+
16+
RUN apt-get update && apt-get install -y postgresql-client
17+
RUN go mod download
18+
RUN go get github.com/stretchr/testify/assert
19+
RUN go get github.com/segmentio/kafka-go
20+
21+
CMD ["go", "test", "-tags", "integration", "./integration-tests", "-v"]

Makefile

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# Integration test configuration
2+
INTEGRATION_COMPOSE_FILE := docker-compose.integration.yml
3+
INTEGRATION_COMPOSE := docker compose -f $(INTEGRATION_COMPOSE_FILE)
4+
15
docker-psql:
26
docker compose exec postgres psql -U btokens
37

@@ -6,14 +10,14 @@ docker-dev:
610

711
docker-test:
812
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm -p 2416:2416 challenge-bypass bash -c \
9-
"(aws dynamodb delete-table \
13+
"export AWS_PAGER='' && (aws dynamodb delete-table \
1014
--table-name redemptions --endpoint-url http://dynamodb:8000 --region us-west-2 || \
1115
aws dynamodb create-table \
1216
--attribute-definitions AttributeName=id,AttributeType=S \
1317
--key-schema AttributeName=id,KeyType=HASH \
1418
--billing-mode PAY_PER_REQUEST \
1519
--table-name redemptions --endpoint-url http://dynamodb:8000 --region us-west-2 ) \
16-
&& go test -v ./..."
20+
&& go test -v -tags='!integration' ./..."
1721

1822
docker-lint:
1923
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm -p 2416:2416 challenge-bypass golangci-lint run
@@ -34,3 +38,50 @@ generate-avro:
3438

3539
lint:
3640
docker run --rm -v "$$(pwd):/app" --workdir /app golangci/golangci-lint:v2.1.6 golangci-lint run -v ./...
41+
42+
# Integration test commands
43+
.PHONY: integration-test
44+
integration-test: integration-test-clean
45+
@echo "🏗️ Building services..."
46+
@$(INTEGRATION_COMPOSE) build
47+
48+
@echo "🚀 Starting services..."
49+
@$(INTEGRATION_COMPOSE) up -d
50+
51+
@echo "⏳ Waiting for services to be ready..."
52+
@for i in $$(seq 1 10); do \
53+
echo -n "$$i... "; \
54+
sleep 1; \
55+
done; \
56+
echo ""
57+
58+
@echo "🏗️ Building test runner..."
59+
@$(INTEGRATION_COMPOSE) --profile test build test-runner
60+
61+
@echo "🧪 Running integration tests..."
62+
@TEST_NAME="$${TEST_NAME:-}" && \
63+
if [ -n "$$TEST_NAME" ]; then \
64+
echo "Running specific test: $$TEST_NAME"; \
65+
$(INTEGRATION_COMPOSE) --profile test run --rm test-runner go test -v -tags=integration ./... -run=$$TEST_NAME || (echo "❌ Tests failed!"; $(MAKE) integration-test-clean; exit 1); \
66+
else \
67+
echo "Running all tests"; \
68+
$(INTEGRATION_COMPOSE) --profile test run --rm test-runner || (echo "❌ Tests failed!"; $(MAKE) integration-test-clean; exit 1); \
69+
fi
70+
71+
@echo "🧹 Cleaning up..."
72+
@$(MAKE) integration-test-clean
73+
74+
@echo "✅ Integration tests completed successfully!"
75+
76+
.PHONY: integration-test-clean
77+
integration-test-clean:
78+
@echo "🧹 Cleaning up containers and volumes..."
79+
@$(INTEGRATION_COMPOSE) --profile test down -v --remove-orphans 2>/dev/null || true
80+
81+
.PHONY: integration-test-logs
82+
integration-test-logs:
83+
@$(INTEGRATION_COMPOSE) logs -f
84+
85+
# Alias for consistency with existing naming convention
86+
.PHONY: docker-integration-test
87+
docker-integration-test: integration-test

README.md

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,79 @@ This project uses [golangci-lint](https://golangci-lint.run/) for linting, this
2121
To run locally use `make lint` which runs linting using docker however if you want to run it locally using a binary release (which can be faster) follow the [installation instructions for your platform](https://golangci-lint.run/usage/install/) and then run `golangci-lint run -v ./...`
2222

2323
## Testing
24-
Run the below command in order to test changes, if you have an M1 / M2 Mac (or ARM based processor) follow the steps below to setup docker to be able to run the tests
24+
25+
### Unit Tests
26+
27+
Run the below command in order to test changes, if you have an M1 / M2 Mac (or ARM based processor) follow the steps below to setup docker to be able to run the tests
2528
```
2629
make docker-test
2730
```
2831

32+
### Integration Tests
33+
34+
The project includes comprehensive integration tests that verify the entire system working together with all dependencies.
35+
36+
#### What the Integration Tests Do
37+
38+
The integration tests:
39+
- Spin up a complete environment with PostgreSQL, Kafka, Zookeeper, LocalStack (for DynamoDB), and the application
40+
- Test end-to-end flows including:
41+
- Token redemption flows through Kafka
42+
- Token signing flows through Kafka
43+
- Database persistence and retrieval
44+
- DynamoDB operations
45+
- Verify the application correctly processes messages between Kafka topics
46+
- Ensure proper communication between all services
47+
48+
#### Running Integration Tests
49+
50+
To run the integration tests, simply use:
51+
52+
```bash
53+
# run all integration tests
54+
make integration-test
55+
# or run a specific integration test
56+
make integration-test TEST_NAME=TestTokenIssuanceViaKafkaAndRedeemViaHTTPFlow
57+
```
58+
59+
This command will:
60+
1. Clean up any existing test containers
61+
2. Build all required services
62+
3. Start the test environment (PostgreSQL, Kafka, Zookeeper, LocalStack)
63+
4. Wait for all services to be healthy and ready (~30 seconds)
64+
5. Build and run the test suite
65+
6. Automatically clean up all containers and volumes after completion
66+
67+
#### Manual Cleanup
68+
69+
If the tests are interrupted or you need to manually clean up the test environment:
70+
71+
```bash
72+
make integration-test-clean
73+
```
74+
75+
This will remove all test containers, networks, and volumes created by the integration tests.
76+
77+
#### Viewing Logs
78+
79+
To debug issues or view what's happening during the tests:
80+
81+
```bash
82+
make integration-test-logs
83+
```
84+
85+
This will tail the logs from all services in the integration test environment.
86+
87+
#### Test Configuration
88+
89+
The integration tests use a separate `docker-compose.integration.yml` file which:
90+
- Creates isolated test topics in Kafka
91+
- Uses a dedicated test database
92+
- Runs LocalStack for DynamoDB emulation
93+
- Configures all services with test-specific settings
94+
2995
### Have an M1 / M2 (ARM) Mac?
96+
3097
1.) In Docker Desktop, go to: `Settings -> Docker Engine` <br />
3198
#### Modify file to include
3299
```

docker-compose.integration.yml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
services:
2+
cbp:
3+
build: .
4+
depends_on:
5+
postgres:
6+
condition: service_healthy
7+
kafka:
8+
condition: service_healthy
9+
localstack:
10+
condition: service_healthy
11+
environment:
12+
- DATABASE_URL=postgresql://testuser:testpassword@postgres:5432/testdb?sslmode=disable
13+
- VPC_KAFKA_BROKERS=kafka:9092
14+
- DYNAMODB_ENDPOINT=http://localstack:4566
15+
- ENV=test
16+
- REDEEM_CONSUMER_TOPIC=test.consumer.redeem
17+
- REDEEM_PRODUCER_TOPIC=test.producer.redeem
18+
- SIGN_CONSUMER_TOPIC=test.consumer.sign
19+
- SIGN_PRODUCER_TOPIC=test.producer.sign
20+
- CONSUMER_GROUP=integration-service
21+
- AWS_REGION=us-east-1
22+
- AWS_ACCESS_KEY_ID=test
23+
- AWS_SECRET_ACCESS_KEY=test
24+
ports:
25+
- "2416:2416"
26+
- "9090:9090"
27+
28+
postgres:
29+
image: postgres:latest
30+
environment:
31+
- POSTGRES_USER=testuser
32+
- POSTGRES_PASSWORD=testpassword
33+
- POSTGRES_DB=testdb
34+
ports:
35+
- "5432:5432"
36+
healthcheck:
37+
test: ["CMD-SHELL", "pg_isready -U testuser -d testdb"]
38+
interval: 5s
39+
timeout: 2s
40+
retries: 10
41+
42+
zookeeper:
43+
image: wurstmeister/zookeeper:latest
44+
ports:
45+
- "2181:2181"
46+
healthcheck:
47+
test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep imok"]
48+
interval: 5s
49+
timeout: 2s
50+
retries: 10
51+
52+
kafka:
53+
image: wurstmeister/kafka:latest
54+
depends_on:
55+
zookeeper:
56+
condition: service_healthy
57+
ports:
58+
- "9092:9092"
59+
environment:
60+
- KAFKA_ADVERTISED_HOST_NAME=kafka
61+
- KAFKA_ADVERTISED_PORT=9092
62+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
63+
- KAFKA_CREATE_TOPICS=test.consumer.redeem:1:1,test.producer.redeem:1:1,test.consumer.sign:1:1,test.producer.sign:1:1
64+
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
65+
# Reduce memory usage for testing
66+
- KAFKA_HEAP_OPTS=-Xmx256M -Xms128M
67+
healthcheck:
68+
test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list"]
69+
interval: 10s
70+
timeout: 5s
71+
retries: 10
72+
73+
localstack:
74+
image: localstack/localstack:latest
75+
ports:
76+
- "4566:4566"
77+
environment:
78+
- SERVICES=dynamodb
79+
- DEBUG=1
80+
healthcheck:
81+
test: ["CMD", "sh", "-c", "awslocal dynamodb list-tables --region us-east-1 || exit 1"]
82+
interval: 5s
83+
timeout: 5s
84+
retries: 15
85+
86+
test-runner:
87+
build:
88+
context: .
89+
dockerfile: Dockerfile.test
90+
profiles:
91+
- test
92+
depends_on:
93+
cbp:
94+
condition: service_started
95+
kafka:
96+
condition: service_healthy
97+
environment:
98+
- DATABASE_URL=postgresql://testuser:testpassword@postgres:5432/testdb?sslmode=disable
99+
- DYNAMODB_ENDPOINT=http://localstack:4566
100+
- ENV=test
101+
- TEST_SHOULD_WRITE_REDEEM_REQUESTS_HERE=test.consumer.redeem
102+
- TEST_SHOULD_READ_REDEEM_REQUESTS_HERE=test.producer.redeem
103+
- TEST_SHOULD_WRITE_SIGNING_REQUESTS_HERE=test.consumer.sign
104+
- TEST_SHOULD_READ_SIGNING_REQUESTS_HERE=test.producer.sign
105+
- CONSUMER_GROUP=test
106+
command: ["go", "test", "-tags", "integration", "./integration-tests", "-v", "-count=1"]

go.mod

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ require (
1010
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.18.4
1111
github.com/brave-intl/bat-go/libs v0.0.0-20250515184320-890ea7676fc7
1212
github.com/brave-intl/challenge-bypass-ristretto-ffi v0.0.0-20220418231828-419995e4a873
13-
github.com/getsentry/raven-go v0.2.0
1413
github.com/go-chi/chi/v5 v5.2.2
1514
github.com/go-chi/httplog/v3 v3.2.2
1615
github.com/golang-migrate/migrate/v4 v4.18.3
@@ -23,7 +22,7 @@ require (
2322
github.com/satori/go.uuid v1.2.0
2423
github.com/segmentio/kafka-go v0.4.38
2524
github.com/segmentio/kafka-go/sasl/aws_msk_iam_v2 v0.1.0
26-
github.com/stretchr/testify v1.9.0
25+
github.com/stretchr/testify v1.10.0
2726
)
2827

2928
require (
@@ -40,7 +39,6 @@ require (
4039
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
4140
github.com/aws/smithy-go v1.22.2 // indirect
4241
github.com/beorn7/perks v1.0.1 // indirect
43-
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
4442
github.com/cespare/xxhash/v2 v2.3.0 // indirect
4543
github.com/davecgh/go-spew v1.1.1 // indirect
4644
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect

go.sum

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,6 @@ github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdb
7070
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
7171
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
7272
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
73-
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
74-
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
7573
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
7674
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
7775
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -96,8 +94,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
9694
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
9795
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
9896
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
99-
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
100-
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
10197
github.com/getsentry/sentry-go v0.17.0 h1:UustVWnOoDFHBS7IJUB2QK/nB5pap748ZEp0swnQJak=
10298
github.com/getsentry/sentry-go v0.17.0/go.mod h1:B82dxtBvxG0KaPD8/hfSV+VcHD+Lg/xUS4JuQn1P4cM=
10399
github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
@@ -252,8 +248,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
252248
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
253249
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
254250
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
255-
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
256-
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
251+
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
252+
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
257253
github.com/throttled/throttled/v2 v2.12.0 h1:IezKE1uHlYC/0Al05oZV6Ar+uN/znw3cy9J8banxhEY=
258254
github.com/throttled/throttled/v2 v2.12.0/go.mod h1:+EAvrG2hZAQTx8oMpBu8fq6Xmm+d1P2luKK7fIY1Esc=
259255
github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw=

0 commit comments

Comments
 (0)