Skip to content

Commit 32c39f4

Browse files
GnaneshGnaniGnaneshGnani
authored
Merge branch 'main' into sketchlib-rust-migration
2 parents fbaecb5 + 5946aa9 commit 32c39f4

125 files changed

Lines changed: 1692 additions & 6859 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/docker.yml‎

Lines changed: 81 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,23 @@ name: Docker Builds
22

33
on:
44
push:
5-
branches: [ main ]
6-
paths:
7-
- 'asap-common/installation/**'
8-
- 'asap-planner/**'
9-
- 'asap-tools/prometheus-client/**'
10-
- '.github/workflows/docker.yml'
5+
tags:
6+
- 'v[0-9]+.[0-9]+.[0-9]+*'
117
pull_request:
12-
branches: [ main ]
8+
branches: [main]
139
paths:
1410
- 'asap-common/installation/**'
1511
- 'asap-planner/**'
16-
- 'asap-tools/prometheus-client/**'
12+
- 'asap-sketch-ingest/**'
13+
- 'asap-query-engine/**'
14+
- 'asap-tools/queriers/prometheus-client/**'
1715
- '.github/workflows/docker.yml'
1816
workflow_dispatch:
1917

18+
permissions:
19+
packages: write
20+
contents: read
21+
2022
jobs:
2123
build-images:
2224
runs-on: ubuntu-latest
@@ -27,31 +29,86 @@ jobs:
2729
- name: Set up Docker Buildx
2830
uses: docker/setup-buildx-action@v3
2931

32+
- name: Login to GHCR
33+
uses: docker/login-action@v3
34+
with:
35+
registry: ghcr.io
36+
username: ${{ github.repository_owner }}
37+
password: ${{ secrets.GITHUB_TOKEN }}
38+
39+
# --- Base image (Python, fast) ---
3040
- name: Build base image
31-
working-directory: asap-common/installation
3241
run: |
3342
docker build \
3443
-t sketchdb-base:latest \
35-
-f Dockerfile \
36-
..
44+
-f asap-common/installation/Dockerfile \
45+
asap-common
46+
47+
- name: Push base image
48+
if: startsWith(github.ref, 'refs/tags/')
49+
run: |
50+
docker tag sketchdb-base:latest ghcr.io/projectasap/asap-base:${{ github.ref_name }}
51+
docker tag sketchdb-base:latest ghcr.io/projectasap/asap-base:latest
52+
docker push ghcr.io/projectasap/asap-base:${{ github.ref_name }}
53+
docker push ghcr.io/projectasap/asap-base:latest
3754
38-
- name: Test base image
55+
# --- Planner (Python, depends on base) ---
56+
- name: Build planner image
3957
run: |
40-
docker run --rm sketchdb-base:latest python --version
41-
docker run --rm sketchdb-base:latest pip list
58+
docker build \
59+
-t asap-planner:local \
60+
-f asap-planner/Dockerfile \
61+
asap-planner
62+
63+
- name: Push planner image
64+
if: startsWith(github.ref, 'refs/tags/')
65+
run: |
66+
docker tag asap-planner:local ghcr.io/projectasap/asap-planner:${{ github.ref_name }}
67+
docker tag asap-planner:local ghcr.io/projectasap/asap-planner:latest
68+
docker push ghcr.io/projectasap/asap-planner:${{ github.ref_name }}
69+
docker push ghcr.io/projectasap/asap-planner:latest
4270
43-
- name: Build Controller Docker image
44-
working-directory: asap-planner
71+
# --- Sketch Ingest (Python, depends on base) ---
72+
- name: Build sketch-ingest image
4573
run: |
4674
docker build \
47-
-t sketchdb-controller:latest \
48-
-f Dockerfile \
49-
.
75+
-t asap-sketch-ingest:local \
76+
-f asap-sketch-ingest/Dockerfile \
77+
asap-sketch-ingest
78+
79+
- name: Push sketch-ingest image
80+
if: startsWith(github.ref, 'refs/tags/')
81+
run: |
82+
docker tag asap-sketch-ingest:local ghcr.io/projectasap/asap-sketch-ingest:${{ github.ref_name }}
83+
docker tag asap-sketch-ingest:local ghcr.io/projectasap/asap-sketch-ingest:latest
84+
docker push ghcr.io/projectasap/asap-sketch-ingest:${{ github.ref_name }}
85+
docker push ghcr.io/projectasap/asap-sketch-ingest:latest
5086
51-
- name: Build PrometheusClient Docker image
52-
working-directory: asap-tools/prometheus-client
87+
# --- Prometheus Client (Python, depends on base) ---
88+
- name: Build prometheus-client image
5389
run: |
5490
docker build \
55-
-t sketchdb-prometheus-client:latest \
56-
-f Dockerfile \
57-
.
91+
-t asap-prometheus-client:local \
92+
-f asap-tools/queriers/prometheus-client/Dockerfile \
93+
asap-tools/queriers/prometheus-client
94+
95+
- name: Push prometheus-client image
96+
if: startsWith(github.ref, 'refs/tags/')
97+
run: |
98+
docker tag asap-prometheus-client:local ghcr.io/projectasap/asap-prometheus-client:${{ github.ref_name }}
99+
docker tag asap-prometheus-client:local ghcr.io/projectasap/asap-prometheus-client:latest
100+
docker push ghcr.io/projectasap/asap-prometheus-client:${{ github.ref_name }}
101+
docker push ghcr.io/projectasap/asap-prometheus-client:latest
102+
103+
# --- Query Engine (Rust, slow — uses GHCR layer cache) ---
104+
- name: Build and push query-engine
105+
uses: docker/build-push-action@v6
106+
with:
107+
context: .
108+
file: asap-query-engine/Dockerfile
109+
push: ${{ startsWith(github.ref, 'refs/tags/') }}
110+
tags: |
111+
ghcr.io/projectasap/asap-query-engine:${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'pr-test' }}
112+
ghcr.io/projectasap/asap-query-engine:latest
113+
cache-from: type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache
114+
cache-to: type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache,mode=max

‎.github/workflows/python.yml‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,21 @@ on:
55
branches: [ main ]
66
paths:
77
- 'asap-sketch-ingest/**'
8-
- 'asap-tools/prometheus-client/**'
8+
- 'asap-tools/queriers/prometheus-client/**'
99
- 'asap-planner/**'
1010
- 'asap-tools/**'
11-
- 'asap-tools/prometheus-exporters/**'
11+
- 'asap-tools/data-sources/prometheus-exporters/**'
1212
- 'asap-tools/execution-utilities/**'
1313
- 'asap-common/dependencies/py/**'
1414
- '.github/workflows/python.yml'
1515
pull_request:
1616
branches: [ main ]
1717
paths:
1818
- 'asap-sketch-ingest/**'
19-
- 'asap-tools/prometheus-client/**'
19+
- 'asap-tools/queriers/prometheus-client/**'
2020
- 'asap-planner/**'
2121
- 'asap-tools/**'
22-
- 'asap-tools/prometheus-exporters/**'
22+
- 'asap-tools/data-sources/prometheus-exporters/**'
2323
- 'asap-tools/execution-utilities/**'
2424
- 'asap-common/dependencies/py/**'
2525
- '.github/workflows/python.yml'
@@ -49,7 +49,7 @@ jobs:
4949
- 'asap-sketch-ingest/**'
5050
- 'asap-common/dependencies/py/**'
5151
prometheus_client:
52-
- 'asap-tools/prometheus-client/**'
52+
- 'asap-tools/queriers/prometheus-client/**'
5353
- 'asap-common/dependencies/py/**'
5454
controller:
5555
- 'asap-planner/**'
@@ -58,7 +58,7 @@ jobs:
5858
- 'asap-tools/**'
5959
- 'asap-common/dependencies/py/**'
6060
prometheus_exporters:
61-
- 'asap-tools/prometheus-exporters/**'
61+
- 'asap-tools/data-sources/prometheus-exporters/**'
6262
- 'asap-common/dependencies/py/**'
6363
execution_utilities:
6464
- 'asap-tools/execution-utilities/**'
@@ -110,19 +110,19 @@ jobs:
110110
run: |
111111
python -m pip install --upgrade pip
112112
pip install black==24.8.0 flake8==6.1.0 mypy types-requests types-PyYAML typing-extensions numpy prometheus-client urllib3
113-
if [ -f asap-tools/prometheus-client/requirements.txt ]; then pip install -r asap-tools/prometheus-client/requirements.txt; fi
113+
if [ -f asap-tools/queriers/prometheus-client/requirements.txt ]; then pip install -r asap-tools/queriers/prometheus-client/requirements.txt; fi
114114
- name: Check formatting with Black
115-
working-directory: asap-tools/prometheus-client
115+
working-directory: asap-tools/queriers/prometheus-client
116116
run: black --check --diff .
117117
- name: Lint with flake8
118-
working-directory: asap-tools/prometheus-client
118+
working-directory: asap-tools/queriers/prometheus-client
119119
run: |
120120
# Stop the build if there are Python syntax errors or undefined names
121-
flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
121+
flake8 . --config=../../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
122122
# Exit-zero treats all errors as warnings
123-
flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics
123+
flake8 . --config=../../../.flake8 --count --exit-zero --max-complexity=10 --statistics
124124
- name: Type check with mypy
125-
working-directory: asap-tools/prometheus-client
125+
working-directory: asap-tools/queriers/prometheus-client
126126
run: mypy .
127127

128128
test-controller:
@@ -200,22 +200,22 @@ jobs:
200200
run: |
201201
python -m pip install --upgrade pip
202202
pip install black==24.8.0 flake8==6.1.0 mypy isort
203-
if [ -f asap-tools/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/prometheus-exporters/requirements.txt; fi
203+
if [ -f asap-tools/data-sources/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/data-sources/prometheus-exporters/requirements.txt; fi
204204
- name: Check formatting with Black
205-
working-directory: asap-tools/prometheus-exporters
205+
working-directory: asap-tools/data-sources/prometheus-exporters
206206
run: black --check --diff .
207207
- name: Check import sorting with isort
208-
working-directory: asap-tools/prometheus-exporters
208+
working-directory: asap-tools/data-sources/prometheus-exporters
209209
run: isort --check-only --diff --settings-file .isort.cfg .
210210
- name: Lint with flake8
211-
working-directory: asap-tools/prometheus-exporters
211+
working-directory: asap-tools/data-sources/prometheus-exporters
212212
run: |
213213
# Stop the build if there are Python syntax errors or undefined names
214-
flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
214+
flake8 . --config=../../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
215215
# Exit-zero treats all errors as warnings
216-
flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics
216+
flake8 . --config=../../../.flake8 --count --exit-zero --max-complexity=10 --statistics
217217
- name: Type check with mypy
218-
working-directory: asap-tools/prometheus-exporters
218+
working-directory: asap-tools/data-sources/prometheus-exporters
219219
run: mypy . --config-file=.mypy.ini
220220

221221
test-execution-utilities:

‎.github/workflows/rust.yml‎

Lines changed: 10 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ on:
88
- 'asap-common/dependencies/rs/**'
99
- 'asap-common/tests/**'
1010
- 'asap-common/sketch-core/**'
11+
- 'Cargo.toml'
12+
- 'Cargo.lock'
1113
- '.github/workflows/rust.yml'
1214
pull_request:
1315
branches: [ main ]
@@ -16,6 +18,8 @@ on:
1618
- 'asap-common/dependencies/rs/**'
1719
- 'asap-common/tests/**'
1820
- 'asap-common/sketch-core/**'
21+
- 'Cargo.toml'
22+
- 'Cargo.lock'
1923
- '.github/workflows/rust.yml'
2024
workflow_dispatch:
2125

@@ -28,12 +32,6 @@ jobs:
2832
steps:
2933
- uses: actions/checkout@v4
3034

31-
# - name: Configure git for private dependencies
32-
# run: git config --global url."https://x-access-token:${{ secrets.PRIVATE_REPO_TOKEN }}@github.com/".insteadOf "https://github.com/"
33-
34-
# - name: Clone sketchlib-rust
35-
# run: git clone https://github.com/ProjectASAP/sketchlib-rust.git
36-
3735
- name: Install Rust
3836
uses: dtolnay/rust-toolchain@stable
3937
with:
@@ -50,40 +48,13 @@ jobs:
5048
~/.cargo/registry
5149
~/.cargo/git
5250
target
53-
key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
54-
55-
- name: Validate lockfile
56-
run: |
57-
cp Cargo.lock Cargo.lock.bak
58-
cargo generate-lockfile
59-
if ! diff -q Cargo.lock Cargo.lock.bak >/dev/null; then
60-
echo "::error::Cargo.lock is out of sync. Run 'cargo update' or 'rm Cargo.lock && cargo build' locally and commit the updated Cargo.lock."
61-
exit 1
62-
fi
63-
rm Cargo.lock.bak
64-
working-directory: .
51+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
6552

6653
- name: Check formatting
67-
run: cargo fmt -- --check
68-
working-directory: asap-query-engine
69-
70-
- name: Check formatting (sketch_db_common)
71-
run: cargo fmt -- --check
72-
working-directory: asap-common/dependencies/rs/sketch_db_common
54+
run: cargo fmt --all -- --check
7355

7456
- name: Run clippy
75-
run: cargo clippy --all-targets --all-features -- -D warnings
76-
working-directory: asap-query-engine
77-
env:
78-
RUSTC_WRAPPER: sccache
79-
80-
- name: Check formatting (sketch-core)
81-
run: cargo fmt -- --check
82-
working-directory: asap-common/sketch-core
83-
84-
- name: Run clippy (sketch-core)
85-
run: cargo clippy --all-targets --all-features -- -D warnings
86-
working-directory: asap-common/sketch-core
57+
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
8758
env:
8859
RUSTC_WRAPPER: sccache
8960

@@ -92,12 +63,6 @@ jobs:
9263
steps:
9364
- uses: actions/checkout@v4
9465

95-
# - name: Configure git for private dependencies
96-
# run: git config --global url."https://x-access-token:${{ secrets.PRIVATE_REPO_TOKEN }}@github.com/".insteadOf "https://github.com/"
97-
98-
# - name: Clone sketchlib-rust
99-
# run: git clone https://github.com/ProjectASAP/sketchlib-rust.git
100-
10166
- name: Install Rust
10267
uses: dtolnay/rust-toolchain@stable
10368
with:
@@ -113,42 +78,10 @@ jobs:
11378
~/.cargo/registry
11479
~/.cargo/git
11580
target
116-
key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
117-
118-
- name: Validate lockfile
119-
run: |
120-
cp Cargo.lock Cargo.lock.bak
121-
cargo generate-lockfile
122-
if ! diff -q Cargo.lock Cargo.lock.bak >/dev/null; then
123-
echo "::error::Cargo.lock is out of sync. Run 'cargo update' or 'rm Cargo.lock && cargo build' locally and commit the updated Cargo.lock."
124-
exit 1
125-
fi
126-
rm Cargo.lock.bak
127-
128-
- name: Run QueryEngineRust tests
129-
run: cargo test
130-
working-directory: asap-query-engine
131-
env:
132-
RUSTC_WRAPPER: sccache
133-
134-
- name: Run sql_utilities tests
135-
run: cargo test
136-
working-directory: asap-common/dependencies/rs/sql_utilities
137-
env:
138-
RUSTC_WRAPPER: sccache
139-
140-
- name: Run promql_utilities tests
141-
run: cargo test
142-
working-directory: asap-common/dependencies/rs/promql_utilities
143-
env:
144-
RUSTC_WRAPPER: sccache
81+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
14582

146-
- name: Run sketch-core tests
147-
run: cargo test
148-
working-directory: asap-common/sketch-core
149-
- name: Run sketch_db_common tests
150-
run: cargo test
151-
working-directory: asap-common/dependencies/rs/sketch_db_common
83+
- name: Run all workspace tests
84+
run: cargo test --workspace
15285
env:
15386
RUSTC_WRAPPER: sccache
15487

@@ -166,7 +99,3 @@ jobs:
16699

167100
- name: Build Docker image
168101
run: docker build -f asap-query-engine/Dockerfile -t sketchdb-queryengine-rust:latest .
169-
# run: |
170-
# echo "${{ secrets.PRIVATE_REPO_TOKEN }}" > /tmp/git_token
171-
# docker build --secret id=git_token,src=/tmp/git_token -f QueryEngineRust/Dockerfile -t sketchdb-queryengine-rust:latest .
172-
# rm -f /tmp/git_token

‎.gitignore‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
target/
22
experiment_outputs/
33

4-
# Private repo, vendored locally until open sourced (see GitHub issue)
5-
sketchlib-rust/
6-
74
# Runtime and generated files
85
metadata/
96
preprocessed_configs/

0 commit comments

Comments
 (0)