Skip to content

Commit 467e771

Browse files
committed
Add e2e crates and update workspace & CI
Introduce dedicated e2e crates and reorganize workspace/tests for end-to-end testing: add kalam-cli-e2e and rust-e2e SDK entries, move many CLI test files into cli/kalam-cli-e2e, and update workspace members in top-level Cargo.toml. Update CI/workflow settings (dart workspaces path and release build features) and refresh developer docs (AGENTS.md) with new e2e commands and build notes. Cargo.lock and various Cargo.toml files were updated to reflect new packages and workspace layout; backend Cargo.toml features comment clarified.
1 parent 181e698 commit 467e771

282 files changed

Lines changed: 922 additions & 956 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/dart-sdk.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ jobs:
153153
with:
154154
shared-key: dart-bridge
155155
cache-on-failure: true
156-
workspaces: link -> link/target
156+
workspaces: link/kalam-link-dart -> target
157157

158158
- name: Setup Flutter
159159
uses: subosito/flutter-action@v2

.github/workflows/release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -954,7 +954,7 @@ jobs:
954954
cargo build `
955955
-p kalamdb-server `
956956
--no-default-features `
957-
--features embedded-ui,traceability `
957+
--features embedded-ui,traceability,mimalloc,cloud-aws `
958958
--profile release-dist `
959959
--timings `
960960
--bin kalamdb-server

AGENTS.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,21 @@ Keep context small. Read only the files needed for the current task. Do not scan
6767
- Use `cargo nextest run` for tests unless explicitly told otherwise.
6868
- CLI smoke tests require a running server. Start the backend first, then run smoke tests from `cli`.
6969
- Smoke tests are required before committing changes that affect backend or CLI behavior.
70-
- For CLI e2e tests, run `cargo nextest run --features e2e-tests` without `--no-fail-fast`; fix the first failure, then rerun.
70+
- For CLI e2e tests, run `cargo nextest run -p kalam-cli-e2e` without `--no-fail-fast`; fix the first failure, then rerun.
7171
- For performance tests, benchmarks, or perf e2e cases, report each relevant runtime in seconds.
7272
- Add `#[ntest::timeout(time)]` to async tests using observed healthy runtime x 1.5.
7373

7474
## Commands
7575

7676
- Backend build: `cd backend && cargo build`
7777
- Backend run: `cd backend && cargo run --bin kalamdb-server`
78+
- Backend fast check (local dev, no S3/mimalloc/tracing): `cargo check -p kalamdb-server`
79+
- Backend prod-like build: `cargo build -p kalamdb-server --no-default-features --features embedded-ui,mimalloc,traceability,cloud-aws`
7880
- CLI build: `cd cli && cargo build --release`
7981
- CLI smoke: `cd cli && KALAMDB_SERVER_URL="http://localhost:3000" KALAMDB_ROOT_PASSWORD="mypass" cargo test --test smoke -- --nocapture`
8082
- Full sweep: start the backend server, then run `./scripts/test-all.sh` from the repo root.
83+
- Rust SDK (fast iteration): `cargo check -p kalam-client --features native-sdk,consumer,healthcheck`.
84+
- Rust SDK e2e: `cargo test -p kalam-client-e2e` (requires running server).
8185
- Version verification after SDK version changes: `python3 scripts/versions.py verify`
8286

8387
## SDK And Docs

Cargo.lock

Lines changed: 73 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,16 @@ members = [
4545
"backend/crates/kalamdb-pg",
4646
"backend/crates/kalamdb-server-auth",
4747
"backend",
48+
"cli",
49+
"cli/crates/kalam-schema-diff",
50+
"cli/kalam-cli-e2e",
4851
"link/link-common",
4952
"link/sdks/rust",
53+
"link/sdks/rust-e2e",
5054
"link/kalam-link-wasm",
5155
"link/kalam-consumer-wasm",
5256
"link/kalam-link-dart",
53-
"cli",
54-
"cli/crates/kalam-schema-diff"
57+
"link/sdks/python",
5558
]
5659
exclude = ["benchv2"]
5760

backend/Cargo.toml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,13 @@ num_cpus = { workspace = true }
7373
libc = { workspace = true }
7474

7575
[features]
76-
default = ["embedded-ui", "mimalloc", "traceability"]
76+
# Dev builds use `embedded-ui` only for faster compile (no mimalloc C build or traceability).
77+
# Production/release/docker builds pass explicit features — see docker/build and .github/workflows/release.yml.
78+
default = ["embedded-ui"]
7779
mimalloc = ["dep:mimalloc", "kalamdb-core/mimalloc", "kalamdb-observability/mimalloc"]
7880
embedded-ui = ["kalamdb-api/embedded-ui"]
7981
traceability = ["kalamdb-core/traceability", "kalamdb-observability/traceability"]
82+
cloud-aws = ["kalamdb-core/cloud-aws"]
8083
otel = ["dep:tracing-opentelemetry", "dep:opentelemetry", "dep:opentelemetry_sdk", "dep:opentelemetry-otlp"]
8184
# End-to-end tests that require a full test server instance
8285
e2e-tests = []

backend/crates/kalamdb-commons/Cargo.toml

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ categories = ["database"]
1212

1313
[dependencies]
1414
# workspace deps
15-
dashmap = { workspace = true }
1615
once_cell = { workspace = true }
1716
hex = { workspace = true }
18-
sha2 = { workspace = true }
1917
thiserror = { workspace = true }
2018
datafusion-common = { workspace = true, optional = true, default-features = false }
21-
parking_lot = { workspace = true }
22-
storekey = { workspace = true }
19+
dashmap = { workspace = true, optional = true }
20+
parking_lot = { workspace = true, optional = true }
21+
storekey = { workspace = true, optional = true }
22+
kalamdb-observability = { workspace = true, default-features = false, optional = true }
2323
# Optional dependencies - only enabled via features
2424
serde = { workspace = true, optional = true }
2525
serde_json = { workspace = true, optional = true }
@@ -33,8 +33,7 @@ rmp-serde = { workspace = true, optional = true }
3333
rmp = { workspace = true, optional = true }
3434
nanoid = { workspace = true, optional = true }
3535
log = { workspace = true, optional = true }
36-
kalamdb-observability = { workspace = true, default-features = false }
37-
tracing = { workspace = true }
36+
tracing = { workspace = true, optional = true }
3837

3938
[dev-dependencies]
4039
criterion = { workspace = true }
@@ -45,6 +44,7 @@ serde = [
4544
"dep:serde",
4645
"dep:serde_json",
4746
]
47+
storage = ["dep:dashmap", "dep:parking_lot", "dep:storekey"]
4848
arrow = [
4949
"serde",
5050
"dep:arrow",
@@ -66,22 +66,25 @@ conversions = [
6666
"rows",
6767
"schema-metadata",
6868
"dep:uuid",
69+
"dep:kalamdb-observability",
6970
]
7071
serialization = [
7172
"rows",
7273
"dep:flatbuffers",
7374
"dep:flexbuffers",
75+
"dep:kalamdb-observability",
7476
]
7577
arrow-utils = [
7678
"arrow",
7779
"rows",
7880
]
7981
websocket-auth = ["serde"]
8082
msgpack = ["serde", "dep:rmp", "dep:rmp-serde"]
81-
traceability = ["kalamdb-observability/traceability"]
83+
traceability = ["dep:kalamdb-observability", "dep:tracing", "kalamdb-observability/traceability"]
8284
# Full feature set including Arrow/DataFusion-backed utilities (for server use)
8385
full = [
8486
"serde",
87+
"storage",
8588
"rows",
8689
"conversions",
8790
"serialization",
@@ -106,3 +109,4 @@ path = "src/lib.rs"
106109
[[bench]]
107110
name = "snowflake_generator"
108111
harness = false
112+
required-features = ["storage"]

backend/crates/kalamdb-commons/src/helpers/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@ pub mod arrow_utils;
55
pub mod file_helpers;
66
pub mod naming;
77
pub mod security;
8+
#[cfg(feature = "storage")]
89
pub mod string_interner;
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
// IDs module
2+
#[cfg(feature = "storage")]
23
pub mod row_id;
34
pub mod seq_id;
5+
#[cfg(feature = "storage")]
46
pub mod snowflake;
57

8+
#[cfg(feature = "storage")]
69
pub use row_id::{SharedTableRowId, StreamTableRowId, UserTableRowId};
710
pub use seq_id::SeqId;
11+
#[cfg(feature = "storage")]
812
pub use snowflake::SnowflakeGenerator;

backend/crates/kalamdb-commons/src/ids/seq_id.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use serde::{
1313
Deserialize, Deserializer, Serialize, Serializer,
1414
};
1515

16+
#[cfg(feature = "storage")]
1617
use crate::{ids::SnowflakeGenerator, StorageKey};
1718

1819
/// Sequence ID for MVCC versioning
@@ -112,6 +113,7 @@ impl SeqId {
112113
/// This packs the timestamp together with the largest worker/sequence values
113114
/// so the returned SeqId encompasses every Snowflake generated at or before
114115
/// `timestamp_millis`.
116+
#[cfg(feature = "storage")]
115117
pub fn max_id_for_timestamp(timestamp_millis: u64) -> Result<Self, String> {
116118
// let normalized = timestamp_millis.max(Self::EPOCH);
117119
let id = SnowflakeGenerator::max_id_for_timestamp(timestamp_millis)?;
@@ -195,6 +197,7 @@ impl From<SeqId> for i64 {
195197
}
196198
}
197199

200+
#[cfg(feature = "storage")]
198201
impl StorageKey for SeqId {
199202
fn storage_key(&self) -> Vec<u8> {
200203
self.to_bytes().to_vec()
@@ -268,6 +271,7 @@ mod tests {
268271
}
269272

270273
#[test]
274+
#[cfg(feature = "storage")]
271275
fn test_seq_id_max_id_for_timestamp() {
272276
let ts = SeqId::EPOCH + 5000;
273277
let seq_id = SeqId::max_id_for_timestamp(ts).expect("seq id");

0 commit comments

Comments
 (0)