Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ jobs:
run: sudo apt-get install -y libsqlite3-dev

- name: Run Go tests
run: CGO_ENABLED=1 go test ./internal/... -v
# ./... rather than ./internal/... so cmd/devcloud's ServicePlugin
# conformance test runs in CI too.
run: CGO_ENABLED=1 go test ./... -v
env:
CGO_ENABLED: "1"

Expand Down
11 changes: 8 additions & 3 deletions .github/workflows/smithy-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ jobs:
run: sudo apt-get install -y libsqlite3-dev

- name: Download latest Smithy models
run: bash scripts/download-smithy-models.sh
# --refresh re-downloads the committed models; without it every model
# already in the tree is skipped and this job can never find a change.
run: bash scripts/download-smithy-models.sh --refresh

- name: Run code generation
run: |
Expand All @@ -39,16 +41,19 @@ jobs:

- name: Check for changes
id: changes
# --porcelain rather than `git diff --quiet` so a newly generated
# package (an untracked directory) counts as a change instead of being
# silently dropped on the runner.
run: |
if git diff --quiet; then
if [ -z "$(git status --porcelain)" ]; then
echo "changed=false" >> $GITHUB_OUTPUT
else
echo "changed=true" >> $GITHUB_OUTPUT
fi

- name: Run tests
if: steps.changes.outputs.changed == 'true'
run: CGO_ENABLED=1 go test ./internal/... -v
run: CGO_ENABLED=1 go test ./... -v

- name: Create Pull Request
if: steps.changes.outputs.changed == 'true'
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ changelog:
@changie batch $(VERSION) && changie merge

stats:
@svcs=$$(awk '/^services:/{found=1;next} /^[^ \t]/{found=0} found && /^[ \t]+[a-z][a-z0-9_]+:$$/' internal/config/default.yaml | wc -l | tr -d ' '); \
@svcs=$$(grep -rho 'DefaultRegistry.Register(' internal/services | wc -l | tr -d ' '); \
ops=$$(grep -r 'case "' internal/services/*/provider.go 2>/dev/null | wc -l | tr -d ' '); \
echo "Services: $$svcs"; \
echo "Operations: $$ops"
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ Today DevCloud targets **AWS**. Our long-term goal is to support the full range
- **104 AWS services** across all major categories (run `make stats` for current counts; see [services-matrix.md](docs/services-matrix.md))
- **boto3-compatible** — a 729-test compatibility suite runs in CI (`make test-compat`); works with most boto3 apps, and unsupported operations return a clean AWS error, never a false success
- **Cross-service integration** — CFN provisioning, DDB Streams → Lambda, EventBridge targets, S3 → Lambda
- **Smithy-driven codegen** — auto-generate Go interfaces, types, and serializers from Smithy models
- **Smithy-driven codegen** — auto-generate Go types, routers, and error catalogues from Smithy models
- **Weekly auto-sync** — GitHub Actions keeps generated code up to date with upstream AWS API changes
- **Single binary, zero-config** — one Docker image, one port (4747), no config file required (embedded defaults)
- **Environment variable overrides** — `DEVCLOUD_SERVICES`, `DEVCLOUD_DATA_DIR`, `DEVCLOUD_PORT` for quick configuration without YAML
- **SDK/CLI compatible** — works with the AWS SDK, CLI, Terraform, CDK out of the box
- **Admin API** — opt-in REST + WebSocket at `/devcloud/api/*` for service status, resource listing, and live request logs (`admin.enabled: true`). The web dashboard UI lives in a separate repository.
- **Admin API** — opt-in REST at `/devcloud/api/*` for service status, resource listing, and recent request logs (`admin.enabled: true`). The web dashboard UI lives in a separate repository.

## Quick Start

Expand Down
2 changes: 1 addition & 1 deletion changes/unreleased/Added-20260718-081112.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Added
body: Add SQS dead-letter redrive operations (ListDeadLetterSourceQueues, StartMessageMoveTask, ListMessageMoveTasks, CancelMessageMoveTask) SQS now covers all 23 operations
body: Add the four SQS dead-letter redrive operations (ListDeadLetterSourceQueues, Start/List/CancelMessageMoveTask), completing SQS at 23 operations
time: 2026-07-18T08:11:12.483907+09:00
custom:
Issue: "94"
2 changes: 1 addition & 1 deletion changes/unreleased/Added-20260718-222651.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Added
body: Document the stable `ServicePlugin` plugin API in docs/plugin-api.md (method contracts, config keys, error convention, and the v1.x stability policy), enforced by a conformance test over every registered service
body: Document the `ServicePlugin` API and its v1.x stability policy in docs/plugin-api.md, enforced by a conformance test over every registered service
time: 2026-07-18T22:26:51.495862+09:00
custom:
Issue: "96"
2 changes: 1 addition & 1 deletion changes/unreleased/Added-20260718-222652.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Added
body: Enable and add boto3 compatibility tests for CodeConnections, DMS, and Verified Permissions (previously scaffolded but not enabled)
body: Enable boto3 compatibility tests for CodeConnections, DMS, and Verified Permissions
time: 2026-07-18T22:26:52.000000+09:00
custom:
Issue: "96"
2 changes: 1 addition & 1 deletion changes/unreleased/Added-20260718-233026.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Added
body: 'Add 24 operations to core services: EC2 instance lifecycle (Start/Stop/Reboot), DescribeAvailabilityZones/DescribeRegions, security group rules (Authorize/Revoke ingress+egress), Elastic IP association, DeleteSecurityGroup/DeleteSubnet/DescribeTags/DeleteTags, and IAM group policy management (attach/detach/put/get/delete/list)'
body: 'Add 24 core-service operations: EC2 instance lifecycle, region/AZ and tag queries, security group rules, Elastic IP association, and IAM group policy management'
time: 2026-07-18T23:30:26.666352+09:00
custom:
Issue: "97"
2 changes: 1 addition & 1 deletion changes/unreleased/Added-20260719-001541.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Added
body: 'Add a generic Smithy-driven CRUD fallback engine (internal/shared/crud) that auto-serves ~2,200 standard CRUD-shaped operations across all 46 JSON-protocol services with plausible, store-backed responses; hand-implemented operations always take precedence and unclassifiable operations still return an honest error. Documented in docs/crud-engine.md'
body: Serve ~2,200 standard CRUD operations across the 46 JSON-protocol services from a Smithy-driven fallback engine (`internal/shared/crud`); hand-written operations take precedence and unclassifiable ones still return an error. See docs/crud-engine.md
time: 2026-07-19T00:15:41.279904+09:00
custom:
Issue: "98"
2 changes: 1 addition & 1 deletion changes/unreleased/Changed-20260724-000001.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Changed
body: Rename the `internal/dashboard` package to `internal/admin` and the `dashboard.enabled` config key to `admin.enabled`, reflecting its role as a general-purpose admin/introspection API rather than a UI-bound dashboard. The old `dashboard.enabled` key is still honoured for one release with a deprecation warning.
body: Rename `internal/dashboard` to `internal/admin` and the `dashboard.enabled` config key to `admin.enabled`. The old key is honoured for one release with a deprecation warning.
time: 2026-07-24T00:00:01.000000+09:00
custom:
Issue: "111"
2 changes: 1 addition & 1 deletion changes/unreleased/Changed-20260725-000001.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Changed
body: Skip request log collection on the AWS request hot path when the admin API is disabled (the default). The log collector is now built only when `admin.enabled` is true, avoiding a per-request mutex lock and ring-buffer write that no consumer could read.
body: Build the request log collector only when `admin.enabled` is set, dropping a per-request mutex lock and ring-buffer write from the AWS request hot path
time: 2026-07-25T00:00:01.000000+09:00
custom:
Issue: "112"
5 changes: 5 additions & 0 deletions changes/unreleased/Changed-20260730-000001.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
kind: Changed
body: 'The `services` config block is now optional: omit it and every registered service starts with `data_dir ./data/<service>`; list services and only those start. Replaces 325 lines of identical boilerplate.'
time: 2026-07-30T00:00:01.000000+09:00
custom:
Issue: "120"
2 changes: 1 addition & 1 deletion changes/unreleased/Fixed-20260718-222651.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Fixed
body: Correct `X-Amz-Target` service routing for AWS services whose target prefix is dotted (e.g. CodeConnections, DMS); they previously misrouted to CloudTrail via an overbroad `com` prefix match
body: Route `X-Amz-Target` prefixes that contain dots (CodeConnections, DMS) to the right service; they misrouted to CloudTrail
time: 2026-07-18T22:26:51.484673+09:00
custom:
Issue: "96"
2 changes: 1 addition & 1 deletion changes/unreleased/Fixed-20260718-222652.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Fixed
body: Unimplemented operations now return a `NotImplemented` error (HTTP 501) instead of a misleading empty `200` success, so SDKs no longer see a false success and can tell a not-yet-emulated real operation apart from a genuinely invalid action; applied uniformly across all service providers (the ChangeResourceRecordSets record-action validation path keeps `InvalidChangeBatch`)
body: Unimplemented operations return `NotImplemented` (HTTP 501) instead of an empty `200`, so SDKs can tell a not-yet-emulated operation from an invalid one
time: 2026-07-18T22:26:52.000000+09:00
custom:
Issue: "96"
2 changes: 1 addition & 1 deletion changes/unreleased/Fixed-20260720-054448.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Fixed
body: 'EC2: Describe{SecurityGroups,Tags,Addresses} now honor GroupId/AllocationId/PublicIp selectors and Filters (and raise NotFound for unknown IDs) instead of returning every resource; delete-by-name removes a single security group; batch Start/Stop is atomic; AssociateAddress validates the instance; duplicate/absent security-group rules are rejected; Authorize/Revoke accept GroupName; all-traffic rules omit ports'
body: 'EC2: Describe{SecurityGroups,Tags,Addresses} honor their selectors and Filters instead of returning every resource, unknown IDs raise NotFound, batch Start/Stop is atomic, and duplicate or absent security group rules are rejected'
time: 2026-07-20T05:44:48.649862+09:00
custom:
Issue: "97"
2 changes: 1 addition & 1 deletion changes/unreleased/Fixed-20260720-055500.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Fixed
body: 'DynamoDB: numeric (''N'') sort/range keys now order by true numeric value across the full 38-digit precision via a NUMTEXT SQLite collation, instead of a float CAST that lost precision past 2^53'
body: 'DynamoDB: numeric sort keys order by true value over the full 38 digits via a NUMTEXT collation, instead of a float CAST that lost precision past 2^53'
time: 2026-07-20T05:55:00.000000+09:00
custom:
Issue: "97"
5 changes: 5 additions & 0 deletions changes/unreleased/Fixed-20260730-000002.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
kind: Fixed
body: 'The weekly Smithy model sync could never report an update: it skipped every model already in the tree (all of them), and 14 entries of its hand-maintained service list 404''d upstream. The list is now derived from the tree, the workflow re-downloads with `--refresh`, and downloads are atomic so a failed fetch cannot delete a committed model.'
time: 2026-07-30T00:00:02.000000+09:00
custom:
Issue: "120"
2 changes: 1 addition & 1 deletion changes/unreleased/Removed-20260724-000000.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
kind: Removed
body: Extract the Next.js web dashboard frontend to a separate repository. The Go binary no longer builds, bundles, or serves a web UI; it exposes only the opt-in admin API at `/devcloud/api/*`.
body: Extract the web dashboard to a separate repository. The binary builds and serves no UI, only the opt-in admin API at `/devcloud/api/*`.
time: 2026-07-24T00:00:00.000000+09:00
custom:
Issue: "111"
5 changes: 5 additions & 0 deletions changes/unreleased/Removed-20260730-000000.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
kind: Removed
body: 'Remove dead surface found by a repo-wide over-engineering audit: the event bus and admin WebSocket, the `GetMetrics` plugin API and `/devcloud/api/metrics` (resource counts remain on `/devcloud/api/services`), `shared.ResourceStore`, `gateway.ExtractAccountID`, and the `auth.enabled` key — still read for one release so enabling it warns that credentials are accepted regardless. Drops the `gorilla/websocket` dependency.'
time: 2026-07-30T00:00:00.000000+09:00
custom:
Issue: "120"
15 changes: 14 additions & 1 deletion cmd/codegen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ import (
"github.com/skyoo2003/devcloud/internal/codegen"
)

// handWritten services get no generated package: their provider is implemented
// by hand elsewhere (sts lives in internal/services/iam alongside its store).
// Their generated stubs were deleted in #91 and #96; without this skip every
// `make codegen` run resurrects them as untracked files that the sync
// workflow's diff check cannot see. sts is Query-protocol, so it contributes
// nothing to the JSON-only CRUD registry either.
var handWritten = map[string]bool{"sts": true}

func main() {
modelsDir := flag.String("models", "./smithy-models", "Directory containing Smithy JSON model files")
outputDir := flag.String("output", "./internal/generated", "Output directory for generated code")
Expand Down Expand Up @@ -59,6 +67,11 @@ func main() {
continue
}

if handWritten[model.ServiceID] {
fmt.Printf("Skipping %s (%s): hand-written provider\n", model.ServiceName, model.ServiceID)
continue
}

fmt.Printf("Generating %s (%s)...\n", model.ServiceName, model.ServiceID)
if err := gen.GenerateAll(model, *outputDir, *scaffoldDir); err != nil {
fmt.Fprintf(os.Stderr, "Error generating %s: %v\n", model.ServiceID, err)
Expand Down Expand Up @@ -86,7 +99,7 @@ func main() {
fmt.Fprintf(os.Stderr, "Error creating crudregistry dir: %v\n", err)
os.Exit(1)
}
if err := os.WriteFile(filepath.Join(regDir, "registry_gen.go"), []byte(content), 0644); err != nil {
if err := codegen.WriteGo(filepath.Join(regDir, "registry_gen.go"), content); err != nil {
fmt.Fprintf(os.Stderr, "Error writing CRUD registry: %v\n", err)
os.Exit(1)
}
Expand Down
43 changes: 0 additions & 43 deletions cmd/devcloud/buffer.go

This file was deleted.

49 changes: 0 additions & 49 deletions cmd/devcloud/buffer_test.go

This file was deleted.

Loading