From 8d1bee70471172232707aca153a83cf7ea959073 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 9 Aug 2026 16:51:58 +0900 Subject: [PATCH 1/6] fix(storage): swap sqlite driver to pure Go so released binaries run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .goreleaser.yaml builds with CGO_ENABLED=0 while the store imported mattn/go-sqlite3, which is cgo-only. That combination compiles: the driver ships a stub for the no-cgo build, and the stub fails at run time. Every tagged binary from v0.2.0 onward therefore exited at startup with init s3: enable WAL: Binary was compiled with 'CGO_ENABLED=0', go-sqlite3 requires cgo to work. This is a stub so the tar.gz/zip artifacts and the Homebrew formula were dead on arrival. Only the container image worked, because docker/Dockerfile builds with CGO_ENABLED=1 against sqlite-dev. modernc.org/sqlite is a pure Go translation of SQLite and needs no toolchain per target. Its RegisterCollationUtf8 takes func(left, right string) int, the exact signature of compareNumericText, so NUMTEXT ports as-is — and because registration is process-wide, the per-connection ConnectHook goes away. The public surface of this package is unchanged, so none of its 108 importers move. IsUniqueConstraintError already matched on the message text rather than a driver error type, and the on-disk file format is SQLite's, so existing data_dir contents still open. Verified: full suite green under both CGO_ENABLED=1 and CGO_ENABLED=0; the boto3 suite green (775) against a CGO_ENABLED=0 binary, which previously could not start; all six release targets cross-compile. The compatibility suite runs about 15% slower (63s against 55s), the expected cost of transpiled C. --- go.mod | 11 ++++++- go.sum | 52 ++++++++++++++++++++++++++++++-- internal/storage/sqlite/store.go | 25 +++++++-------- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index d6921de..6603e48 100644 --- a/go.mod +++ b/go.mod @@ -5,15 +5,24 @@ go 1.26 toolchain go1.26.1 require ( - github.com/mattn/go-sqlite3 v1.14.49 github.com/stretchr/testify v1.11.1 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.56.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + golang.org/x/sys v0.47.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 1221b8c..e7a2414 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,14 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -8,18 +16,58 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= -github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/storage/sqlite/store.go b/internal/storage/sqlite/store.go index e20ee0b..f95d008 100644 --- a/internal/storage/sqlite/store.go +++ b/internal/storage/sqlite/store.go @@ -10,21 +10,22 @@ import ( "path/filepath" "strings" - sqlite3 "github.com/mattn/go-sqlite3" + sqlite "modernc.org/sqlite" ) -// driverName is a sqlite3 driver variant that registers the NUMTEXT collation -// on every connection. NUMTEXT orders TEXT columns holding numeric strings by -// true numeric value (exact across DynamoDB's full 38-digit precision), which -// a float CAST cannot do past 2^53. -const driverName = "sqlite3_numtext" - +// driverName is the driver modernc.org/sqlite registers on import. It is a pure +// Go translation of SQLite, chosen because DevCloud's released binaries are +// built with CGO_ENABLED=0: a cgo-only driver still compiles under that flag +// and then exits at startup, which is how every tagged binary through v0.2.0 +// shipped unable to open its own database. +const driverName = "sqlite" + +// NUMTEXT orders TEXT columns holding numeric strings by true numeric value +// (exact across DynamoDB's full 38-digit precision), which a float CAST cannot +// do past 2^53. Registration is process-wide here, so every connection carries +// it without a per-connection hook. func init() { - sql.Register(driverName, &sqlite3.SQLiteDriver{ - ConnectHook: func(conn *sqlite3.SQLiteConn) error { - return conn.RegisterCollation("NUMTEXT", compareNumericText) - }, - }) + sqlite.MustRegisterCollationUtf8("NUMTEXT", compareNumericText) } func compareNumericText(a, b string) int { From 96ae3c84ef6392772baefd5f9617e63b41240176 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 9 Aug 2026 21:17:19 +0900 Subject: [PATCH 2/6] build: drop the C toolchain and test the mode that ships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the driver in pure Go, nothing in the tree needs cgo or SQLite headers. Six workflows installed libsqlite3-dev, two Dockerfiles installed gcc/musl-dev/sqlite-dev, the runtime image installed sqlite-libs, and troubleshooting told users to install headers to fix a build error they can no longer hit. All of it goes. The substantive part is CGO_ENABLED: everything now builds and tests at 0, the mode .goreleaser.yaml publishes in. Testing CGO_ENABLED=1 while shipping CGO_ENABLED=0 is how a binary that cannot open its own database passed every gate for two releases. The boto3 suite in particular now runs against a binary built exactly as the released one is, so those 775 tests are the smoke test for what users download. Verified: full suite green at CGO_ENABLED=0 (109 packages); golangci-lint clean; the linux/amd64 binary is static with no reference to libsqlite3, which is why the runtime image no longer needs sqlite-libs. The container build itself is unverified locally — no Docker daemon here — and cd.yml does not run on pull requests, so it is first exercised on merge. --- .github/workflows/ci.yml | 15 ++++++++------- .github/workflows/codeql.yml | 4 ---- .github/workflows/compat.yml | 7 +++---- .github/workflows/lint.yml | 3 --- .github/workflows/release.yml | 12 ++++++------ .github/workflows/smithy-sync.yml | 7 ++----- Makefile | 2 +- docker/Dockerfile | 7 ++++--- docker/Dockerfile.dev | 3 +-- docs/release.md | 2 +- docs/troubleshooting.md | 11 ----------- 11 files changed, 26 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 152b09e..b00b9f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,20 +25,21 @@ jobs: with: go-version: "1.26" - - name: Install SQLite dev headers - run: sudo apt-get install -y libsqlite3-dev - + # CGO_ENABLED=0 throughout, matching .goreleaser.yaml: the SQLite driver + # is pure Go, and testing the mode that ships is the point — a cgo-only + # driver once compiled here and then died at startup in every release + # archive, because nothing exercised the released build configuration. - name: Run Go tests # ./... rather than ./internal/... so cmd/devcloud's ServicePlugin # conformance test runs in CI too. - run: CGO_ENABLED=1 go test ./... -v + run: go test ./... -v env: - CGO_ENABLED: "1" + CGO_ENABLED: "0" - name: Build Go binaries run: | - CGO_ENABLED=1 go build -o dist/devcloud ./cmd/devcloud - CGO_ENABLED=1 go build -o dist/codegen ./cmd/codegen + CGO_ENABLED=0 go build -o dist/devcloud ./cmd/devcloud + CGO_ENABLED=0 go build -o dist/codegen ./cmd/codegen # internal/generated is committed but derived. The Go tests check the fidelity # manifest's shape — floors, registered services, the CRUD registry — none of diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d7e6022..a463f8d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -32,10 +32,6 @@ jobs: with: go-version: "1.26" - - name: Install SQLite dev headers - if: matrix.language == 'go' - run: sudo apt-get install -y libsqlite3-dev - - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index 0b456af..492a5e9 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -17,11 +17,10 @@ jobs: with: go-version: "1.26" - - name: Install SQLite dev headers - run: sudo apt-get install -y libsqlite3-dev - + # CGO_ENABLED=0 is the mode .goreleaser.yaml publishes, so these tests + # double as the smoke test for the binary users actually download. - name: Build devcloud binary - run: CGO_ENABLED=1 go build -o dist/devcloud ./cmd/devcloud + run: CGO_ENABLED=0 go build -o dist/devcloud ./cmd/devcloud - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b187e71..f1bb63b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,9 +21,6 @@ jobs: with: go-version: "1.26" - - name: Install SQLite dev headers - run: sudo apt-get install -y libsqlite3-dev - - name: Run golangci-lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbd0ba8..e32fa20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,10 +74,10 @@ jobs: with: go-version-file: go.mod cache-dependency-path: go.sum - - name: Install SQLite dev headers - run: sudo apt-get install -y libsqlite3-dev - name: Run Go tests - run: CGO_ENABLED=1 go test ./... + # CGO_ENABLED=0, the mode GoReleaser publishes in — a gate that tests a + # build configuration the release does not use vouches for nothing. + run: CGO_ENABLED=0 go test ./... # The Go tests cannot see a stale fidelity manifest — that is the whole reason # ci.yml has a codegen-drift job — so waiting only on `test` would let a tag @@ -128,10 +128,10 @@ jobs: with: go-version-file: go.mod cache-dependency-path: go.sum - - name: Install SQLite dev headers - run: sudo apt-get install -y libsqlite3-dev - name: Build devcloud binary - run: CGO_ENABLED=1 go build -o dist/devcloud ./cmd/devcloud + # Same build configuration as .goreleaser.yaml, so the 775 tests below + # exercise the artifact this tag is about to publish. + run: CGO_ENABLED=0 go build -o dist/devcloud ./cmd/devcloud - name: Set up Python uses: actions/setup-python@v7 with: diff --git a/.github/workflows/smithy-sync.yml b/.github/workflows/smithy-sync.yml index 4de28b6..9046df7 100644 --- a/.github/workflows/smithy-sync.yml +++ b/.github/workflows/smithy-sync.yml @@ -20,9 +20,6 @@ jobs: with: go-version: "1.26" - - name: Install SQLite dev headers - run: sudo apt-get install -y libsqlite3-dev - - name: Download latest Smithy models # --refresh re-downloads the committed models; without it every model # already in the tree is skipped and this job can never find a change. @@ -30,7 +27,7 @@ jobs: - name: Run code generation run: | - CGO_ENABLED=1 go run ./cmd/codegen \ + go run ./cmd/codegen \ -models ./smithy-models \ -output ./internal/generated \ -templates ./internal/codegen/templates \ @@ -53,7 +50,7 @@ jobs: - name: Run tests if: steps.changes.outputs.changed == 'true' - run: CGO_ENABLED=1 go test ./... -v + run: CGO_ENABLED=0 go test ./... -v - name: Create Pull Request if: steps.changes.outputs.changed == 'true' diff --git a/Makefile b/Makefile index 4d773a8..353404c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ build: go build -o dist/codegen ./cmd/codegen test: - CGO_ENABLED=1 go test ./... -v + CGO_ENABLED=0 go test ./... -v test-compat: cd tests/compatibility && pip install -q -r requirements.txt && pytest -v diff --git a/docker/Dockerfile b/docker/Dockerfile index 81aca64..f182dbc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,17 +1,18 @@ # Stage 1: Build Go binary FROM golang:1.26-alpine AS go-builder -RUN apk add --no-cache gcc musl-dev sqlite-dev WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -ENV CGO_ENABLED=1 +# No C toolchain, no SQLite headers: the driver is pure Go. That is the same +# property that lets the released binaries build without one. +ENV CGO_ENABLED=0 RUN go build -o /devcloud ./cmd/devcloud RUN go build -o /codegen ./cmd/codegen # Stage 2: Runtime FROM alpine:3.20 -RUN apk add --no-cache sqlite-libs ca-certificates su-exec +RUN apk add --no-cache ca-certificates su-exec RUN adduser -D -H -h /app appuser WORKDIR /app COPY --from=go-builder /devcloud /app/devcloud diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index ba5322c..ef248a1 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,5 +1,4 @@ FROM golang:1.26-alpine -RUN apk add --no-cache gcc musl-dev sqlite-dev RUN adduser -D -H -h /app appuser WORKDIR /app COPY go.mod go.sum ./ @@ -7,7 +6,7 @@ RUN go mod download COPY . . RUN chown -R appuser:appuser /app USER appuser -ENV CGO_ENABLED=1 +ENV CGO_ENABLED=0 RUN go build -o /app/devcloud ./cmd/devcloud EXPOSE 4747 CMD ["/app/devcloud"] diff --git a/docs/release.md b/docs/release.md index 587f8fe..68e1e40 100644 --- a/docs/release.md +++ b/docs/release.md @@ -50,7 +50,7 @@ The rest are only caught here. generator overwrites the outputs it still emits but never removes one it has stopped emitting, so regenerating in place leaves a retired file looking current. - [ ] **boto3 compatibility passes** — `make test-compat`. -- [ ] **Go tests pass** — `CGO_ENABLED=1 go test ./...`. +- [ ] **Go tests pass** — `CGO_ENABLED=0 go test ./...`. - [ ] **`main` is green**, including lint and CodeQL. - [ ] **Every unreleased fragment carries an issue number** — `grep -L 'Issue: "[0-9]' changes/unreleased/*.yaml` prints nothing. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index da387d6..eb9752f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -4,17 +4,6 @@ Common issues and how to resolve them. If your problem isn't listed here, check ## Installation & Build -### `sqlite3.h: No such file or directory` during `make build` - -DevCloud requires SQLite development headers because the server uses CGO for SQLite access. - -- **macOS**: `brew install sqlite3` -- **Ubuntu / Debian**: `sudo apt-get install libsqlite3-dev` -- **Fedora / RHEL**: `sudo dnf install sqlite-devel` -- **Alpine**: `apk add sqlite-dev build-base` - -Then rebuild with `CGO_ENABLED=1 make build`. - ### `go: module github.com/skyoo2003/devcloud: Go 1.26 required` Upgrade Go to 1.26 or later. Check with `go version`. See [go.dev/dl](https://go.dev/dl/). From e230fb5311fc68f4716db5b6afc6233e12f206b4 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 9 Aug 2026 21:18:20 +0900 Subject: [PATCH 3/6] docs(changelog): fragment for the pure Go sqlite driver --- changes/unreleased/Fixed-20260809-190000.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/unreleased/Fixed-20260809-190000.yaml diff --git a/changes/unreleased/Fixed-20260809-190000.yaml b/changes/unreleased/Fixed-20260809-190000.yaml new file mode 100644 index 0000000..a4983a3 --- /dev/null +++ b/changes/unreleased/Fixed-20260809-190000.yaml @@ -0,0 +1,5 @@ +kind: Fixed +body: Downloadable binaries now start. Releases were built with CGO_ENABLED=0 while the SQLite driver required cgo, so every tar.gz/zip binary, the Homebrew formula and the versioned `*-alpine` images exited at startup with "go-sqlite3 requires cgo to work. This is a stub" — only the rolling `latest` image, built separately with cgo, worked. The driver is now pure Go, so no build needs a C toolchain or SQLite headers +time: 2026-08-09T19:00:00.000000+09:00 +custom: + Issue: "128" From e1e3a2db9d45794ac6f269cdd1821a469bb78c13 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 9 Aug 2026 22:18:54 +0900 Subject: [PATCH 4/6] fix(storage): keep SQLite's busy timeout through the driver swap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mattn/go-sqlite3 set PRAGMA busy_timeout to 5000ms on every connection whether or not the DSN asked for it (sqlite3.go:1201, :1623). modernc.org/sqlite applies the pragma only when the DSN carries _busy_timeout or _timeout (sqlite.go:364-365), and Open passed a bare path, so the swap dropped that 5-second retry window and left SQLite's default of no waiting at all. The window matters here. The gateway serves on net/http (gateway.go:68,97), so handlers run concurrently, and Open sets no connection limit — database/sql opens as many connections as callers ask for. Two handlers writing one service database therefore hold two SQLite connections contending for a single write lock. DynamoDB's RWMutex does not prevent this: PutItem and DeleteItem take RLock (store.go:360, :470), which guards the in-memory table map, not the write transaction beneath it. s3 metadata tags, secretsmanager, sns and cloudwatchlogs take no lock at all. Nothing in the tree retries on SQLITE_BUSY, so the loser's error surfaces to the SDK caller as a spurious "database is locked" — measured at 0.96ms to failure, against the 5s it used to wait. The setting travels in the DSN rather than as an Exec beside the journal_mode pragma because busy_timeout is per-connection state. One Exec configures whichever pooled connection answers and leaves the rest at zero: measured at 1.06ms to failure, indistinguishable from no fix. journal_mode gets away with a single Exec only because the database file records it. 5000ms restores mattn's prior behaviour exactly. The regression test holds the write lock for 200ms and asserts the second writer waits it out; it fails with SQLITE_BUSY in 0.10s against the unfixed Open. Reported in review on #128. --- internal/storage/sqlite/store.go | 9 ++++++- internal/storage/sqlite/store_test.go | 35 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/internal/storage/sqlite/store.go b/internal/storage/sqlite/store.go index f95d008..81aa8f3 100644 --- a/internal/storage/sqlite/store.go +++ b/internal/storage/sqlite/store.go @@ -57,7 +57,14 @@ func Open(dbPath string, migrations []Migration) (*Store, error) { return nil, fmt.Errorf("create data dir: %w", err) } - db, err := sql.Open(driverName, dbPath) + // busy_timeout has to travel in the DSN so every pooled connection gets it. + // It is per-connection state, unlike journal_mode below, which the database + // file remembers — a single Exec here would configure whichever connection + // happened to answer and leave the rest at SQLite's default of no waiting. + // mattn/go-sqlite3 applied 5s unconditionally; modernc.org/sqlite installs + // no busy handler unless asked, so without this a second concurrent writer + // fails instantly with SQLITE_BUSY instead of waiting its turn. + db, err := sql.Open(driverName, dbPath+"?_pragma=busy_timeout(5000)") if err != nil { return nil, fmt.Errorf("open sqlite: %w", err) } diff --git a/internal/storage/sqlite/store_test.go b/internal/storage/sqlite/store_test.go index 1802d56..72fae02 100644 --- a/internal/storage/sqlite/store_test.go +++ b/internal/storage/sqlite/store_test.go @@ -5,6 +5,7 @@ package sqlite import ( "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -115,3 +116,37 @@ func TestNumtextCollationOrdering(t *testing.T) { assert.Equal(t, []string{"10", "100", "2"}, read("v")) // byte order assert.Equal(t, []string{"2", "10", "100"}, read("v COLLATE NUMTEXT")) // numeric order } + +// TestOpen_SecondWriterWaitsForTheLock pins the busy timeout Open puts in the +// DSN. The gateway serves requests concurrently and Open sets no connection +// limit, so two handlers writing the same service database hold two different +// SQLite connections and contend for one write lock. Without a busy handler the +// loser fails in under a millisecond and the SDK caller sees a spurious +// "database is locked". +func TestOpen_SecondWriterWaitsForTheLock(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + store, err := Open(dbPath, []Migration{ + {Version: 1, SQL: `CREATE TABLE items (id TEXT PRIMARY KEY)`}, + }) + require.NoError(t, err) + defer func() { _ = store.Close() }() + + // One connection takes the write lock and holds it well inside the timeout. + tx, err := store.DB().Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id) VALUES ('held')`) + require.NoError(t, err) + + const hold = 200 * time.Millisecond + go func() { + time.Sleep(hold) + _ = tx.Rollback() + }() + + start := time.Now() + _, err = store.DB().Exec(`INSERT INTO items (id) VALUES ('waited')`) + elapsed := time.Since(start) + + require.NoError(t, err, "second writer got SQLITE_BUSY instead of waiting for the lock") + assert.GreaterOrEqual(t, elapsed, hold, "second writer returned before the lock was released") +} From 88b3d2c5881dd2a587ee420ce762474c9d8c9e14 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 9 Aug 2026 22:21:09 +0900 Subject: [PATCH 5/6] build: run the pre-commit Go hooks in the mode releases ship MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The go-vet and go-build hooks forced CGO_ENABLED=1. That was never a response to the cgo-only SQLite driver — both lines date from the bootstrap commit 7d3b17d, before internal/storage/sqlite existed — but with the driver now pure Go it is the last place in the tree that asks a contributor for a C toolchain. Someone without one had every commit rejected by hooks testing a build configuration the project no longer produces. CGO_ENABLED=0 matches .goreleaser.yaml, ci.yml, compat.yml, smithy-sync.yml and the Makefile, so the hooks now vet and build what the release actually contains rather than a second, untested mode. Verified: CGO_ENABLED=0 go vet ./... and go build ./... both exit 0, and no CGO_ENABLED=1 remains anywhere in the tree. --- .pre-commit-config.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a5fade..8db99a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,16 +29,19 @@ repos: language: system types: [go] exclude: ^internal/generated/ + # CGO_ENABLED=0, matching .goreleaser.yaml and the CI workflows: it is the + # mode releases are built in, and no dependency needs a C toolchain, so a + # contributor without one still gets these hooks. - id: go-vet name: go vet - entry: bash -c 'CGO_ENABLED=1 go vet ./...' + entry: bash -c 'CGO_ENABLED=0 go vet ./...' language: system pass_filenames: false types: [go] exclude: ^internal/generated/ - id: go-build name: go build - entry: bash -c 'CGO_ENABLED=1 go build ./...' + entry: bash -c 'CGO_ENABLED=0 go build ./...' language: system pass_filenames: false types: [go] From f68ad8864ecaac32a3b6243cc2c18fbda18db09d Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 9 Aug 2026 23:22:11 +0900 Subject: [PATCH 6/6] test(storage): take the start timestamp before the unlock goroutine The busy-timeout regression test read start after launching the goroutine that releases the write lock, so a goroutine scheduled first could roll back less than hold after the timestamp and fail the elapsed >= hold assertion on scheduling alone. --- internal/storage/sqlite/store_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/storage/sqlite/store_test.go b/internal/storage/sqlite/store_test.go index 72fae02..0b28599 100644 --- a/internal/storage/sqlite/store_test.go +++ b/internal/storage/sqlite/store_test.go @@ -137,13 +137,16 @@ func TestOpen_SecondWriterWaitsForTheLock(t *testing.T) { _, err = tx.Exec(`INSERT INTO items (id) VALUES ('held')`) require.NoError(t, err) + // start is read before the goroutine launches. Taken after, the sleep could + // already be underway and the rollback land less than hold after start, + // failing the assertion on scheduling alone. const hold = 200 * time.Millisecond + start := time.Now() go func() { time.Sleep(hold) _ = tx.Rollback() }() - start := time.Now() _, err = store.DB().Exec(`INSERT INTO items (id) VALUES ('waited')`) elapsed := time.Since(start)