diff --git a/acceptance/bundle/validate/strict/output.txt b/acceptance/bundle/validate/strict/output.txt index ca25c721f59..11f3f4944d8 100644 --- a/acceptance/bundle/validate/strict/output.txt +++ b/acceptance/bundle/validate/strict/output.txt @@ -33,7 +33,7 @@ Warning: invalid value "INVALID_TYPE" for enum field. Valid values are [complex] at variables.my_variable.type in databricks.yml:6:11 -Warning: invalid value "INVALID_TYPE" for enum field. Valid values are [whl jar] +Warning: invalid value "INVALID_TYPE" for enum field. Valid values are [whl jar tgz] at artifacts.my_artifact.type in databricks.yml:16:15 diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index f54ad43b576..e2bff61ad16 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -63,6 +63,22 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } + // A `tgz` artifact with `include`/`git` (and no user `build` command) is built + // by DABs itself. Produce the tarball, then expand globs so its output file is + // picked up for upload just like a build-command output. + if a.BuildCommand == "" && a.Type == config.ArtifactTarball && (len(a.Include) > 0 || a.Git != nil) { + cmdio.LogProgress(ctx, fmt.Sprintf("Building %s...", artifactName)) + if err := buildTarballArtifact(ctx, b, artifactName, a); err != nil { + logdiag.LogError(ctx, err) + break + } + bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName}) + a = b.Config.Artifacts[artifactName] + if logdiag.HasError(ctx) { + break + } + } + if a.Type == "whl" && a.DynamicVersion && cacheDir != "" { b.Metrics.AddBoolValue(metrics.ArtifactDynamicVersionIsSet, true) for ind, artifactFile := range a.Files { diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index 9f8b2e6eaed..ae93f946c19 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -3,6 +3,7 @@ package artifacts import ( "context" "errors" + "fmt" "maps" "os" "path/filepath" @@ -50,6 +51,17 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "") b.Metrics.AddBoolValue(metrics.ArtifactFilesIsSet, len(artifact.Files) != 0) + // A `tgz` artifact with `include`/`git` is built by DABs itself in the build + // phase (see artifacts.Build). `build` and `git`/`include` are mutually + // exclusive: either the user's command produces the tarball, or DABs does. + native := artifact.Type == config.ArtifactTarball && (len(artifact.Include) > 0 || artifact.Git != nil) + if native && artifact.BuildCommand != "" { + logdiag.LogError(ctx, fmt.Errorf("artifact %q: `build` cannot be combined with `git`/`include`", artifactName)) + } + if native && len(artifact.Files) == 0 { + logdiag.LogError(ctx, fmt.Errorf("artifact %q: a tgz artifact needs a `files` entry naming the output path", artifactName)) + } + l := b.Config.GetLocation("artifacts." + artifactName) dirPath := filepath.Dir(l.File) @@ -88,7 +100,9 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics logdiag.LogError(ctx, errors.New("misconfigured artifact: please specify 'build' or 'files' property")) } - if len(artifact.Files) > 0 && artifact.BuildCommand == "" { + // Skip glob expansion for a DABs-built tgz: its output file does not exist yet + // (it is produced in the build phase, which expands globs afterward). + if len(artifact.Files) > 0 && artifact.BuildCommand == "" && !native { bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName}) } diff --git a/bundle/artifacts/tarball.go b/bundle/artifacts/tarball.go new file mode 100644 index 00000000000..c09e1b48255 --- /dev/null +++ b/bundle/artifacts/tarball.go @@ -0,0 +1,153 @@ +package artifacts + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deploy/files" + "github.com/databricks/cli/libs/fileset" + libsync "github.com/databricks/cli/libs/sync" + "github.com/databricks/cli/libs/vfs" +) + +// tarballEpoch stamps every entry so the archive is reproducible: identical +// contents produce identical bytes regardless of file mtimes. Mirrors +// aicode.buildCodeSnapshot (the two packers could later share one implementation). +var tarballEpoch = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + +// buildTarballArtifact produces the gzipped tarball for a `type: tgz` artifact that +// DABs builds itself (no user `build` command). When `git` is set the tarball is a +// snapshot of that ref; otherwise it packs the working tree scoped to `include` +// paths (gitignore-honored). The result is written to the artifact's single output +// file, which the normal artifact upload path then uploads and references. +func buildTarballArtifact(ctx context.Context, b *bundle.Bundle, name string, a *config.Artifact) error { + if len(a.Files) != 1 { + return fmt.Errorf("artifact %q: a tgz artifact needs exactly one `files` entry naming the output path", name) + } + out := a.Files[0].Source // made absolute by artifacts.Prepare + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return err + } + f, err := os.Create(out) + if err != nil { + return err + } + defer f.Close() + + if a.Git != nil { + return tarballFromGit(ctx, b, a, f) + } + return tarballFromInclude(ctx, b, a, f) +} + +// tarballFromInclude packs the working tree, scoped to a.Include, using the bundle's +// sync walker so filtering matches bundle file sync (.gitignore + sync.include/exclude). +func tarballFromInclude(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w io.Writer) error { + opts, err := files.GetSyncOptions(ctx, b) + if err != nil { + return err + } + paths := a.Include + if len(paths) == 0 { + paths = []string{"."} + } + fl, err := libsync.NewFileList(ctx, opts.WorktreeRoot, opts.LocalRoot, paths, opts.Include, opts.Exclude) + if err != nil { + return err + } + list, err := fl.Files(ctx) + if err != nil { + return err + } + // Sort so the archive byte stream doesn't depend on walk order. + slices.SortFunc(list, func(x, y fileset.File) int { + return strings.Compare(x.Relative, y.Relative) + }) + + gzw := gzip.NewWriter(w) + tw := tar.NewWriter(gzw) + for _, file := range list { + if err := addFileToTarball(tw, b.SyncRoot, file); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + return gzw.Close() +} + +// tarballFromGit snapshots a git ref via `git archive`, so the archive reflects the +// committed tree at that ref rather than the working tree. Commit wins over Branch. +func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w io.Writer) error { + ref := a.Git.Commit + if ref == "" { + ref = a.Git.Branch + } + if ref == "" { + return errors.New("git artifact: specify git.commit or git.branch") + } + args := []string{"-C", b.SyncRootPath, "archive", "--format=tar.gz", ref} + if len(a.Include) > 0 { + args = append(args, "--") + args = append(args, a.Include...) + } + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Stdout = w + var stderr strings.Builder + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git archive %s: %w: %s", ref, err, stderr.String()) + } + return nil +} + +func addFileToTarball(tw *tar.Writer, root vfs.Path, f fileset.File) error { + rc, err := root.Open(f.Relative) + if err != nil { + return fmt.Errorf("open %s: %w", f.Relative, err) + } + defer rc.Close() + + info, err := rc.Stat() + if err != nil { + return fmt.Errorf("stat %s: %w", f.Relative, err) + } + // Only regular files; the walker never yields directories and symlinks are out + // of scope for a code snapshot. + if !info.Mode().IsRegular() { + return nil + } + + // Preserve the owner execute bit; normalize the rest. + mode := int64(0o644) + if info.Mode().Perm()&0o100 != 0 { + mode = 0o755 + } + hdr := &tar.Header{ + Typeflag: tar.TypeReg, + Name: f.Relative, + Size: info.Size(), + Mode: mode, + ModTime: tarballEpoch, + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("tar header for %s: %w", f.Relative, err) + } + if _, err := io.Copy(tw, rc); err != nil { + return fmt.Errorf("write %s: %w", f.Relative, err) + } + return nil +} diff --git a/bundle/artifacts/tarball_test.go b/bundle/artifacts/tarball_test.go new file mode 100644 index 00000000000..f37b69a1b18 --- /dev/null +++ b/bundle/artifacts/tarball_test.go @@ -0,0 +1,78 @@ +package artifacts + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// readTar returns entry name -> content, mode bits, and modtime. +func readTar(t *testing.T, b []byte) (map[string]string, map[string]int64, map[string]time.Time) { + t.Helper() + gzr, err := gzip.NewReader(bytes.NewReader(b)) + require.NoError(t, err) + tr := tar.NewReader(gzr) + content := map[string]string{} + modes := map[string]int64{} + mtimes := map[string]time.Time{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + body, err := io.ReadAll(tr) + require.NoError(t, err) + content[hdr.Name] = string(body) + modes[hdr.Name] = hdr.Mode + mtimes[hdr.Name] = hdr.ModTime + } + return content, modes, mtimes +} + +func TestAddFileToTarball(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "src", "train.py"), []byte("print('x')\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "run.sh"), []byte("echo hi\n"), 0o755)) + + root := vfs.MustNew(dir) + list, err := fileset.New(root).Files() + require.NoError(t, err) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + for _, f := range list { + require.NoError(t, addFileToTarball(tw, root, f)) + } + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + content, modes, mtimes := readTar(t, buf.Bytes()) + + // Entries keep their sync-root-relative slash paths. + assert.Equal(t, "print('x')\n", content["src/train.py"]) + assert.Equal(t, "echo hi\n", content["run.sh"]) + + // Reproducible: every entry is stamped with the fixed epoch. + assert.True(t, mtimes["src/train.py"].Equal(tarballEpoch)) + + // Owner execute bit preserved; other files normalized to 0644. Windows has no + // execute bit so skip the mode assertions there. + if runtime.GOOS != "windows" { + assert.Equal(t, int64(0o755), modes["run.sh"]) + assert.Equal(t, int64(0o644), modes["src/train.py"]) + } +} diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go index cb029bcb4be..4cd647b8659 100644 --- a/bundle/config/artifact.go +++ b/bundle/config/artifact.go @@ -12,14 +12,28 @@ const ArtifactPythonWheel ArtifactType = `whl` const ArtifactJar ArtifactType = `jar` +// ArtifactTarball is a gzipped tar of source files, built by DABs itself from +// `include` paths and/or a `git` ref rather than by a user `build` command. +// Uploaded and referenced like any other artifact file (e.g. as an AI Runtime +// task's code_source_path). +const ArtifactTarball ArtifactType = `tgz` + // Values returns all valid ArtifactType values func (ArtifactType) Values() []ArtifactType { return []ArtifactType{ ArtifactPythonWheel, ArtifactJar, + ArtifactTarball, } } +// ArtifactGit pins a `tgz` artifact to a git ref, so the tarball is a snapshot of +// that ref rather than of the working tree. Commit wins over Branch when both set. +type ArtifactGit struct { + Branch string `json:"branch,omitempty"` + Commit string `json:"commit,omitempty"` +} + type ArtifactFile struct { Source string `json:"source"` @@ -46,4 +60,13 @@ type Artifact struct { Executable exec.ExecutableType `json:"executable,omitempty"` DynamicVersion bool `json:"dynamic_version,omitempty"` + + // Include lists paths (relative to the bundle root) to pack into a `tgz` + // artifact, filtered like bundle file sync (.gitignore-honored). Building the + // tarball is done by DABs, so this is mutually exclusive with `build`. + Include []string `json:"include,omitempty"` + + // Git, when set on a `tgz` artifact, snapshots the given ref instead of the + // working tree. Mutually exclusive with `build`. + Git *ArtifactGit `json:"git,omitempty"` } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e47f89c505a..95924095300 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -53,14 +53,27 @@ artifacts: "source": "description": |- Required. The artifact source file. + "git": + "description": |- + For a `tgz` artifact, snapshot this git ref instead of the working tree. Mutually exclusive with `build`. + "$fields": + "branch": + "description": |- + The git branch to snapshot. + "commit": + "description": |- + The git commit to snapshot. Takes precedence over `branch` when both are set. + "include": + "description": |- + For a `tgz` artifact, the paths (relative to the bundle root) to pack into the tarball, filtered like bundle file sync (`.gitignore`-honored). Mutually exclusive with `build`. "path": "description": |- The local path of the directory for the artifact. "type": "description": |- - Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`. + The type of the artifact. Valid values are `whl`, `jar`, and `tgz`. Required if the artifact is a Python wheel. "markdown_description": |- - Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`. + The type of the artifact. Valid values are `whl`, `jar`, and `tgz`. Required if the artifact is a Python wheel. bundle: "description": |- The bundle attributes when deploying to this target. diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 5e62e921687..25d181f9733 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -6,7 +6,7 @@ package generated // EnumFields maps [dyn.Pattern] to valid enum values they should have. var EnumFields = map[string][]string{ "artifacts.*.executable": {"bash", "sh", "cmd"}, - "artifacts.*.type": {"whl", "jar"}, + "artifacts.*.type": {"whl", "jar", "tgz"}, "permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 37246ebdff4..a4d996c04fd 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3167,14 +3167,22 @@ "description": "The relative or absolute path to the built artifact files.", "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile" }, + "git": { + "description": "For a `tgz` artifact, snapshot this git ref instead of the working tree. Mutually exclusive with `build`.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactGit" + }, + "include": { + "description": "For a `tgz` artifact, the paths (relative to the bundle root) to pack into the tarball, filtered like bundle file sync (`.gitignore`-honored). Mutually exclusive with `build`.", + "$ref": "#/$defs/slice/string" + }, "path": { "description": "The local path of the directory for the artifact.", "$ref": "#/$defs/string" }, "type": { - "description": "Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.", + "description": "The type of the artifact. Valid values are `whl`, `jar`, and `tgz`. Required if the artifact is a Python wheel.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType", - "markdownDescription": "Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`." + "markdownDescription": "The type of the artifact. Valid values are `whl`, `jar`, and `tgz`. Required if the artifact is a Python wheel." } }, "additionalProperties": false @@ -3206,6 +3214,28 @@ } ] }, + "config.ArtifactGit": { + "oneOf": [ + { + "type": "object", + "properties": { + "branch": { + "description": "The git branch to snapshot.", + "$ref": "#/$defs/string" + }, + "commit": { + "description": "The git commit to snapshot. Takes precedence over `branch` when both are set.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "config.ArtifactType": { "type": "string" },