[POC] DABs AIR native fields

acceptance/bundle/validate/strict/output.txt

@@ -33,7 +33,7 @@ Warning: invalid value "INVALID_TYPE" for enum field. Valid values are [complex]
   at variables.my_variable.type
   in databricks.yml:6:11
 
-Warning: invalid value "INVALID_TYPE" for enum field. Valid values are [whl jar]
+Warning: invalid value "INVALID_TYPE" for enum field. Valid values are [whl jar tgz]
   at artifacts.my_artifact.type
   in databricks.yml:16:15
 

bundle/artifacts/build.go

@@ -63,6 +63,22 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 
 		}
 
+		// A `tgz` artifact with `include`/`git` (and no user `build` command) is built
+		// by DABs itself. Produce the tarball, then expand globs so its output file is
+		// picked up for upload just like a build-command output.
+		if a.BuildCommand == "" && a.Type == config.ArtifactTarball && (len(a.Include) > 0 || a.Git != nil) {
+			cmdio.LogProgress(ctx, fmt.Sprintf("Building %s...", artifactName))
+			if err := buildTarballArtifact(ctx, b, artifactName, a); err != nil {
+				logdiag.LogError(ctx, err)
+				break
+			}
+			bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName})
+			a = b.Config.Artifacts[artifactName]
+			if logdiag.HasError(ctx) {
+				break
+			}
+		}
+
 		if a.Type == "whl" && a.DynamicVersion && cacheDir != "" {
 			b.Metrics.AddBoolValue(metrics.ArtifactDynamicVersionIsSet, true)
 			for ind, artifactFile := range a.Files {

bundle/artifacts/prepare.go

@@ -3,6 +3,7 @@ package artifacts
 import (
 	"context"
 	"errors"
+	"fmt"
 	"maps"
 	"os"
 	"path/filepath"
@@ -50,6 +51,17 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 		b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "")
 		b.Metrics.AddBoolValue(metrics.ArtifactFilesIsSet, len(artifact.Files) != 0)
 
+		// A `tgz` artifact with `include`/`git` is built by DABs itself in the build
+		// phase (see artifacts.Build). `build` and `git`/`include` are mutually
+		// exclusive: either the user's command produces the tarball, or DABs does.
+		native := artifact.Type == config.ArtifactTarball && (len(artifact.Include) > 0 || artifact.Git != nil)
+		if native && artifact.BuildCommand != "" {
+			logdiag.LogError(ctx, fmt.Errorf("artifact %q: `build` cannot be combined with `git`/`include`", artifactName))
+		}
+		if native && len(artifact.Files) == 0 {
+			logdiag.LogError(ctx, fmt.Errorf("artifact %q: a tgz artifact needs a `files` entry naming the output path", artifactName))
+		}
+
 		l := b.Config.GetLocation("artifacts." + artifactName)
 		dirPath := filepath.Dir(l.File)
 
@@ -88,7 +100,9 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 			logdiag.LogError(ctx, errors.New("misconfigured artifact: please specify 'build' or 'files' property"))
 		}
 
-		if len(artifact.Files) > 0 && artifact.BuildCommand == "" {
+		// Skip glob expansion for a DABs-built tgz: its output file does not exist yet
+		// (it is produced in the build phase, which expands globs afterward).
+		if len(artifact.Files) > 0 && artifact.BuildCommand == "" && !native {
 			bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName})
 		}
 

bundle/artifacts/tarball.go

@@ -0,0 +1,153 @@
+package artifacts
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/deploy/files"
+	"github.com/databricks/cli/libs/fileset"
+	libsync "github.com/databricks/cli/libs/sync"
+	"github.com/databricks/cli/libs/vfs"
+)
+
+// tarballEpoch stamps every entry so the archive is reproducible: identical
+// contents produce identical bytes regardless of file mtimes. Mirrors
+// aicode.buildCodeSnapshot (the two packers could later share one implementation).
+var tarballEpoch = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+
+// buildTarballArtifact produces the gzipped tarball for a `type: tgz` artifact that
+// DABs builds itself (no user `build` command). When `git` is set the tarball is a
+// snapshot of that ref; otherwise it packs the working tree scoped to `include`
+// paths (gitignore-honored). The result is written to the artifact's single output
+// file, which the normal artifact upload path then uploads and references.
+func buildTarballArtifact(ctx context.Context, b *bundle.Bundle, name string, a *config.Artifact) error {
+	if len(a.Files) != 1 {
+		return fmt.Errorf("artifact %q: a tgz artifact needs exactly one `files` entry naming the output path", name)
+	}
+	out := a.Files[0].Source // made absolute by artifacts.Prepare
+	if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
+		return err
+	}
+	f, err := os.Create(out)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if a.Git != nil {
+		return tarballFromGit(ctx, b, a, f)
+	}
+	return tarballFromInclude(ctx, b, a, f)
+}
+
+// tarballFromInclude packs the working tree, scoped to a.Include, using the bundle's
+// sync walker so filtering matches bundle file sync (.gitignore + sync.include/exclude).
+func tarballFromInclude(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w io.Writer) error {
+	opts, err := files.GetSyncOptions(ctx, b)
+	if err != nil {
+		return err
+	}
+	paths := a.Include
+	if len(paths) == 0 {
+		paths = []string{"."}
+	}
+	fl, err := libsync.NewFileList(ctx, opts.WorktreeRoot, opts.LocalRoot, paths, opts.Include, opts.Exclude)
+	if err != nil {
+		return err
+	}
+	list, err := fl.Files(ctx)
+	if err != nil {
+		return err
+	}
+	// Sort so the archive byte stream doesn't depend on walk order.
+	slices.SortFunc(list, func(x, y fileset.File) int {
+		return strings.Compare(x.Relative, y.Relative)
+	})
+
+	gzw := gzip.NewWriter(w)
+	tw := tar.NewWriter(gzw)
+	for _, file := range list {
+		if err := addFileToTarball(tw, b.SyncRoot, file); err != nil {
+			return err
+		}
+	}
+	if err := tw.Close(); err != nil {
+		return err
+	}
+	return gzw.Close()
+}
+
+// tarballFromGit snapshots a git ref via `git archive`, so the archive reflects the
+// committed tree at that ref rather than the working tree. Commit wins over Branch.
+func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w io.Writer) error {
+	ref := a.Git.Commit
+	if ref == "" {
+		ref = a.Git.Branch
+	}
+	if ref == "" {
+		return errors.New("git artifact: specify git.commit or git.branch")
+	}
+	args := []string{"-C", b.SyncRootPath, "archive", "--format=tar.gz", ref}
+	if len(a.Include) > 0 {
+		args = append(args, "--")
+		args = append(args, a.Include...)
+	}
+	cmd := exec.CommandContext(ctx, "git", args...)
+	cmd.Stdout = w
+	var stderr strings.Builder
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("git archive %s: %w: %s", ref, err, stderr.String())
+	}
+	return nil
+}
+
+func addFileToTarball(tw *tar.Writer, root vfs.Path, f fileset.File) error {
+	rc, err := root.Open(f.Relative)
+	if err != nil {
+		return fmt.Errorf("open %s: %w", f.Relative, err)
+	}
+	defer rc.Close()
+
+	info, err := rc.Stat()
+	if err != nil {
+		return fmt.Errorf("stat %s: %w", f.Relative, err)
+	}
+	// Only regular files; the walker never yields directories and symlinks are out
+	// of scope for a code snapshot.
+	if !info.Mode().IsRegular() {
+		return nil
+	}
+
+	// Preserve the owner execute bit; normalize the rest.
+	mode := int64(0o644)
+	if info.Mode().Perm()&0o100 != 0 {
+		mode = 0o755
+	}
+	hdr := &tar.Header{
+		Typeflag: tar.TypeReg,
+		Name:     f.Relative,
+		Size:     info.Size(),
+		Mode:     mode,
+		ModTime:  tarballEpoch,
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		return fmt.Errorf("tar header for %s: %w", f.Relative, err)
+	}
+	if _, err := io.Copy(tw, rc); err != nil {
+		return fmt.Errorf("write %s: %w", f.Relative, err)
+	}
+	return nil
+}

bundle/artifacts/tarball_test.go

@@ -0,0 +1,78 @@
+package artifacts
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// readTar returns entry name -> content, mode bits, and modtime.
+func readTar(t *testing.T, b []byte) (map[string]string, map[string]int64, map[string]time.Time) {
+	t.Helper()
+	gzr, err := gzip.NewReader(bytes.NewReader(b))
+	require.NoError(t, err)
+	tr := tar.NewReader(gzr)
+	content := map[string]string{}
+	modes := map[string]int64{}
+	mtimes := map[string]time.Time{}
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		require.NoError(t, err)
+		body, err := io.ReadAll(tr)
+		require.NoError(t, err)
+		content[hdr.Name] = string(body)
+		modes[hdr.Name] = hdr.Mode
+		mtimes[hdr.Name] = hdr.ModTime
+	}
+	return content, modes, mtimes
+}
+
+func TestAddFileToTarball(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "src", "train.py"), []byte("print('x')\n"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "run.sh"), []byte("echo hi\n"), 0o755))
+
+	root := vfs.MustNew(dir)
+	list, err := fileset.New(root).Files()
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	gzw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gzw)
+	for _, f := range list {
+		require.NoError(t, addFileToTarball(tw, root, f))
+	}
+	require.NoError(t, tw.Close())
+	require.NoError(t, gzw.Close())
+
+	content, modes, mtimes := readTar(t, buf.Bytes())
+
+	// Entries keep their sync-root-relative slash paths.
+	assert.Equal(t, "print('x')\n", content["src/train.py"])
+	assert.Equal(t, "echo hi\n", content["run.sh"])
+
+	// Reproducible: every entry is stamped with the fixed epoch.
+	assert.True(t, mtimes["src/train.py"].Equal(tarballEpoch))
+
+	// Owner execute bit preserved; other files normalized to 0644. Windows has no
+	// execute bit so skip the mode assertions there.
+	if runtime.GOOS != "windows" {
+		assert.Equal(t, int64(0o755), modes["run.sh"])
+		assert.Equal(t, int64(0o644), modes["src/train.py"])
+	}
+}

bundle/config/artifact.go

@@ -12,14 +12,28 @@ const ArtifactPythonWheel ArtifactType = `whl`
 
 const ArtifactJar ArtifactType = `jar`
 
+// ArtifactTarball is a gzipped tar of source files, built by DABs itself from
+// `include` paths and/or a `git` ref rather than by a user `build` command.
+// Uploaded and referenced like any other artifact file (e.g. as an AI Runtime
+// task's code_source_path).
+const ArtifactTarball ArtifactType = `tgz`
+
 // Values returns all valid ArtifactType values
 func (ArtifactType) Values() []ArtifactType {
 	return []ArtifactType{
 		ArtifactPythonWheel,
 		ArtifactJar,
+		ArtifactTarball,
 	}
 }
 
+// ArtifactGit pins a `tgz` artifact to a git ref, so the tarball is a snapshot of
+// that ref rather than of the working tree. Commit wins over Branch when both set.
+type ArtifactGit struct {
+	Branch string `json:"branch,omitempty"`
+	Commit string `json:"commit,omitempty"`
+}
+
 type ArtifactFile struct {
 	Source string `json:"source"`
 
@@ -46,4 +60,13 @@ type Artifact struct {
 	Executable exec.ExecutableType `json:"executable,omitempty"`
 
 	DynamicVersion bool `json:"dynamic_version,omitempty"`
+
+	// Include lists paths (relative to the bundle root) to pack into a `tgz`
+	// artifact, filtered like bundle file sync (.gitignore-honored). Building the
+	// tarball is done by DABs, so this is mutually exclusive with `build`.
+	Include []string `json:"include,omitempty"`
+
+	// Git, when set on a `tgz` artifact, snapshots the given ref instead of the
+	// working tree. Mutually exclusive with `build`.
+	Git *ArtifactGit `json:"git,omitempty"`
 }

bundle/internal/schema/annotations.yml

@@ -53,14 +53,27 @@ artifacts:
         "source":
           "description": |-
             Required. The artifact source file.
+    "git":
+      "description": |-
+        For a `tgz` artifact, snapshot this git ref instead of the working tree. Mutually exclusive with `build`.
+      "$fields":
+        "branch":
+          "description": |-
+            The git branch to snapshot.
+        "commit":
+          "description": |-
+            The git commit to snapshot. Takes precedence over `branch` when both are set.
+    "include":
+      "description": |-
+        For a `tgz` artifact, the paths (relative to the bundle root) to pack into the tarball, filtered like bundle file sync (`.gitignore`-honored). Mutually exclusive with `build`.
     "path":
       "description": |-
         The local path of the directory for the artifact.
     "type":
       "description": |-
-        Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.
+        The type of the artifact. Valid values are `whl`, `jar`, and `tgz`. Required if the artifact is a Python wheel.
       "markdown_description": |-
-        Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.
+        The type of the artifact. Valid values are `whl`, `jar`, and `tgz`. Required if the artifact is a Python wheel.
 bundle:
   "description": |-
     The bundle attributes when deploying to this target.