diff --git a/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml b/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml new file mode 100644 index 00000000000..15b90fec1bb --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: pipeline_and_deploy diff --git a/acceptance/bundle/generate/pipeline_and_deploy/notebook.py b/acceptance/bundle/generate/pipeline_and_deploy/notebook.py new file mode 100644 index 00000000000..1a5691c338e --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Hello world!") diff --git a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml new file mode 100644 index 00000000000..bbc7fcfd1bd --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_and_deploy/output.txt b/acceptance/bundle/generate/pipeline_and_deploy/output.txt new file mode 100644 index 00000000000..25943c80bcf --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/output.txt @@ -0,0 +1,32 @@ + +=== Upload files to workspace +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/notebook.py --file notebook.py --format AUTO --overwrite + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/test.py --file test.py --format AUTO --overwrite + +=== Create a pipeline that references the filesCreated pipeline + +=== Generate bundle config from the pipelineFile successfully saved to src/notebook.py +File successfully saved to src/test.py +Pipeline configuration successfully saved to resources/out.pipeline.yml + +=== Verify generated yaml has expected fields +=== Deploy the generated bundle +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy/default/files... +Deploying resources... +Deployment complete! + +=== Destroy the deployed bundle +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy/default + +Deleting files... +Destroy complete! + +=== Cleanup: delete the original pipeline and files +>>> errcode [CLI] pipelines delete [PIPELINE_ID] + +>>> errcode [CLI] workspace delete /Workspace/Users/[USERNAME]/notebook + +>>> errcode [CLI] workspace delete /Workspace/Users/[USERNAME]/test.py diff --git a/acceptance/bundle/generate/pipeline_and_deploy/script b/acceptance/bundle/generate/pipeline_and_deploy/script new file mode 100644 index 00000000000..97d078d86f6 --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/script @@ -0,0 +1,42 @@ +title "Upload files to workspace" +trace $CLI workspace import "/Workspace/Users/${CURRENT_USER_NAME}/notebook.py" --file notebook.py --format AUTO --overwrite +trace $CLI workspace import "/Workspace/Users/${CURRENT_USER_NAME}/test.py" --file test.py --format AUTO --overwrite + +title "Create a pipeline that references the files" +PIPELINE_ID=$($CLI pipelines create --json '{ + "name": "test-pipeline", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/'${CURRENT_USER_NAME}'/notebook" + } + }, + { + "file": { + "path": "/Workspace/Users/'${CURRENT_USER_NAME}'/test.py" + } + } + ] +}' | jq -r '.pipeline_id') +echo "Created pipeline" +add_repl.py "$PIPELINE_ID" PIPELINE_ID + +cleanup() { + title "Cleanup: delete the original pipeline and files" + trace errcode $CLI pipelines delete "$PIPELINE_ID" + trace errcode $CLI workspace delete "/Workspace/Users/${CURRENT_USER_NAME}/notebook" + trace errcode $CLI workspace delete "/Workspace/Users/${CURRENT_USER_NAME}/test.py" +} +trap cleanup EXIT + +title "Generate bundle config from the pipeline" +$CLI bundle generate pipeline --existing-pipeline-id "$PIPELINE_ID" --key out --config-dir resources --source-dir src --force 2>&1 | sort + +title "Verify generated yaml has expected fields" +cat resources/out.pipeline.yml | contains.py "libraries:" "- notebook:" "path: ../src/notebook.py" "- file:" "path: ../src/test.py" > /dev/null + +title "Deploy the generated bundle" +trace $CLI bundle deploy + +title "Destroy the deployed bundle" +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/generate/pipeline_and_deploy/test.py b/acceptance/bundle/generate/pipeline_and_deploy/test.py new file mode 100644 index 00000000000..693eaecb2b1 --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/test.py @@ -0,0 +1 @@ +print("Hello!") diff --git a/acceptance/bundle/generate/pipeline_and_deploy/test.toml b/acceptance/bundle/generate/pipeline_and_deploy/test.toml new file mode 100644 index 00000000000..3eba85b404a --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/test.toml @@ -0,0 +1,13 @@ +Local = true +Cloud = true + +Ignore = [ + "databricks.yml", + "resources/*", + "src/*", + ".databricks", +] + +[Env] +# MSYS2 automatically converts absolute paths on Windows; disable for the workspace path. +MSYS_NO_PATHCONV = "1" diff --git a/integration/bundle/generate_pipeline_test.go b/integration/bundle/generate_pipeline_test.go deleted file mode 100644 index 984d555a86d..00000000000 --- a/integration/bundle/generate_pipeline_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package bundle_test - -import ( - "context" - "os" - "path" - "path/filepath" - "strings" - "testing" - - "github.com/databricks/cli/integration/internal/acc" - "github.com/databricks/cli/internal/testcli" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/google/uuid" - "github.com/stretchr/testify/require" -) - -func TestGenerateFromExistingPipelineAndDeploy(t *testing.T) { - ctx, wt := acc.WorkspaceTest(t) - gt := &generatePipelineTest{T: wt, w: wt.W} - - uniqueId := uuid.New().String() - bundleRoot := initTestTemplate(t, ctx, "with_includes", map[string]any{ - "unique_id": uniqueId, - }) - - pipelineId, name := gt.createTestPipeline(ctx) - t.Cleanup(func() { - gt.destroyPipeline(context.WithoutCancel(ctx), pipelineId) - }) - - ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot) - c := testcli.NewRunner(t, ctx, "bundle", "generate", "pipeline", - "--existing-pipeline-id", pipelineId, - "--config-dir", filepath.Join(bundleRoot, "resources"), - "--source-dir", filepath.Join(bundleRoot, "src")) - _, _, err := c.Run() - require.NoError(t, err) - - _, err = os.Stat(filepath.Join(bundleRoot, "src", "notebook.py")) - require.NoError(t, err) - - _, err = os.Stat(filepath.Join(bundleRoot, "src", "test.py")) - require.NoError(t, err) - - matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "generated_pipeline_*.yml")) - require.NoError(t, err) - require.Len(t, matches, 1) - - // check the content of generated yaml - fileName := matches[0] - data, err := os.ReadFile(fileName) - require.NoError(t, err) - generatedYaml := string(data) - - // Replace pipeline name - generatedYaml = strings.ReplaceAll(generatedYaml, name, testutil.RandomName("copy-generated-pipeline-")) - err = os.WriteFile(fileName, []byte(generatedYaml), 0o644) - require.NoError(t, err) - - require.Contains(t, generatedYaml, "libraries:") - require.Contains(t, generatedYaml, "- notebook:") - require.Contains(t, generatedYaml, "path: "+filepath.ToSlash(filepath.Join("..", "src", "notebook.py"))) - require.Contains(t, generatedYaml, "- file:") - require.Contains(t, generatedYaml, "path: "+filepath.ToSlash(filepath.Join("..", "src", "test.py"))) - - deployBundle(t, ctx, bundleRoot) - - destroyBundle(t, ctx, bundleRoot) -} - -type generatePipelineTest struct { - T *acc.WorkspaceT - w *databricks.WorkspaceClient -} - -func (gt *generatePipelineTest) createTestPipeline(ctx context.Context) (string, string) { - t := gt.T - w := gt.w - - tmpdir := acc.TemporaryWorkspaceDir(t, "generate-pipeline-") - f, err := filer.NewWorkspaceFilesClient(w, tmpdir) - require.NoError(t, err) - - err = f.Write(ctx, "notebook.py", strings.NewReader("# Databricks notebook source\nprint('Hello world!'))")) - require.NoError(t, err) - - err = f.Write(ctx, "test.py", strings.NewReader("print('Hello!')")) - require.NoError(t, err) - - nodeTypeId := testutil.GetCloud(t).NodeTypeID() - - name := testutil.RandomName("generated-pipeline-") - resp, err := w.Pipelines.Create(ctx, pipelines.CreatePipeline{ - Name: name, - Libraries: []pipelines.PipelineLibrary{ - { - Notebook: &pipelines.NotebookLibrary{ - Path: path.Join(tmpdir, "notebook"), - }, - }, - { - File: &pipelines.FileLibrary{ - Path: path.Join(tmpdir, "test.py"), - }, - }, - }, - Clusters: []pipelines.PipelineCluster{ - { - CustomTags: map[string]string{ - "Tag1": "Yes", - "Tag2": "24X7", - "Tag3": "APP-1234", - }, - NodeTypeId: nodeTypeId, - NumWorkers: 2, - SparkConf: map[string]string{ - "spark.databricks.enableWsfs": "true", - "spark.databricks.hive.metastore.glueCatalog.enabled": "true", - "spark.databricks.pip.ignoreSSL": "true", - }, - }, - }, - }) - require.NoError(t, err) - - return resp.PipelineId, name -} - -func (gt *generatePipelineTest) destroyPipeline(ctx context.Context, pipelineId string) { - err := gt.w.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ - PipelineId: pipelineId, - }) - require.NoError(gt.T, err) -}