diff --git a/mlops_stacks/.github/workflows/README.md b/mlops_stacks/.github/workflows/README.md index 46c85882..8c991e8a 100644 --- a/mlops_stacks/.github/workflows/README.md +++ b/mlops_stacks/.github/workflows/README.md @@ -1,7 +1,8 @@ # CI/CD Workflow Definitions This directory contains CI/CD workflow definitions using [GitHub Actions](https://docs.github.com/en/actions), -under ``workflows``. These workflows cover testing and deployment of both ML code (for model training, batch inference, etc) and the -Databricks ML resource definitions under ``mlops_stacks/resources``. +under ``workflows``. These workflows cover testing and deployment of both ML code (for model training, batch inference, etc) and +Databricks ML resource definitions. To set up CI/CD for a new project, -please refer to [ML resource config - set up CI CD](../../mlops_stacks/resources/README.md#set-up-ci-and-cd). +please refer to [Setting up CI/CD](<../../README.md#Setting up CI/CD>) and following the [MLOps Setup Guide](../../docs/mlops-setup.md) +to make sure your `WORKFLOW_TOKEN` secret has been properly set up with `Workflow` permissions. diff --git a/mlops_stacks/.github/workflows/deploy-cicd.yml b/mlops_stacks/.github/workflows/deploy-cicd.yml new file mode 100644 index 00000000..3d93cbaf --- /dev/null +++ b/mlops_stacks/.github/workflows/deploy-cicd.yml @@ -0,0 +1,75 @@ +# This GitHub workflow sets up CI/CD workflows for a previously instantiated MLOps Stacks Project. +name: Deploy CICD for Project Workflow + +on: + workflow_dispatch: + inputs: + project_name: + description: 'Project Name' + required: true + default: mlops_stacks + +env: + DATABRICKS_HOST: https://adb-xxxx.xx.azuredatabricks.net + ARM_TENANT_ID: ${{ secrets.STAGING_AZURE_SP_TENANT_ID }} + ARM_CLIENT_ID: ${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }} + ARM_CLIENT_SECRET: ${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }} + + +jobs: + cicd: + runs-on: ubuntu-latest + steps: + - name: Get current timestamp + id: timestamp + run: | + echo "timestamp=$(date +'%s')" >> "$GITHUB_ENV" + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + token: ${{ secrets.WORKFLOW_TOKEN }} + - uses: databricks/setup-cli@v0.236.0 + - name: Convert Project Name to Only Alphanumeric Characters + run: | + PROJECT_NAME_ALPHA=$(echo "${{ github.event.inputs.project_name }}" | tr ' -' '_') + echo "PROJECT_NAME_ALPHA=$PROJECT_NAME_ALPHA" >> "$GITHUB_ENV" + - name: Install jq + run: sudo apt-get install jq + - name: Unzip Bundle and Append Parameters to Input JSON + id: unzip + run: | + tar -xzvf cicd.tar.gz + INPUT_CLOUD_1=$(jq -r '.input_cloud' cicd_params.json) + INPUT_CLOUD_2=$(jq -r '.input_cloud' "$PROJECT_NAME_ALPHA/project_params.json") + if [ "$INPUT_CLOUD_1" != "$INPUT_CLOUD_2" ]; then + printf "Error: CICD cloud '%s' does not match project cloud '%s'\n" "$INPUT_CLOUD_1" "$INPUT_CLOUD_2" + exit 1 + fi + printf '%s' "$(jq '. += {"input_project_name":"${{ github.event.inputs.project_name }}"}' cicd_params.json)" > cicd_params.json + printf '%s' "$(jq -s '.[0] + .[1]' cicd_params.json "$PROJECT_NAME_ALPHA/project_params.json")" > cicd_params.json + - name: Update databricks.yml + id: update + run: | + echo -e " staging:\n variables:\n catalog_name: staging\n workspace:\n host: https://adb-xxxx.xx.azuredatabricks.net\n\n prod:\n variables:\n catalog_name: prod\n workspace:\n host: https://adb-xxxx.xx.azuredatabricks.net\n\n test:\n variables:\n catalog_name: test\n workspace:\n host: https://adb-xxxx.xx.azuredatabricks.net" >> "$(PROJECT_NAME_ALPHA)\databricks.yml" + - name: Initialize Bundle + id: initialize + run: | + databricks bundle init ./cicd --config-file "cicd_params.json" + - name: Commit Changes + id: commit + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global user.name "Deploy CICD Bot" + git config --global user.email "noreply-cicd-bot@databricks.com" + git checkout -b add-cicd-for-${{ github.event.inputs.project_name }}-${{ env.timestamp }} + git add .github "$PROJECT_NAME_ALPHA/databricks.yml" + git commit -m "Add CICD for ${{ github.event.inputs.project_name }}" + git push origin add-cicd-for-${{ github.event.inputs.project_name }}-${{ env.timestamp }} + + - name: Create Pull Request + id: pr + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + gh pr create --base main --head add-cicd-for-${{ github.event.inputs.project_name }}-${{ env.timestamp }} --title "Deploy CICD for ${{ github.event.inputs.project_name }}" --body "This PR was generated by the Deploy CICD workflow." --reviewer ${{ github.actor }} diff --git a/mlops_stacks/.github/workflows/mlops_stacks-lint-cicd-workflow-files.yml b/mlops_stacks/.github/workflows/lint-cicd-workflow-files.yml similarity index 93% rename from mlops_stacks/.github/workflows/mlops_stacks-lint-cicd-workflow-files.yml rename to mlops_stacks/.github/workflows/lint-cicd-workflow-files.yml index 59a5c5be..3db8be19 100644 --- a/mlops_stacks/.github/workflows/mlops_stacks-lint-cicd-workflow-files.yml +++ b/mlops_stacks/.github/workflows/lint-cicd-workflow-files.yml @@ -9,7 +9,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download actionlint id: get_actionlint run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) diff --git a/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-prod.yml b/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-prod.yml index 87b216b3..1c1c26af 100644 --- a/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-prod.yml +++ b/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-prod.yml @@ -18,14 +18,15 @@ env: ARM_TENANT_ID: ${{ secrets.PROD_AZURE_SP_TENANT_ID }} ARM_CLIENT_ID: ${{ secrets.PROD_AZURE_SP_APPLICATION_ID }} ARM_CLIENT_SECRET: ${{ secrets.PROD_AZURE_SP_CLIENT_SECRET }} + jobs: prod: concurrency: mlops_stacks-prod-bundle-job - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: databricks/setup-cli@main + - uses: actions/checkout@v4 + - uses: databricks/setup-cli@v0.236.0 - name: Validate Bundle For Prod id: validate run: | diff --git a/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-staging.yml b/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-staging.yml index ca6603d2..526a432f 100644 --- a/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-staging.yml +++ b/mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-staging.yml @@ -18,14 +18,15 @@ env: ARM_TENANT_ID: ${{ secrets.STAGING_AZURE_SP_TENANT_ID }} ARM_CLIENT_ID: ${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }} ARM_CLIENT_SECRET: ${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }} + jobs: staging: concurrency: mlops_stacks-staging-bundle-job - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: databricks/setup-cli@main + - uses: actions/checkout@v4 + - uses: databricks/setup-cli@v0.236.0 - name: Validate Bundle For Staging id: validate run: | diff --git a/mlops_stacks/.github/workflows/mlops_stacks-bundle-ci.yml b/mlops_stacks/.github/workflows/mlops_stacks-bundle-ci.yml index b79003a6..cccfdf12 100644 --- a/mlops_stacks/.github/workflows/mlops_stacks-bundle-ci.yml +++ b/mlops_stacks/.github/workflows/mlops_stacks-bundle-ci.yml @@ -5,7 +5,9 @@ name: Bundle validation for mlops_stacks on: workflow_dispatch: - pull_request_target: + pull_request: + paths: + - 'mlops_stacks/**' defaults: run: @@ -18,26 +20,28 @@ env: PROD_ARM_TENANT_ID: ${{ secrets.PROD_AZURE_SP_TENANT_ID }} PROD_ARM_CLIENT_ID: ${{ secrets.PROD_AZURE_SP_APPLICATION_ID }} PROD_ARM_CLIENT_SECRET: ${{ secrets.PROD_AZURE_SP_CLIENT_SECRET }} + jobs: staging: concurrency: mlops_stacks-staging-bundle-job - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - - uses: databricks/setup-cli@main + - uses: databricks/setup-cli@v0.236.0 - name: Validate Bundle For Staging id: validate env: ARM_TENANT_ID: ${{ env.STAGING_ARM_TENANT_ID }} ARM_CLIENT_ID: ${{ env.STAGING_ARM_CLIENT_ID }} ARM_CLIENT_SECRET: ${{ env.STAGING_ARM_CLIENT_SECRET }} + run: | databricks bundle validate -t staging > ../validate_output.txt - name: Create Comment with Bundle Configuration - uses: actions/github-script@v6 + uses: actions/github-script@v7 id: comment with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -62,22 +66,23 @@ jobs: prod: concurrency: mlops_stacks-prod-bundle-job - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - - uses: databricks/setup-cli@main + - uses: databricks/setup-cli@v0.236.0 - name: Validate Bundle For Prod id: validate env: ARM_TENANT_ID: ${{ env.PROD_ARM_TENANT_ID }} ARM_CLIENT_ID: ${{ env.PROD_ARM_CLIENT_ID }} ARM_CLIENT_SECRET: ${{ env.PROD_ARM_CLIENT_SECRET }} + run: | databricks bundle validate -t prod > ../validate_output.txt - name: Create Comment with Bundle Configuration - uses: actions/github-script@v6 + uses: actions/github-script@v7 id: comment with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/mlops_stacks/.github/workflows/mlops_stacks-run-tests.yml b/mlops_stacks/.github/workflows/mlops_stacks-run-tests.yml index 99f2d7c4..85e42758 100644 --- a/mlops_stacks/.github/workflows/mlops_stacks-run-tests.yml +++ b/mlops_stacks/.github/workflows/mlops_stacks-run-tests.yml @@ -1,7 +1,10 @@ -name: ML Code Tests for mlops_stacks +name: Training Unit and Integration Tests for mlops_stacks on: workflow_dispatch: pull_request: + paths: + - 'mlops_stacks/**' + - '.github/workflows/mlops_stacks-run-tests.yml' defaults: run: @@ -11,17 +14,18 @@ env: ARM_TENANT_ID: ${{ secrets.STAGING_AZURE_SP_TENANT_ID }} ARM_CLIENT_ID: ${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }} ARM_CLIENT_SECRET: ${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }} + -concurrency: mlops_stacks-feature-training-integration-test-staging +concurrency: mlops_stacks-training-integration-test-staging jobs: unit_tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: '3.10' - name: Install dependencies run: | python -m pip install --upgrade pip @@ -33,11 +37,11 @@ jobs: integration_test: needs: unit_tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 - - uses: databricks/setup-cli@main + uses: actions/checkout@v4 + - uses: databricks/setup-cli@v0.236.0 - name: Validate Bundle For Test Deployment Target in Staging Workspace id: validate run: | diff --git a/mlops_stacks/README.md b/mlops_stacks/README.md index 71e105c6..3c0ed910 100644 --- a/mlops_stacks/README.md +++ b/mlops_stacks/README.md @@ -3,40 +3,108 @@ This directory contains an ML project based on the default [Databricks MLOps Stacks](https://github.com/databricks/mlops-stacks), defining a production-grade ML pipeline for automated retraining and batch inference of an ML model on tabular data. +The "Getting Started" docs can be found at https://learn.microsoft.com/azure/databricks/dev-tools/bundles/mlops-stacks. -See the [Project overview](docs/project-overview.md) for details on the ML pipeline and code structure -in this repo. +See the full pipeline structure below. The [MLOps Stacks README](https://github.com/databricks/mlops-stacks/blob/main/Pipeline.md) +contains additional details on how ML pipelines are tested and deployed across each of the dev, staging, prod environments below. + +![MLOps Stacks diagram](docs/images/mlops-stack-summary.png) + + +## Code structure +This project contains the following components: + +| Component | Description | +|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ML Code | Example ML project code, with unit tested Python modules and notebooks | +| ML Resources as Code | ML pipeline resources (training and batch inference jobs with schedules, etc) configured and deployed through [databricks CLI bundles](https://learn.microsoft.com/azure/databricks/dev-tools/cli/bundle-cli) | +| CI/CD | [GitHub Actions](https://github.com/actions) workflows to test and deploy ML code and resources + | + +contained in the following files: + +``` +mlops_stacks <- Root directory. Both monorepo and polyrepo are supported. +│ +├── mlops_stacks <- Contains python code, notebooks and ML resources related to one ML project. +│ │ +│ ├── requirements.txt <- Specifies Python dependencies for ML code (for example: model training, batch inference). +│ │ +│ ├── databricks.yml <- databricks.yml is the root bundle file for the ML project that can be loaded by databricks CLI bundles. It defines the bundle name, workspace URL and resource config component to be included. +│ │ +│ ├── training <- Training folder contains Notebook that trains and registers the model. +│ │ +│ ├── validation <- Optional model validation step before deploying a model. +│ │ +│ ├── monitoring <- Model monitoring, feature monitoring, etc. +│ │ +│ ├── deployment <- Deployment and Batch inference workflows +│ │ │ +│ │ ├── batch_inference <- Batch inference code that will run as part of scheduled workflow. +│ │ │ +│ │ ├── model_deployment <- As part of CD workflow, deploy the registered model by assigning it the appropriate alias. +│ │ +│ │ +│ ├── tests <- Unit tests for the ML project, including the modules under `features`. +│ │ +│ ├── resources <- ML resource (ML jobs, MLflow models) config definitions expressed as code, across dev/staging/prod/test. +│ │ +│ ├── model-workflow-resource.yml <- ML resource config definition for model training, validation, deployment workflow +│ │ +│ ├── batch-inference-workflow-resource.yml <- ML resource config definition for batch inference workflow +│ │ +│ ├── ml-artifacts-resource.yml <- ML resource config definition for model and experiment +│ │ +│ ├── monitoring-resource.yml <- ML resource config definition for quality monitoring workflow +│ +├── .github <- Configuration folder for CI/CD using GitHub Actions. The CI/CD workflows deploy ML resources defined in the `./resources/*` folder with databricks CLI bundles. +│ +├── docs <- Contains documentation for the repo. +│ +├── cicd.tar.gz <- Contains CI/CD bundle that should be deployed by deploy-cicd.yml to set up CI/CD for projects. +``` ## Using this repo The table below links to detailed docs explaining how to use this repo for different use cases. -If you're a data scientist just getting started with this repo for a brand new ML project, we recommend starting with -the [Project overview](docs/project-overview.md) and -[ML quickstart](docs/ml-developer-guide.md). -When you're satisfied with initial ML experimentation (e.g. validated that a model with reasonable performance can be -trained on your dataset) and ready to deploy production training/inference +This project comes with example ML code to train, validate and deploy a regression model to predict NYC taxi fares. +If you're a data scientist just getting started with this repo for a brand new ML project, we recommend +adapting the provided example code to your ML problem. Then making and +testing ML code changes on Databricks or your local machine. Follow the instructions from +the [project README](./mlops_stacks/README.md). + + +When you're ready to deploy production training/inference pipelines, ask your ops team to follow the [MLOps setup guide](docs/mlops-setup.md) to configure CI/CD and deploy production ML pipelines. After that, follow the [ML pull request guide](docs/ml-pull-request.md) -and [ML resource config guide](mlops_stacks/resources/README.md) to propose, test, and deploy changes to production ML code (e.g. update model parameters) + and [ML resource config guide](mlops_stacks/resources/README.md) to propose, test, and deploy changes to production ML code (e.g. update model parameters) or pipeline resources (e.g. use a larger instance type for model training) via pull request. | Role | Goal | Docs | |-------------------------------|------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| First-time users of this repo | Understand the ML pipeline and code structure in this repo | [Project overview](docs/project-overview.md) | -| Data Scientist | Get started writing ML code for a brand new project | [ML quickstart](docs/ml-developer-guide.md) | -| Data Scientist | Update production ML code (e.g. model training logic) for an existing project | [ML pull request guide](docs/ml-pull-request.md) | -| Data Scientist | Modify production model ML resources, e.g. model training or inference jobs | [ML resource config guide](mlops_stacks/resources/README.md) | +| Data Scientist | Get started writing ML code for a brand new project | [project README](./mlops_stacks/README.md) | | MLOps / DevOps | Set up CI/CD for the current ML project | [MLOps setup guide](docs/mlops-setup.md) | +| Data Scientist | Update production ML code (e.g. model training logic) for an existing project | [ML pull request guide](docs/ml-pull-request.md) | +| Data Scientist | Modify production model ML resources, e.g. model training or inference jobs | [ML resource config guide](mlops_stacks/resources/README.md) | + +## Setting up CI/CD +This stack comes with a workflow to set up CI/CD for projects that can be found in + +`.github/workflows/deploy-cicd.yml`. + + +To set up CI/CD for projects that were created through MLOps Stacks with the `Project_Only` parameter, +run the above mentioned workflow, specifying the `project_name` as a parameter. For example, for the monorepo case: -## Monorepo +1. Setup your repository by initializing MLOps Stacks via Databricks CLI with the `CICD_and_Project` or `CICD_Only` parameter. +2. Follow the [MLOps Setup Guide](./docs/mlops-setup.md) to setup authentication and get the repo ready for CI/CD. +3. Create a new project by initializing MLOps Stacks again but this time with the `Project_Only` parameter. +4. Run the `deploy-cicd.yml` workflow with the `project_name` parameter set to the name of the project you want to set up CI/CD for. -It's possible to use the repo as a monorepo that contains multiple projects. All projects share the same workspaces and service principals. -For example, assuming there's existing repo with root directory name `monorepo_root_dir` and project name `project1` -1. Create another project from `databricks bundle init` with project name `project2` and root directory name `project2`. -2. Copy the internal directory `project2/project2` to root directory of existing repo `monorepo_root_dir/project2`. -3. Copy yaml files from `project2/.github/workflows/` to `monorepo_root_dir/.github/workflows/` and make sure there's no name conflicts. +NOTE: This project has already been initialized with an instantiation of the above workflow, so there's no +need to run it again for project `mlops_stacks`. diff --git a/mlops_stacks/docs/images/MLFlowRunLink.png b/mlops_stacks/docs/images/MLFlowRunLink.png deleted file mode 100644 index 077be8b0..00000000 Binary files a/mlops_stacks/docs/images/MLFlowRunLink.png and /dev/null differ diff --git a/mlops_stacks/docs/images/MLResourceConfigInterface.png b/mlops_stacks/docs/images/MLResourceConfigInterface.png deleted file mode 100644 index 3badf701..00000000 Binary files a/mlops_stacks/docs/images/MLResourceConfigInterface.png and /dev/null differ diff --git a/mlops_stacks/docs/images/mlops-resource-config.png b/mlops_stacks/docs/images/mlops-resource-config.png deleted file mode 100644 index 3bbd0871..00000000 Binary files a/mlops_stacks/docs/images/mlops-resource-config.png and /dev/null differ diff --git a/mlops_stacks/docs/images/mlops-stack-deploy.png b/mlops_stacks/docs/images/mlops-stack-deploy.png new file mode 100644 index 00000000..77b67188 Binary files /dev/null and b/mlops_stacks/docs/images/mlops-stack-deploy.png differ diff --git a/mlops_stacks/docs/images/mlops-stack-summary.png b/mlops_stacks/docs/images/mlops-stack-summary.png index 3fc7f6ee..f44bbd77 100644 Binary files a/mlops_stacks/docs/images/mlops-stack-summary.png and b/mlops_stacks/docs/images/mlops-stack-summary.png differ diff --git a/mlops_stacks/docs/ml-developer-guide.md b/mlops_stacks/docs/ml-developer-guide.md deleted file mode 100644 index fe6afbbb..00000000 --- a/mlops_stacks/docs/ml-developer-guide.md +++ /dev/null @@ -1,64 +0,0 @@ -# ML Developer Guide - -[(back to main README)](../README.md) - -## Table of contents -* [Initial setup](#initial-setup): adapting the provided example code to your ML problem -* [Iterating on ML code](#iterating-on-ml-code): making and testing ML code changes on Databricks or your local machine. -* [Next steps](#next-steps) - -## Initial setup -This project comes with example ML code to train, validate and deploy a regression model to predict NYC taxi fares. - -Subsequent sections explain how to adapt the example code to your ML problem and quickly get -started iterating on model training code. - -## Iterating on ML code - -### Deploy ML code and resources to dev workspace using bundles - -Refer to [Local development and dev workspace](../mlops_stacks/resources/README.md#local-development-and-dev-workspace) -to use databricks CLI bundles to deploy ML code together with resource configs to the dev workspace. - -This will allow you to develop locally and use databricks CLI bundles to deploy to your dev workspace to test out code and config changes. - -### Develop on Databricks using Databricks Repos - -#### Prerequisites -You'll need: -* Access to run commands on a cluster running Databricks Runtime ML version 11.0 or above in your dev Databricks workspace -* To set up [Databricks Repos](https://learn.microsoft.com/azure/databricks/repos/index): see instructions below - -#### Configuring Databricks Repos -To use Repos, [set up git integration](https://learn.microsoft.com/azure/databricks/repos/repos-setup) in your dev workspace. - -If the current project has already been pushed to a hosted Git repo, follow the -[UI workflow](https://learn.microsoft.com/azure/databricks/repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later) -to clone it into your dev workspace and iterate. - -Otherwise, e.g. if iterating on ML code for a new project, follow the steps below: -* Follow the [UI workflow](https://learn.microsoft.com/azure/databricks/repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later) - for creating a repo, but uncheck the "Create repo by cloning a Git repository" checkbox. -* Install the `dbx` CLI via `pip install --upgrade dbx` -* Run `databricks configure --profile mlops_stacks-dev --token --host `, passing the URL of your dev workspace. - This should prompt you to enter an API token -* [Create a personal access token](https://learn.microsoft.com/azure/databricks/dev-tools/auth#personal-access-tokens-for-users) - in your dev workspace and paste it into the prompt from the previous step -* From within the root directory of the current project, use the [dbx sync](https://dbx.readthedocs.io/en/latest/guides/python/devloop/mixed/#using-dbx-sync-repo-for-local-to-repo-synchronization) tool to copy code files from your local machine into the Repo by running - `dbx sync repo --profile mlops_stacks-dev --source . --dest-repo your-repo-name`, where `your-repo-name` should be the last segment of the full repo name (`/Repos/username/your-repo-name`) - -#### Running code on Databricks -You can iterate on the sample ML code by running the provided `mlops_stacks/training/notebooks/Train.py` notebook on Databricks using -[Repos](https://learn.microsoft.com/azure/databricks/repos/index). - - -## Next Steps -If you're iterating on ML code for an existing, already-deployed ML project, follow [Submitting a Pull Request](ml-pull-request.md) -to submit your code for testing and production deployment. - -Otherwise, if exploring a new ML problem and satisfied with the results (e.g. you were able to train -a model with reasonable performance on your dataset), you may be ready to productionize your pipeline. -To do this, follow the [MLOps Setup Guide](mlops-setup.md) to set up CI/CD and deploy -production training/inference pipelines. - -[(back to main README)](../README.md) diff --git a/mlops_stacks/docs/ml-pull-request.md b/mlops_stacks/docs/ml-pull-request.md index 335fb844..b55bf190 100644 --- a/mlops_stacks/docs/ml-pull-request.md +++ b/mlops_stacks/docs/ml-pull-request.md @@ -6,19 +6,11 @@ ML resources, per the [MLOps setup guide](mlops-setup.md). ## Table of contents -* [Intro](#intro) * [Opening a pull request](#opening-a-pull-request) * [Viewing test status and debug logs](#viewing-test-status-and-debug-logs) * [Merging your pull request](#merging-your-pull-request) * [Next steps](#next-steps) -## Intro -After following the -[ML quickstart](ml-developer-guide.md). -to iterate on ML code, the next step is to get -your updated code merged back into the repo for production use. This page walks you through the workflow -for doing so via a pull request. - ## Opening a pull request To push your updated ML code to production, [open a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request @@ -31,19 +23,16 @@ is planned for the future. ## Viewing test status and debug logs Opening a pull request will trigger a[workflow](../.github/workflows/mlops_stacks-run-tests.yml) -that runs unit and integration tests for the model training pipeline on Databricks against a test dataset. +that runs unit and integration tests for the model training (and feature engineering if added) pipeline on Databricks against a test dataset. You can view test status and debug logs from the pull request UI, and push new commits to your pull request branch to address any test failures. The integration test runs the model training notebook in the staging workspace, training, validating, -and registering a new model version in the model registry. The fitted model along with its metrics and params +and registering a new model version in UC. +The fitted model along with its metrics and params will also be logged to an MLflow run. To debug failed integration test runs, click into the Databricks job run -URL printed in the test logs. The job run page will contain a link to the MLflow model training run: - -![Link to MLFlow Run](images/MLFlowRunLink.png) - -Click the MLflow run link to view training metrics or fetch and debug the model as needed. - +URL printed in the test logs. The executed notebook of the job run will contain a link to the MLflow model training run. You can also use the Experiments page in the workspace +to view training metrics or fetch and debug the model as needed. ## Merging your pull request @@ -54,9 +43,13 @@ and then merge it into the upstream repo. After merging your pull request, subsequent runs of the model training and batch inference jobs in staging and production will automatically use your updated ML code. -You can track the state of the ML pipelines for the current project from the MLflow registered model UI. Links: -* [Staging workspace registered model](https://adb-xxxx.xx.azuredatabricks.net#mlflow/models/staging-mlops_stacks-model) -* [Prod workspace registered model](https://adb-xxxx.xx.azuredatabricks.net#mlflow/models/prod-mlops_stacks-model) +You can track the state of the ML pipelines for the current project from the MLflow registered model UI. + +Links: +* [Staging model in UC](https://adb-xxxx.xx.azuredatabricks.net/explore/data/models/staging/mlops_stacks/mlops_stacks-model) +* [Prod model in UC](https://adb-xxxx.xx.azuredatabricks.net/explore/data/models/prod/mlops_stacks/mlops_stacks-model) + + In both the staging and prod workspaces, the MLflow registered model contains links to: * The model versions produced through automated retraining diff --git a/mlops_stacks/docs/mlops-setup.md b/mlops_stacks/docs/mlops-setup.md index 4e30748e..10f42a50 100644 --- a/mlops_stacks/docs/mlops-setup.md +++ b/mlops_stacks/docs/mlops-setup.md @@ -7,6 +7,7 @@ * [Configure CI/CD](#configure-cicd---github-actions) * [Merge PR with initial ML code](#merge-a-pr-with-your-initial-ml-code) * [Create release branch](#create-release-branch) + * [Deploy ML resources and enable production jobs](#deploy-ml-resources-and-enable-production-jobs) * [Next steps](#next-steps) @@ -14,8 +15,7 @@ This page explains how to productionize the current project, setting up CI/CD and ML resource deployment, and deploying ML training and inference jobs. -After following this guide, data scientists can follow the [ML Pull Request](ml-pull-request.md) and -[ML Config](../mlops_stacks/resources/README.md) guides to make changes to ML code or deployed jobs. +After following this guide, data scientists can follow the [ML Pull Request](ml-pull-request.md) guide to make changes to ML code or deployed jobs. ## Create a hosted Git repo Create a hosted Git repo to store project code, if you haven't already done so. From within the project @@ -30,8 +30,10 @@ git remote add upstream Commit the current `README.md` file and other docs to the `main` branch of the repo, to enable forking the repo: ``` + git add README.md docs .gitignore mlops_stacks/resources/README.md git commit -m "Adding project README" + git push upstream main ``` @@ -39,10 +41,11 @@ git push upstream main ### Prerequisites * You must be an account admin to add service principals to the account. -* You must be a Databricks workspace admin in the staging and prod workspaces. Verify that you're an admin by viewing the +* You must be a Databricks workspace admin in the staging and prod workspaces. + Verify that you're an admin by viewing the [staging workspace admin console](https://adb-xxxx.xx.azuredatabricks.net#setting/accounts) and - [prod workspace admin console](https://adb-xxxx.xx.azuredatabricks.net#setting/accounts). If - the admin console UI loads instead of the Databricks workspace homepage, you are an admin. + [prod workspace admin console](https://adb-xxxx.xx.azuredatabricks.net#setting/accounts). + If the admin console UI loads instead of the Databricks workspace homepage, you are an admin. ### Set up authentication for CI/CD #### Set up Service Principal @@ -58,10 +61,31 @@ For your convenience, we also have Terraform modules that can be used to [create -#### Set secrets for CI/CD +#### Configure Service Principal (SP) permissions +If the created project uses **Unity Catalog**, we expect a catalog to exist with the name of the deployment target by default. +For example, if the deployment target is dev, we expect a catalog named dev to exist in the workspace. +If you want to use different catalog names, please update the target names declared in the[mlops_stacks/databricks.yml](../mlops_stacks/databricks.yml) file. +If changing the staging, prod, or test deployment targets, you'll also need to update the workflows located in the .github/workflows directory. + +The SP must have proper permission in each respective environment and the catalog for the environments. + +For the integration test and the ML training job, the SP must have permissions to read the input Delta table and create experiment and models. +i.e. for each environment: +- USE_CATALOG +- USE_SCHEMA +- MODIFY +- CREATE_MODEL +- CREATE_TABLE +For the batch inference job, the SP must have permissions to read input Delta table and modify the output Delta table. +i.e. for each environment +- USAGE permissions for the catalog and schema of the input and output table. +- SELECT permission for the input table. +- MODIFY permission for the output table if it pre-dates your job. +#### Set secrets for CI/CD + After creating the service principals and adding them to the respective staging and prod workspaces, refer to [Manage access tokens for a service principal](https://learn.microsoft.com/azure/databricks/administration-guide/users-groups/service-principals#--manage-access-tokens-for-a-service-principal) and [Get Azure AD tokens for service principals](https://learn.microsoft.com/azure/databricks/dev-tools/api/latest/aad/service-prin-aad-token) @@ -73,10 +97,18 @@ to add the following secrets to GitHub: - `STAGING_AZURE_SP_TENANT_ID` - `STAGING_AZURE_SP_APPLICATION_ID` - `STAGING_AZURE_SP_CLIENT_SECRET` +- `WORKFLOW_TOKEN` : [Github token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic) with workflow permissions. This secret is needed for the Deploy CI/CD Workflow. Be sure to update the [Workflow Permissions](https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token) section under Repo Settings > Actions > General to allow `Read and write permissions`. - +### Setting up CI/CD workflows +After setting up authentication for CI/CD, you can now set up CI/CD workflows. We provide a [Deploy CICD workflow](../.github/workflows/deploy-cicd.yml) that can be used to generate the other CICD workflows mentioned below for projects. +This workflow is manually triggered with `project_name` as parameter. This workflow will need to be triggered for each project to set up its set of CI/CD workflows that can be used to deploy ML resources and run ML jobs in the staging and prod workspaces. +These workflows will be defined under `.github/workflows`. + +If you want to deploy CI/CD for an initialized project (`Project-Only` MLOps Stacks initialization), you can manually run the `deploy-cicd.yml` workflow from the [Github Actions UI](https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow?tool=webui) once the project code has been added to your main repo. +The workflow will create a pull request with all the changes against your main branch. Review and approve it to commit the files to deploy CI/CD for the project. + ## Merge a PR with your initial ML code Create and push a PR branch adding the ML code to the repository. @@ -109,6 +141,7 @@ git checkout main Your production jobs (model training, batch inference) will pull ML code against this branch, while your staging jobs will pull ML code against the `main` branch. Note that the `main` branch will be the source of truth for ML resource configs and CI/CD workflows. For future ML code changes, iterate against the `main` branch and regularly deploy your ML code from staging to production by merging code changes from the `main` branch into the `release` branch. + ## Deploy ML resources and enable production jobs Follow the instructions in [mlops_stacks/resources/README.md](../mlops_stacks/resources/README.md) to deploy ML resources and production jobs. @@ -116,5 +149,6 @@ and production jobs. ## Next steps After you configure CI/CD and deploy training & inference pipelines, notify data scientists working on the current project. They should now be able to follow the -[ML pull request guide](ml-pull-request.md) and [ML resource config guide](../mlops_stacks/resources/README.md) to propose, test, and deploy +[ML pull request guide](ml-pull-request.md) and +[ML resource config guide](../mlops_stacks/resources/README.md) to propose, test, and deploy ML code and pipeline changes to production. diff --git a/mlops_stacks/docs/project-overview.md b/mlops_stacks/docs/project-overview.md deleted file mode 100644 index d3bb0d73..00000000 --- a/mlops_stacks/docs/project-overview.md +++ /dev/null @@ -1,65 +0,0 @@ -# Project Overview - -[(back to main README)](../README.md) - -## ML pipeline structure -This project defines an ML pipeline for automated retraining and batch inference of an ML model -on tabular data. - -See the full pipeline structure below. The [MLOps Stacks README](https://github.com/databricks/mlops-stacks/blob/main/Pipeline.md) -contains additional details on how ML pipelines are tested and deployed across each of the dev, staging, prod environments below. - -![MLOps Stacks diagram](images/mlops-stack-summary.png) - - -## Code structure -This project contains the following components: - -| Component | Description | -|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ML Code | Example ML project code, with unit tested Python modules and notebooks | -| ML Resource Config as Code | ML pipeline resource config (training and batch inference job schedules, etc) configured and deployed through [databricks CLI bundles](https://learn.microsoft.com/azure/databricks/dev-tools/cli/bundle-cli) | -| CI/CD | [GitHub Actions](https://github.com/actions) workflows to test and deploy ML code and resources | - -contained in the following files: - -``` -mlops_stacks <- Root directory. Both monorepo and polyrepo are supported. -│ -├── mlops_stacks <- Contains python code, notebooks and ML resources related to one ML project. -│ │ -│ ├── requirements.txt <- Specifies Python dependencies for ML code (for example: model training, batch inference). -│ │ -│ ├── databricks.yml <- databricks.yml is the root ML resource config file for the ML project that can be loaded by databricks CLI bundles. It defines the bundle name, workspace URL and resource config component to be included. -│ │ -│ ├── training <- Training folder contains Notebook that trains and registers the model. -│ │ -│ ├── validation <- Optional model validation step before deploying a model. -│ │ -│ ├── monitoring <- Model monitoring, feature monitoring, etc. -│ │ -│ ├── deployment <- Deployment and Batch inference workflows -│ │ │ -│ │ ├── batch_inference <- Batch inference code that will run as part of scheduled workflow. -│ │ │ -│ │ ├── model_deployment <- As part of CD workflow, promote model to Production stage in model registry. -│ │ -│ │ -│ ├── tests <- Unit tests for the ML project, including the modules under `features`. -│ │ -│ ├── resources <- ML resource (ML jobs, MLflow models) config definitions expressed as code, across dev/staging/prod/test. -│ │ -│ ├── model-workflow-resource.yml <- ML resource config definition for model training, validation, deployment workflow -│ │ -│ ├── batch-inference-workflow-resource.yml <- ML resource config definition for batch inference workflow -│ │ -│ ├── ml-artifacts-resource.yml <- ML resource config definition for model and experiment -│ │ -│ ├── monitoring-workflow-resource.yml <- ML resource config definition for data monitoring workflow -│ -├── .github <- Configuration folder for CI/CD using GitHub Actions. The CI/CD workflows run the notebooks - under `notebooks` to test and deploy model training code -``` - -## Next Steps -See the [main README](../README.md#using-this-repo) for additional links on how to work with this repo. diff --git a/mlops_stacks/mlops_stacks/README.md b/mlops_stacks/mlops_stacks/README.md index cd28b5f7..ff7f5bca 100644 --- a/mlops_stacks/mlops_stacks/README.md +++ b/mlops_stacks/mlops_stacks/README.md @@ -1,5 +1,112 @@ # mlops_stacks +This project comes with example ML code to train, validate and deploy a regression model to predict NYC taxi fares. +If you're a data scientist just getting started with this repo for a brand new ML project, we recommend +adapting the provided example code to your ML problem. Then making and +testing ML code changes on Databricks or your local machine. -This directory contains python code, notebooks and ML resource configs related to one ML project. +The "Getting Started" docs can be found at https://learn.microsoft.com/azure/databricks/dev-tools/bundles/mlops-stacks. -See the [Project overview](../docs/project-overview.md) for details on code structure of project directory. \ No newline at end of file +## Table of contents +* [Code structure](#code-structure): structure of this project. + +* [Iterating on ML code](#iterating-on-ml-code): making and testing ML code changes on Databricks or your local machine. +* [Next steps](#next-steps) + +This directory contains an ML project based on the default +[Databricks MLOps Stacks](https://github.com/databricks/mlops-stacks), +defining a production-grade ML pipeline for automated retraining and batch inference of an ML model on tabular data. + +## Code structure +This project contains the following components: + +| Component | Description | +|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ML Code | Example ML project code, with unit tested Python modules and notebooks | +| ML Resources as Code | ML pipeline resources (training and batch inference jobs with schedules, etc) configured and deployed through [databricks CLI bundles](https://learn.microsoft.com/azure/databricks/dev-tools/cli/bundle-cli) | + +contained in the following files: + +``` +mlops_stacks <- Root directory. Both monorepo and polyrepo are supported. +│ +├── mlops_stacks <- Contains python code, notebooks and ML resources related to one ML project. +│ │ +│ ├── requirements.txt <- Specifies Python dependencies for ML code (for example: model training, batch inference). +│ │ +│ ├── databricks.yml <- databricks.yml is the root bundle file for the ML project that can be loaded by databricks CLI bundles. It defines the bundle name, workspace URL and resource config component to be included. +│ │ +│ ├── training <- Training folder contains Notebook that trains and registers the model. +│ │ +│ ├── validation <- Optional model validation step before deploying a model. +│ │ +│ ├── monitoring <- Model monitoring, feature monitoring, etc. +│ │ +│ ├── deployment <- Deployment and Batch inference workflows +│ │ │ +│ │ ├── batch_inference <- Batch inference code that will run as part of scheduled workflow. +│ │ │ +│ │ ├── model_deployment <- As part of CD workflow, deploy the registered model by assigning it the appropriate alias. +│ │ +│ │ +│ ├── tests <- Unit tests for the ML project, including the modules under `features`. +│ │ +│ ├── resources <- ML resource (ML jobs, MLflow models) config definitions expressed as code, across dev/staging/prod/test. +│ │ +│ ├── model-workflow-resource.yml <- ML resource config definition for model training, validation, deployment workflow +│ │ +│ ├── batch-inference-workflow-resource.yml <- ML resource config definition for batch inference workflow +│ │ +│ ├── ml-artifacts-resource.yml <- ML resource config definition for model and experiment +│ │ +│ ├── monitoring-resource.yml <- ML resource config definition for quality monitoring workflow +``` + + +## Iterating on ML code + +### Deploy ML code and resources to dev workspace using Bundles + +Refer to [Local development and dev workspace](./resources/README.md#local-development-and-dev-workspace) +to use databricks CLI bundles to deploy ML code together with ML resource configs to dev workspace. + +This will allow you to develop locally and use databricks CLI bundles to deploy to your dev workspace to test out code and config changes. + +### Develop on Databricks using Databricks Git folders + +#### Prerequisites +You'll need: +* Access to run commands on a cluster running Databricks Runtime ML version 11.0 or above in your dev Databricks workspace +* To set up [Databricks Git folders](https://learn.microsoft.com/azure/databricks/repos/index): see instructions below + +#### Configuring Databricks Git folders +To use Git folders, [set up git integration](https://learn.microsoft.com/azure/databricks/repos/repos-setup) in your dev workspace. + +If the current project has already been pushed to a hosted Git repo, follow the +[UI workflow](https://learn.microsoft.com/azure/databricks/repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later) +to clone it into your dev workspace and iterate. + +Otherwise, e.g. if iterating on ML code for a new project, follow the steps below: +* Follow the [UI workflow](https://learn.microsoft.com/azure/databricks/repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later) + for creating a repo, but uncheck the "Create repo by cloning a Git repository" checkbox. +* Install the `dbx` CLI via `pip install --upgrade dbx` +* Run `databricks configure --profile mlops_stacks-dev --token --host `, passing the URL of your dev workspace. + This should prompt you to enter an API token +* [Create a personal access token](https://learn.microsoft.com/azure/databricks/dev-tools/auth/pat) + in your dev workspace and paste it into the prompt from the previous step +* From within the root directory of the current project, use the [dbx sync](https://dbx.readthedocs.io/en/latest/guides/python/devloop/mixed/#using-dbx-sync-repo-for-local-to-repo-synchronization) tool to copy code files from your local machine into the Repo by running + `dbx sync repo --profile mlops_stacks-dev --source . --dest-repo your-repo-name`, where `your-repo-name` should be the last segment of the full repo name (`/Repos/username/your-repo-name`) + + + +## Next Steps + +When you're satisfied with initial ML experimentation (e.g. validated that a model with reasonable performance can be trained on your dataset) and ready to deploy production training/inference pipelines, ask your ops team to set up CI/CD for the current ML project if they haven't already. CI/CD can be set up as part of the + +MLOps Stacks initialization even if it was skipped in this case, or this project can be added to a repo setup with CI/CD already, following the directions under "Setting up CI/CD" in the repo root directory README. + +To add CI/CD to this repo: + 1. Run `databricks bundle init mlops-stacks` via the Databricks CLI + 2. Select the option to only initialize `CICD_Only` + 3. Provide the root directory of this project and answer the subsequent prompts + +More details can be found on the homepage [MLOps Stacks README](https://github.com/databricks/mlops-stacks/blob/main/README.md). diff --git a/mlops_stacks/mlops_stacks/databricks.yml b/mlops_stacks/mlops_stacks/databricks.yml index 63bc11d0..c015305c 100644 --- a/mlops_stacks/mlops_stacks/databricks.yml +++ b/mlops_stacks/mlops_stacks/databricks.yml @@ -1,5 +1,8 @@ # The name of the bundle. run `databricks bundle schema` to see the full bundle settings schema. bundle: + # Do not modify the below line, this autogenerated field is used by the Databricks backend. + uuid: 11ae0fb5-b4fd-4db9-8ef5-7c20f0df54f0 + name: mlops_stacks variables: @@ -9,30 +12,44 @@ variables: model_name: description: Model name for the model training. default: mlops_stacks-model + catalog_name: + description: The catalog name to save the trained model include: - # Resources folder contains ML artifact resources for the ml project that defines model and experiment - # And workflows resources for the ml project including model training -> validation -> deployment, - # batch inference, data monitoring, metric refresh, alerts and triggering retraining - - ./resources/*.yml + # Resources folder contains ML artifact resources for the ML project that defines model and experiment + # And workflows resources for the ML project including model training -> validation -> deployment, + # batch inference, quality monitoring, metric refresh, alerts and triggering retraining + - ./resources/batch-inference-workflow-resource.yml + - ./resources/ml-artifacts-resource.yml + - ./resources/model-workflow-resource.yml + # TODO: uncomment once monitoring inference table has been created + # - ./resources/monitoring-resource.yml # Deployment Target specific values for workspace targets: - dev: + dev: # UC Catalog Name + mode: development default: true + variables: + catalog_name: dev workspace: # TODO: add dev workspace URL host: staging: + variables: + catalog_name: staging workspace: host: https://adb-xxxx.xx.azuredatabricks.net prod: + variables: + catalog_name: prod workspace: host: https://adb-xxxx.xx.azuredatabricks.net test: + variables: + catalog_name: test workspace: host: https://adb-xxxx.xx.azuredatabricks.net - diff --git a/mlops_stacks/mlops_stacks/deployment/batch_inference/notebooks/BatchInference.py b/mlops_stacks/mlops_stacks/deployment/batch_inference/BatchInference.py similarity index 78% rename from mlops_stacks/mlops_stacks/deployment/batch_inference/notebooks/BatchInference.py rename to mlops_stacks/mlops_stacks/deployment/batch_inference/BatchInference.py index e569401b..722f977c 100644 --- a/mlops_stacks/mlops_stacks/deployment/batch_inference/notebooks/BatchInference.py +++ b/mlops_stacks/mlops_stacks/deployment/batch_inference/BatchInference.py @@ -28,35 +28,11 @@ dbutils.widgets.text("output_table_name", "", label="Output Table Name") # Unity Catalog registered model name to use for the trained mode. dbutils.widgets.text( - "model_name", "dev.my-mlops-project.mlops_stacks-model", label="Full (Three-Level) Model Name" + "model_name", "dev.mlops_stacks.mlops_stacks-model", label="Full (Three-Level) Model Name" ) # COMMAND ---------- -import os - -notebook_path = '/Workspace/' + os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) -%cd $notebook_path - -# COMMAND ---------- - -# MAGIC %pip install -r ../../../requirements.txt - -# COMMAND ---------- - -dbutils.library.restartPython() - -# COMMAND ---------- - -import sys -import os -notebook_path = '/Workspace/' + os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) -%cd $notebook_path -%cd .. -sys.path.append("../..") - -# COMMAND ---------- - # DBTITLE 1,Define input and output variables env = dbutils.widgets.get("env") @@ -66,7 +42,7 @@ assert input_table_name != "", "input_table_name notebook parameter must be specified" assert output_table_name != "", "output_table_name notebook parameter must be specified" assert model_name != "", "model_name notebook parameter must be specified" -alias = "Champion" +alias = "champion" model_uri = f"models:/{model_name}@{alias}" # COMMAND ---------- @@ -74,7 +50,7 @@ from mlflow import MlflowClient # Get model version from alias -client = MlflowClient(registry_uri="databricks-uc") +client = MlflowClient() model_version = client.get_model_version_by_alias(model_name, alias).version # COMMAND ---------- diff --git a/mlops_stacks/mlops_stacks/deployment/batch_inference/README.md b/mlops_stacks/mlops_stacks/deployment/batch_inference/README.md index 3e1930c6..32fe5a19 100644 --- a/mlops_stacks/mlops_stacks/deployment/batch_inference/README.md +++ b/mlops_stacks/mlops_stacks/deployment/batch_inference/README.md @@ -10,6 +10,7 @@ df = spark.table( ).drop("fare_amount") df.write.mode("overwrite").saveAsTable( - name=".my-mlops-project.feature_store_inference_input" + name=".mlops_stacks.feature_store_inference_input" ) ``` + diff --git a/mlops_stacks/mlops_stacks/deployment/batch_inference/predict.py b/mlops_stacks/mlops_stacks/deployment/batch_inference/predict.py index c08b61d2..c186b286 100644 --- a/mlops_stacks/mlops_stacks/deployment/batch_inference/predict.py +++ b/mlops_stacks/mlops_stacks/deployment/batch_inference/predict.py @@ -9,19 +9,19 @@ def predict_batch( Apply the model at the specified URI for batch inference on the table with name input_table_name, writing results to the table with name output_table_name """ - mlflow.set_registry_uri("databricks-uc") table = spark_session.table(input_table_name) predict = mlflow.pyfunc.spark_udf( - spark_session, model_uri, result_type="string", env_manager="virtualenv" + spark_session, model_uri, result_type="double" ) + output_df = ( table.withColumn("prediction", predict(struct(*table.columns))) - .withColumn("model_version", lit(model_version)) - .withColumn("inference_timestamp", to_timestamp(lit(ts))) + .withColumn("model_id", lit(model_version)) + .withColumn("timestamp", to_timestamp(lit(ts))) ) - output_df.display() + # Model predictions are written to the Delta table provided as input. # Delta is the default format in Databricks Runtime 8.0 and above. - output_df.write.format("delta").mode("overwrite").saveAsTable(output_table_name) \ No newline at end of file + output_df.write.format("delta").mode("overwrite").saveAsTable(output_table_name) diff --git a/mlops_stacks/mlops_stacks/deployment/model_deployment/notebooks/ModelDeployment.py b/mlops_stacks/mlops_stacks/deployment/model_deployment/ModelDeployment.py similarity index 87% rename from mlops_stacks/mlops_stacks/deployment/model_deployment/notebooks/ModelDeployment.py rename to mlops_stacks/mlops_stacks/deployment/model_deployment/ModelDeployment.py index f218f09f..5f6199ed 100644 --- a/mlops_stacks/mlops_stacks/deployment/model_deployment/notebooks/ModelDeployment.py +++ b/mlops_stacks/mlops_stacks/deployment/model_deployment/ModelDeployment.py @@ -16,7 +16,7 @@ # * model_uri (required) - URI of the model to deploy. Must be in the format "models://", as described in # https://www.mlflow.org/docs/latest/model-registry.html#fetching-an-mlflow-model-from-the-model-registry # This parameter is read as a task value -# (https://learn.microsoft.com/azure/databricks/dev-tools/databricks-utils#get-command-dbutilsjobstaskvaluesget), +# (https://learn.microsoft.com/azure/databricks/dev-tools/databricks-utils), # rather than as a notebook widget. That is, we assume a preceding task (the Train.py # notebook) has set a task value with key "model_uri". ################################################################################## @@ -29,15 +29,6 @@ # COMMAND ---------- -import os -import sys -notebook_path = '/Workspace/' + os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) -%cd $notebook_path -%cd .. -sys.path.append("../..") - -# COMMAND ---------- - from deploy import deploy model_uri = dbutils.jobs.taskValues.get("Train", "model_uri", debugValue="") diff --git a/mlops_stacks/mlops_stacks/deployment/model_deployment/deploy.py b/mlops_stacks/mlops_stacks/deployment/model_deployment/deploy.py index 8dd72990..56254638 100644 --- a/mlops_stacks/mlops_stacks/deployment/model_deployment/deploy.py +++ b/mlops_stacks/mlops_stacks/deployment/model_deployment/deploy.py @@ -1,10 +1,5 @@ import sys -import pathlib - -sys.path.append(str(pathlib.Path(__file__).parent.parent.parent.resolve())) - -from mlflow.tracking import MlflowClient - +from mlflow import MlflowClient def deploy(model_uri, env): """Deploys an already-registered model in Unity catalog by assigning it the appropriate alias for model deployment. @@ -17,23 +12,22 @@ def deploy(model_uri, env): """ print(f"Deployment running in env: {env}") _, model_name, version = model_uri.split("/") - client = MlflowClient(registry_uri="databricks-uc") + client = MlflowClient() mv = client.get_model_version(model_name, version) - target_alias = "Champion" + target_alias = "champion" if target_alias not in mv.aliases: client.set_registered_model_alias( name=model_name, - alias=target_alias, + alias=target_alias, version=version) print(f"Assigned alias '{target_alias}' to model version {model_uri}.") - - # remove "Challenger" alias if assigning "Champion" alias - if target_alias == "Champion" and "Challenger" in mv.aliases: - print(f"Removing 'Challenger' alias from model version {model_uri}.") + + # remove "challenger" alias if assigning "champion" alias + if target_alias == "champion" and "challenger" in mv.aliases: + print(f"Removing 'challenger' alias from model version {model_uri}.") client.delete_registered_model_alias( name=model_name, - alias="Challenger") - + alias="challenger") if __name__ == "__main__": diff --git a/mlops_stacks/mlops_stacks/monitoring/MonitoredMetricViolationCheck.py b/mlops_stacks/mlops_stacks/monitoring/MonitoredMetricViolationCheck.py new file mode 100644 index 00000000..77cafb9f --- /dev/null +++ b/mlops_stacks/mlops_stacks/monitoring/MonitoredMetricViolationCheck.py @@ -0,0 +1,61 @@ +# Databricks notebook source +################################################################################## +# This notebook runs a sql query and set the result as job task value +# +# This notebook has the following parameters: +# +# * table_name_under_monitor (required) - The name of a table that is currently being monitored +# * metric_to_monitor (required) - Metric to be monitored for threshold violation +# * metric_violation_threshold (required) - Threshold value for metric violation +# * num_evaluation_windows (required) - Number of windows to check for violation +# * num_violation_windows (required) - Number of windows that need to violate the threshold +################################################################################## + +# List of input args needed to run the notebook as a job. +# Provide them via DB widgets or notebook arguments. +# +# Name of the table that is currently being monitored +dbutils.widgets.text( + "table_name_under_monitor", "dev.mlops_stacks.predictions", label="Full (three-Level) table name" +) +# Metric to be used for threshold violation check +dbutils.widgets.text( + "metric_to_monitor", "root_mean_squared_error", label="Metric to be monitored for threshold violation" +) + +# Threshold value to be checked +dbutils.widgets.text( + "metric_violation_threshold", "100", label="Threshold value for metric violation" +) + +# Threshold value to be checked +dbutils.widgets.text( + "num_evaluation_windows", "5", label="Number of windows to check for violation" +) + +# Threshold value to be checked +dbutils.widgets.text( + "num_violation_windows", "2", label="Number of windows that need to violate the threshold" +) + +# COMMAND ---------- + +from metric_violation_check_query import sql_query + +table_name_under_monitor = dbutils.widgets.get("table_name_under_monitor") +metric_to_monitor = dbutils.widgets.get("metric_to_monitor") +metric_violation_threshold = dbutils.widgets.get("metric_violation_threshold") +num_evaluation_windows = dbutils.widgets.get("num_evaluation_windows") +num_violation_windows = dbutils.widgets.get("num_violation_windows") + +formatted_sql_query = sql_query.format( + table_name_under_monitor=table_name_under_monitor, + metric_to_monitor=metric_to_monitor, + metric_violation_threshold=metric_violation_threshold, + num_evaluation_windows=num_evaluation_windows, + num_violation_windows=num_violation_windows) +is_metric_violated = bool(spark.sql(formatted_sql_query).toPandas()["query_result"][0]) + +dbutils.jobs.taskValues.set("is_metric_violated", is_metric_violated) + + diff --git a/mlops_stacks/mlops_stacks/monitoring/README.md b/mlops_stacks/mlops_stacks/monitoring/README.md index 909eb5e1..d020ad3b 100644 --- a/mlops_stacks/mlops_stacks/monitoring/README.md +++ b/mlops_stacks/mlops_stacks/monitoring/README.md @@ -1,5 +1,10 @@ # Monitoring -Databricks Data Monitoring is currently in Private Preview. +To enable monitoring as part of a scheduled Databricks workflow, please: +- Create the inference table that you want to monitor and was passed in as an initialization parameter. +- Update all the TODOs in the [monitoring resource file](../resources/monitoring-resource.yml). +- Uncomment the monitoring workflow from the main Databricks Asset Bundles file [databricks.yml](../databricks.yml). -Please contact a Databricks representative for more information. +For more details, refer to [mlops_stacks/resources/README.md](../resources/README.md). +The implementation supports monitoring of batch inference tables directly. +For real time inference tables, unpacking is required before monitoring can be attached. diff --git a/mlops_stacks/mlops_stacks/monitoring/metric_violation_check_query.py b/mlops_stacks/mlops_stacks/monitoring/metric_violation_check_query.py new file mode 100644 index 00000000..7be5eb29 --- /dev/null +++ b/mlops_stacks/mlops_stacks/monitoring/metric_violation_check_query.py @@ -0,0 +1,79 @@ +# This file is used for the main SQL query that checks the last {num_evaluation_windows} metric violations and whether at least {num_violation_windows} of those runs violate the condition. + +"""The SQL query is divided into three main parts. The first part selects the top {num_evaluation_windows} +values of the metric to be monitored, ordered by the time window, and saves as recent_metrics. +```sql +WITH recent_metrics AS ( + SELECT + {metric_to_monitor}, + window + FROM + {table_name_under_monitor}_profile_metrics + WHERE + column_name = ":table" + AND slice_key IS NULL + AND model_id != "*" + AND log_type = "INPUT" + ORDER BY + window DESC + LIMIT + {num_evaluation_windows} +) +``` +The `column_name = ":table"` and `slice_key IS NULL` conditions ensure that the metric +is selected for the entire table within the given granularity. The `log_type = "INPUT"` +condition ensures that the primary table metrics are considered, but not the baseline +table metrics. The `model_id!= "*"` condition ensures that the metric aggregated across +all model IDs is not selected. + +The second part of the query determines if the metric values have been violated with two cases. +The first case checks if the metric value is greater than the threshold for at least {num_violation_windows} windows: +```sql +(SELECT COUNT(*) FROM recent_metrics WHERE {metric_to_monitor} > {metric_violation_threshold}) >= {num_violation_windows} +``` +The second case checks if the most recent metric value is greater than the threshold. This is to make sure we only trigger retraining +if the most recent window was violated, avoiding unnecessary retraining if the violation was in the past and the metric is now within the threshold: +```sql +(SELECT {metric_to_monitor} FROM recent_metrics ORDER BY window DESC LIMIT 1) > {metric_violation_threshold} +``` + +The final part of the query sets the `query_result` to 1 if both of the above conditions are met, and 0 otherwise: +```sql +SELECT + CASE + WHEN + # Check if the metric value is greater than the threshold for at least {num_violation_windows} windows + AND + # Check if the most recent metric value is greater than the threshold + THEN 1 + ELSE 0 + END AS query_result +``` +""" + +sql_query = """WITH recent_metrics AS ( + SELECT + {metric_to_monitor}, + window + FROM + {table_name_under_monitor}_profile_metrics + WHERE + column_name = ":table" + AND slice_key IS NULL + AND model_id != "*" + AND log_type = "INPUT" + ORDER BY + window DESC + LIMIT + {num_evaluation_windows} +) +SELECT + CASE + WHEN + (SELECT COUNT(*) FROM recent_metrics WHERE {metric_to_monitor} > {metric_violation_threshold}) >= {num_violation_windows} + AND + (SELECT {metric_to_monitor} FROM recent_metrics ORDER BY window DESC LIMIT 1) > {metric_violation_threshold} + THEN 1 + ELSE 0 + END AS query_result +""" diff --git a/mlops_stacks/mlops_stacks/requirements.txt b/mlops_stacks/mlops_stacks/requirements.txt index 286a0f6b..c9329d48 100644 --- a/mlops_stacks/mlops_stacks/requirements.txt +++ b/mlops_stacks/mlops_stacks/requirements.txt @@ -1,8 +1,7 @@ -mlflow==2.7.1 -numpy>=1.23.0 -pandas>=1.4.3 +mlflow>=3.1 +lightgbm +numpy +pandas scikit-learn>=1.1.1 matplotlib>=3.5.2 -Jinja2==3.0.3 -pyspark~=3.3.0 -pytz~=2022.2.1 +pillow>=10.0.1 diff --git a/mlops_stacks/mlops_stacks/resources/README.md b/mlops_stacks/mlops_stacks/resources/README.md index 2618287f..8fe2bf2c 100644 --- a/mlops_stacks/mlops_stacks/resources/README.md +++ b/mlops_stacks/mlops_stacks/resources/README.md @@ -1,12 +1,12 @@ # Databricks ML Resource Configurations -[(back to main README)](../../README.md) +[(back to project README)](../README.md) ## Table of contents * [Intro](#intro) * [Local development and dev workspace](#local-development-and-dev-workspace) +* [Develop and test config changes](#develop-and-test-config-changes) * [CI/CD](#set-up-cicd) * [Deploy initial ML resources](#deploy-initial-ml-resources) -* [Develop and test config changes](#develop-and-test-config-changes) * [Deploy config changes](#deploy-config-changes) ## Intro @@ -21,7 +21,7 @@ During databricks CLI bundles deployment, the root config file will be loaded, v ML Resource Configurations in this directory: - model workflow (`mlops_stacks/resources/model-workflow-resource.yml`) - batch inference workflow (`mlops_stacks/resources/batch-inference-workflow-resource.yml`) - - monitoring workflow (`mlops_stacks/resources/monitoring-workflow-resource.yml`) + - monitoring resource and workflow (`mlops_stacks/resources/monitoring-resource.yml`) - feature engineering workflow (`mlops_stacks/resources/feature-engineering-workflow-resource.yml`) - model definition and experiment definition (`mlops_stacks/resources/ml-artifacts-resource.yml`) @@ -29,10 +29,10 @@ ML Resource Configurations in this directory: ### Deployment Config & CI/CD integration The ML resources can be deployed to databricks workspace based on the databricks CLI bundles deployment config. Deployment configs of different deployment targets share the general ML resource configurations with added ability to specify deployment target specific values (workspace URI, model name, jobs notebook parameters, etc). - This project ships with CI/CD workflows for developing and deploying ML resource configurations based on deployment config. -NOTE: For Model Registry in Unity Catalog, we expect a catalog to exist with the name of the deployment target by default. For example, if the deployment target is `dev`, we expect a catalog named `dev` to exist in the workspace. + +For Model Registry in Unity Catalog, we expect a catalog to exist with the name of the deployment target by default. For example, if the deployment target is `dev`, we expect a catalog named `dev` to exist in the workspace. If you want to use different catalog names, please update the `targets` declared in the `mlops_stacks/databricks.yml` and `mlops_stacks/resources/ml-artifacts-resource.yml` files. If changing the `staging`, `prod`, or `test` deployment targets, you'll need to update the workflows located in the `.github/workflows` directory. @@ -52,9 +52,7 @@ The PR will trigger Python unit tests, followed by an integration test executed Upon merging a PR to the main branch, the main branch content will be deployed to the staging workspace with `staging` environment resource configurations. Upon merging code into the release branch, the release branch content will be deployed to prod workspace with `prod` environment resource configurations. - - -![ML resource config diagram](../../docs/images/mlops-resource-config.png) +![ML resource config diagram](../../docs/images/mlops-stack-deploy.png) ## Local development and dev workspace @@ -64,7 +62,7 @@ To set up the databricks CLI using a Databricks personal access token, take the 1. Follow [databricks CLI](https://learn.microsoft.com/azure/databricks/dev-tools/cli/databricks-cli) to download and set up the databricks CLI locally. 2. Complete the `TODO` in `mlops_stacks/databricks.yml` to add the dev workspace URI under `targets.dev.workspace.host`. -3. [Create a personal access token](https://learn.microsoft.com/azure/databricks/dev-tools/auth#personal-access-tokens-for-users) +3. [Create a personal access token](https://learn.microsoft.com/azure/databricks/dev-tools/auth/pat) in your dev workspace and copy it. 4. Set an env variable `DATABRICKS_TOKEN` with your Databricks personal access token in your terminal. For example, run `export DATABRICKS_TOKEN=dapi12345` if the access token is dapi12345. 5. You can now use the databricks CLI to validate and deploy ML resource configurations to the dev workspace. @@ -74,12 +72,11 @@ Alternatively, you can use the other approaches described in the [databricks CLI ### Validate and provision ML resource configurations 1. After installing the databricks CLI and creating the `DATABRICKS_TOKEN` env variable, change to the `mlops_stacks` directory. 2. Run `databricks bundle validate` to validate the Databricks resource configurations. -3. Run `databricks bundle deploy` to provision the Databricks resource configurations to the dev workspace. The resource configurations and your ML code will be copied together to the dev workspace. The defined resources such as Databricks Workflows, MLflow Model and MLflow Experiment will be provisioned according to the config files under `mlops_stacks/resources`. +3. Run `databricks bundle deploy` to provision the Databricks resource configurations to the dev workspace. The resource configurations and your ML code will be copied together to the dev workspace. The defined resources such as Lakeflow Jobs, MLflow Model and MLflow Experiment will be provisioned according to the config files under `mlops_stacks/resources`. 4. Go to the Databricks dev workspace, check the defined model, experiment and workflows status, and interact with the created workflows. ### Destroy ML resource configurations After development is done, you can run `databricks bundle destroy` to destroy (remove) the defined Databricks resources in the dev workspace. Any model version with `Production` or `Staging` stage will prevent the model from being deleted. Please update the version stage to `None` or `Archived` before destroying the ML resources. - ## Set up CI/CD Please refer to [mlops-setup](../../docs/mlops-setup.md#configure-cicd) for instructions to set up CI/CD. @@ -96,13 +93,37 @@ Upon creating this PR, the CI workflows will be triggered. These CI workflow will run unit and integration tests of the ML code, in addition to validating the Databricks resources to be deployed to both staging and prod workspaces. Once CI passes, merge the PR into the `main` branch. This will deploy an initial set of Databricks resources to the staging workspace. -Resources will be deployed to the prod workspace on pushing code to the `release` branch. +resources will be deployed to the prod workspace on pushing code to the `release` branch. Follow the next section to configure the input and output data tables for the batch inference job. ### Setting up the batch inference job -The batch inference job expects an input Delta table with a schema that your registered model accepts. To use the batch -inference job, set up such a Delta table in both your staging and prod workspaces. +The batch inference job expects an input Delta table with a schema that your registered model accepts (out-of-the-box we provide a working example using the `nyctaxi` dataset with the schema [trip_distance, pickup_zip and dropoff_zip]). To use the batch +inference job, set up such a Delta table in both your staging and prod workspaces. For example, with the `nyctaxi` dataset, the following code can be run in a Databricks notebook to create the batch inference input table: +``` +# To test the batch job the training data can be used as input + +input_table_path = "/databricks-datasets/nyctaxi-with-zipcodes/subsampled" +training_df = spark.read.format("delta").load(input_table_path) + +# drop unused columns +df = df.drop("tpep_pickup_datetime", "tpep_dropoff_datetime") + +# target column used for monitoring, not for predictions +df = df.withColumnRenamed('fare_amount', 'price') + +spark.sql("CREATE DATABASE IF NOT EXISTS my_catalog.my_schema") + +def write_to_table(df, database, table): + (df.write + .format("delta") + .mode("overwrite") + .option("overwriteSchema", "true") + .saveAsTable(f"{database}.{table}")) + +# Write the DataFrame to a Delta table +write_to_table(training_df, database="my_catalog.my_schema", table="batch_input_table") +``` Following this, update the batch_inference_job base parameters in `mlops_stacks/resources/batch-inference-workflow-resource.yml` to pass the name of the input Delta table and the name of the output Delta table to which to write batch predictions. @@ -121,47 +142,82 @@ Its central purpose is to evaluate a registered model and validate its quality b Model validation contains three components: * [model-workflow-resource.yml](./model-workflow-resource.yml) contains the resource config and input parameters for model validation. * [validation.py](../validation/validation.py) defines custom metrics and validation thresholds that are referenced by the above resource config files. -* [notebooks/ModelValidation](../validation/notebooks/ModelValidation.py) contains the validation job implementation. In most cases you don't need to modify this file. +* [ModelValidation](../validation/ModelValidation.py) contains the validation job implementation. In most cases you don't need to modify this file. To set up and enable model validation, update [validation.py](../validation/validation.py) to return desired custom metrics and validation thresholds, then resolve the `TODOs` in the ModelValidation task of [model-workflow-resource.yml](./model-workflow-resource.yml). + +### Setting up monitoring +The monitoring workflow focuses on building a plug-and-play stack component for monitoring the feature drifts and model drifts and retrain based on the +violation threshold defined given the ground truth labels. + +Its central purpose is to track production model performances, feature distributions and comparing different versions. + +Monitoring contains four components: +* [metric_violation_check_query.py](../monitoring/metric_violation_check_query.py) defines a query that checks for violation of the monitored metric. +* [MonitoredMetricViolationCheck](../monitoring/MonitoredMetricViolationCheck.py) acts as an entry point, executing the violation check query against the monitored inference table. +It emits a boolean value based on the query result. +* [monitoring-resource.yml](./monitoring-resource.yml) contains the resource config, inputs parameters for monitoring, and orchestrates model retraining based on monitoring. It first runs the [MonitoredMetricViolationCheck](../monitoring/MonitoredMetricViolationCheck.py) +entry point then decides whether to execute the model retraining workflow. + +To set up and enable monitoring: +* If it is not done already, generate inference table, join it with ground truth labels, and update the table name in [monitoring-resource.yml](./monitoring-resource.yml). +* Resolve the `TODOs` in [monitoring-resource.yml](./monitoring-resource.yml) +* Uncomment the monitoring workflow in [databricks.yml](../databricks.yml) +* OPTIONAL: Update the query in [metric_violation_check_query.py](../monitoring/metric_violation_check_query.py) to customize when the metric is considered to be in violation. + +NOTE: If ground truth labels are not available, you can still set up monitoring but should disable the retraining workflow. + +Retraining Constraints: +The retraining job has constraints for optimal functioning: +* Labels must be provided by the user, joined correctly for retraining history, and available on time with the retraining frequency. +* Retraining Frequency is tightly coupled with the granularity of the monitor. Users should take into account and ensure that their retraining frequency is equal to or close to the granularity of the monitor. + * If the granularity of the monitor is 1 day and retraining frequency is 1 hour, the job will preemptively stop as there is no new data to evaluate retraining criteria + * If the granularity of the monitor is 1 day and retraining frequency is 1 week, retraining would be stale and not be efficient + +Permissions: +Permissions for monitoring are inherited from the original table's permissions. +* Users who own the monitored table or its parent catalog/schema can create, update, and view monitors. +* Users with read permissions on the monitored table can view its monitor. + +Therefore, ensure that service principals are the owners or have the necessary permissions to manage the monitored table. + ## Develop and test config changes ### databricks CLI bundles schema overview To get started, open `mlops_stacks/resources/batch-inference-workflow-resource.yml`. The file contains the ML resource definition of a batch inference job, like: ```$xslt -new_cluster: &new_cluster - new_cluster: - num_workers: 3 - spark_version: 13.3.x-cpu-ml-scala2.12 - node_type_id: Standard_D3_v2 - custom_tags: - clusterSource: mlops-stack/0.2 - resources: jobs: batch_inference_job: name: ${bundle.target}-mlops_stacks-batch-inference-job tasks: - task_key: batch_inference_job - <<: *new_cluster + environment_key: default notebook_task: - notebook_path: ../deployment/batch_inference/notebooks/BatchInference.py + notebook_path: ../deployment/batch_inference/BatchInference.py base_parameters: env: ${bundle.target} input_table_name: batch_inference_input_table_name ... + + environments: + - environment_key: default + spec: + environment_version: "5" + dependencies: + - -r ../requirements.txt ``` The example above defines a Databricks job with name `${bundle.target}-mlops_stacks-batch-inference-job` -that runs the notebook under `mlops_stacks/deployment/batch_inference/notebooks/BatchInference.py` to regularly apply your ML model for batch inference. +that runs the notebook under `mlops_stacks/deployment/batch_inference/BatchInference.py` to regularly apply your ML model for batch inference. -At the start of the resource definition, we declared an anchor `new_cluster` that will be referenced and used later. For more information about anchors in yaml schema, please refer to the [yaml documentation](https://yaml.org/spec/1.2.2/#3222-anchors-and-aliases). +Jobs run on [serverless compute](https://learn.microsoft.com/azure/databricks/jobs/run-serverless-jobs) by default: the task references a serverless environment by its `environment_key` (`default` here), and that key is defined once under `environments`, with its own dependencies installed from `requirements.txt`. Multiple tasks in the same job can share the same `environment_key` to reuse one environment definition. We specify a `batch_inference_job` under `resources/jobs` to define a databricks workflow with internal key `batch_inference_job` and job name `{bundle.target}-mlops_stacks-batch-inference-job`. -The workflow contains a single task with task key `batch_inference_job`. The task runs notebook `../deployment/batch_inference/notebooks/BatchInference.py` with provided parameters `env` and `input_table_name` passing to the notebook. +The workflow contains a single task with task key `batch_inference_job`. The task runs notebook `../deployment/batch_inference/BatchInference.py` with provided parameters `env` and `input_table_name` passing to the notebook. After setting up databricks CLI, you can run command `databricks bundle schema` to learn more about databricks CLI bundles schema. The notebook_path is the relative path starting from the resource yaml file. @@ -187,27 +243,26 @@ targets: variables: batch_inference_input_table: test_table -new_cluster: &new_cluster - new_cluster: - num_workers: 3 - spark_version: 13.3.x-cpu-ml-scala2.12 - node_type_id: Standard_D3_v2 - custom_tags: - clusterSource: mlops-stack/0.2 - resources: jobs: batch_inference_job: name: ${bundle.target}-mlops_stacks-batch-inference-job tasks: - task_key: batch_inference_job - <<: *new_cluster + environment_key: default notebook_task: - notebook_path: ../deployment/batch_inference/notebooks/BatchInference.py + notebook_path: ../deployment/batch_inference/BatchInference.py base_parameters: env: ${bundle.target} input_table_name: ${var.batch_inference_input_table} ... + + environments: + - environment_key: default + spec: + environment_version: "5" + dependencies: + - -r ../requirements.txt ``` The `batch_inference_job` notebook parameter `input_table_name` is using a bundle variable `batch_inference_input_table` with default value "input_table". The variable value will be overwritten with "dev_table" for `dev` environment config and "test_table" for `test` environment config: @@ -221,7 +276,6 @@ To test out a config change, simply edit one of the fields above. For example, i Then follow [Local development and dev workspace](#local-development-and-dev-workspace) to deploy the change to the dev workspace. Alternatively you can open a PR. Continuous integration will then validate the updated config and deploy tests to the to staging workspace. - ## Deploy config changes ### Dev workspace deployment @@ -236,4 +290,4 @@ After merging a PR to the main branch, continuous deployment automation will dep When you about to cut a release, you can create and merge a PR to merge changes from main to release. Continuous deployment automation will deploy `prod` resources to the prod workspace. -[Back to main project README](../../README.md) +[Back to project README](../README.md) diff --git a/mlops_stacks/mlops_stacks/resources/batch-inference-workflow-resource.yml b/mlops_stacks/mlops_stacks/resources/batch-inference-workflow-resource.yml index 6f0cf651..37daca0a 100644 --- a/mlops_stacks/mlops_stacks/resources/batch-inference-workflow-resource.yml +++ b/mlops_stacks/mlops_stacks/resources/batch-inference-workflow-resource.yml @@ -1,12 +1,4 @@ -new_cluster: &new_cluster - new_cluster: - num_workers: 3 - spark_version: 13.3.x-cpu-ml-scala2.12 - node_type_id: Standard_D3_v2 - custom_tags: - clusterSource: mlops-stack/0.2 - -permissions: &permissions +common_permissions: &permissions permissions: - level: CAN_VIEW group_name: users @@ -17,17 +9,23 @@ resources: name: ${bundle.target}-mlops_stacks-batch-inference-job tasks: - task_key: batch_inference_job - <<: *new_cluster + environment_key: default notebook_task: - notebook_path: ../deployment/batch_inference/notebooks/BatchInference.py + notebook_path: ../deployment/batch_inference/BatchInference.py base_parameters: env: ${bundle.target} input_table_name: taxi_scoring_sample # TODO: create input table for inference - output_table_name: ${bundle.target}.my-mlops-project.predictions - model_name: ${bundle.target}.my-mlops-project.${var.model_name} + output_table_name: ${var.catalog_name}.mlops_stacks.predictions + model_name: ${var.catalog_name}.mlops_stacks.${var.model_name} # git source information of current ML resource deployment. It will be persisted as part of the workflow run git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} + environments: + - environment_key: default + spec: + environment_version: "5" + dependencies: + - -r ../requirements.txt schedule: quartz_cron_expression: "0 0 11 * * ?" # daily at 11am timezone_id: UTC diff --git a/mlops_stacks/mlops_stacks/resources/ml-artifacts-resource.yml b/mlops_stacks/mlops_stacks/resources/ml-artifacts-resource.yml index a52f6072..1eb45338 100644 --- a/mlops_stacks/mlops_stacks/resources/ml-artifacts-resource.yml +++ b/mlops_stacks/mlops_stacks/resources/ml-artifacts-resource.yml @@ -1,37 +1,5 @@ -# Deployment target specific values -targets: - dev: - resources: - registered_models: - model: - comment: Registered model in Unity Catalog for the "mlops_stacks" ML Project for ${bundle.target} deployment target. - - test: - resources: - registered_models: - model: - comment: Registered model in Unity Catalog for the "mlops_stacks" ML Project for ${bundle.target} deployment target. - - staging: - resources: - registered_models: - model: - comment: Registered model in Unity Catalog for the "mlops_stacks" ML Project for ${bundle.target} deployment target. - - prod: - resources: - registered_models: - model: - comment: | - Registered model in Unity Catalog for the "mlops_stacks" ML Project. See the corresponding [Git repo]($#{var.git_repo_url}) for details on the project. - - Links: - * [Recurring model training job](https://adb-xxxx.xx.azuredatabricks.net#job/${resources.jobs.model_training_job.id}): trains fresh model versions using the latest ML code. - * [Recurring batch inference job](https://adb-xxxx.xx.azuredatabricks.net#job/${resources.jobs.batch_inference_job.id}): applies the latest ${bundle.target} model version for batch inference. - - -# Allow users to read the experiment -permissions: &permissions +# Allow users to read the experiment +common_permissions: &permissions permissions: - level: CAN_READ group_name: users @@ -46,14 +14,12 @@ grants: &grants # Defines model and experiments resources: registered_models: - model: - name: ${var.model_name} - catalog_name: ${bundle.target} - schema_name: my-mlops-project - <<: *grants - depends_on: - - resources.jobs.model_training_job.id - - resources.jobs.batch_inference_job.id + model: + name: ${var.model_name} + catalog_name: ${var.catalog_name} + schema_name: mlops_stacks + comment: Registered model in Unity Catalog for the "mlops_stacks" ML Project for ${bundle.target} deployment target. + <<: *grants experiments: experiment: diff --git a/mlops_stacks/mlops_stacks/resources/model-workflow-resource.yml b/mlops_stacks/mlops_stacks/resources/model-workflow-resource.yml index 6400322b..e37a4ea7 100644 --- a/mlops_stacks/mlops_stacks/resources/model-workflow-resource.yml +++ b/mlops_stacks/mlops_stacks/resources/model-workflow-resource.yml @@ -1,12 +1,4 @@ -new_cluster: &new_cluster - new_cluster: - num_workers: 3 - spark_version: 13.3.x-cpu-ml-scala2.12 - node_type_id: Standard_D3_v2 - custom_tags: - clusterSource: mlops-stack/0.2 - -permissions: &permissions +common_permissions: &permissions permissions: - level: CAN_VIEW group_name: users @@ -15,29 +7,26 @@ resources: jobs: model_training_job: name: ${bundle.target}-mlops_stacks-model-training-job - job_clusters: - - job_cluster_key: model_training_job_cluster - <<: *new_cluster tasks: - task_key: Train - job_cluster_key: model_training_job_cluster + environment_key: default notebook_task: - notebook_path: ../training/notebooks/Train.py + notebook_path: ../training/Train.py base_parameters: env: ${bundle.target} # TODO: Update training_data_path training_data_path: /databricks-datasets/nyctaxi-with-zipcodes/subsampled experiment_name: ${var.experiment_name} - model_name: ${bundle.target}.my-mlops-project.${var.model_name} + model_name: ${var.catalog_name}.mlops_stacks.${var.model_name} # git source information of current ML resource deployment. It will be persisted as part of the workflow run git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} - task_key: ModelValidation - job_cluster_key: model_training_job_cluster + environment_key: default depends_on: - task_key: Train notebook_task: - notebook_path: ../validation/notebooks/ModelValidation.py + notebook_path: ../validation/ModelValidation.py base_parameters: experiment_name: ${var.experiment_name} # The `run_mode` defines whether model validation is enabled or not. @@ -63,28 +52,25 @@ resources: # The string name of a column from data that contains evaluation labels. # Please refer to targets parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate # TODO: targets - targets: mean_squared_error - # Specifies the name of the function in mlops_stacks/training_validation_deployment/validation/validation.py that returns custom metrics. - # TODO(optional): custom_metrics_loader_function - custom_metrics_loader_function: custom_metrics - # Specifies the name of the function in mlops_stacks/training_validation_deployment/validation/validation.py that returns model validation thresholds. - # TODO(optional): validation_thresholds_loader_function - validation_thresholds_loader_function: validation_thresholds - # Specifies the name of the function in mlops_stacks/training_validation_deployment/validation/validation.py that returns evaluator_config. - # TODO(optional): evaluator_config_loader_function - evaluator_config_loader_function: evaluator_config + targets: fare_amount # git source information of current ML resource deployment. It will be persisted as part of the workflow run git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} - task_key: ModelDeployment - job_cluster_key: model_training_job_cluster + environment_key: default depends_on: - task_key: ModelValidation notebook_task: - notebook_path: ../deployment/model_deployment/notebooks/ModelDeployment.py + notebook_path: ../deployment/model_deployment/ModelDeployment.py base_parameters: env: ${bundle.target} # git source information of current ML resource deployment. It will be persisted as part of the workflow run git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} + environments: + - environment_key: default + spec: + environment_version: "5" + dependencies: + - -r ../requirements.txt schedule: quartz_cron_expression: "0 0 9 * * ?" # daily at 9am timezone_id: UTC diff --git a/mlops_stacks/mlops_stacks/resources/monitoring-resource.yml b/mlops_stacks/mlops_stacks/resources/monitoring-resource.yml new file mode 100644 index 00000000..81a78016 --- /dev/null +++ b/mlops_stacks/mlops_stacks/resources/monitoring-resource.yml @@ -0,0 +1,80 @@ +# Please complete all the TODOs in this file. +# The regression monitor defined here works OOB with this example regression notebook: https://learn.microsoft.com/azure/databricks/_extras/notebooks/source/monitoring/regression-monitor +# NOTE: Monitoring only works on Unity Catalog tables. + +common_permissions: &permissions + permissions: + - level: CAN_VIEW + group_name: users + +resources: + quality_monitors: + mlops_stacks_quality_monitor: + table_name: dev.mlops_stacks.predictions + # TODO: Update the output schema name as per your requirements + output_schema_name: ${bundle.target}.mlops_stacks + # TODO: Update the below parameters as per your requirements + assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + label_col: price + problem_type: PROBLEM_TYPE_REGRESSION + timestamp_col: timestamp + schedule: + quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am + timezone_id: UTC + jobs: + retraining_job: + name: ${bundle.target}-mlops_stacks-monitoring-retraining-job + tasks: + - task_key: monitored_metric_violation_check + environment_key: default + notebook_task: + notebook_path: ../monitoring/MonitoredMetricViolationCheck.py + base_parameters: + env: ${bundle.target} + table_name_under_monitor: dev.mlops_stacks.predictions + # TODO: Update the metric to be monitored and violation threshold + metric_to_monitor: root_mean_squared_error + metric_violation_threshold: 100 + num_evaluation_windows: 5 + num_violation_windows: 2 + # git source information of current ML resource deployment. It will be persisted as part of the workflow run + git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} + + - task_key: is_metric_violated + depends_on: + - task_key: monitored_metric_violation_check + condition_task: + op: EQUAL_TO + left: "{{tasks.monitored_metric_violation_check.values.is_metric_violated}}" + right: "true" + git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} + + - task_key: trigger_retraining + depends_on: + - task_key: is_metric_violated + outcome: "true" + run_job_task: + job_id: ${resources.jobs.model_training_job.id} + git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit} + + environments: + - environment_key: default + spec: + environment_version: "5" + dependencies: + - -r ../requirements.txt + schedule: + quartz_cron_expression: "0 0 18 * * ?" # daily at 6pm + timezone_id: UTC + <<: *permissions + # If you want to turn on notifications for this job, please uncomment the below code, + # and provide a list of emails to the on_failure argument. + # + # email_notifications: + # on_failure: + # - first@company.com + # - second@company.com diff --git a/mlops_stacks/mlops_stacks/resources/monitoring-workflow-resource.yml b/mlops_stacks/mlops_stacks/resources/monitoring-workflow-resource.yml deleted file mode 100644 index 730d16a7..00000000 --- a/mlops_stacks/mlops_stacks/resources/monitoring-workflow-resource.yml +++ /dev/null @@ -1 +0,0 @@ -# TODO: Add data monitoring support for mlops \ No newline at end of file diff --git a/mlops_stacks/mlops_stacks/training/notebooks/Train.py b/mlops_stacks/mlops_stacks/training/Train.py similarity index 88% rename from mlops_stacks/mlops_stacks/training/notebooks/Train.py rename to mlops_stacks/mlops_stacks/training/Train.py index 69271679..3d6373f0 100644 --- a/mlops_stacks/mlops_stacks/training/notebooks/Train.py +++ b/mlops_stacks/mlops_stacks/training/Train.py @@ -10,32 +10,13 @@ # * env (required): - Environment the notebook is run in (staging, or prod). Defaults to "staging". # * training_data_path (required) - Path to the training data. # * experiment_name (required) - MLflow experiment name for the training runs. Will be created if it doesn't exist. -# * model_name (required) - Three-level name (..) to register the trained model in Unity Catalog. -# +# * model_name (required) - Three-level name (..) to register the trained model in Unity Catalog. +# ################################################################################## # COMMAND ---------- -# MAGIC %load_ext autoreload -# MAGIC %autoreload 2 - -# COMMAND ---------- - -import os -notebook_path = '/Workspace/' + os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) -%cd $notebook_path - -# COMMAND ---------- - -# MAGIC %pip install -r ../../requirements.txt - -# COMMAND ---------- - -dbutils.library.restartPython() - -# COMMAND ---------- # DBTITLE 1, Notebook arguments - # List of input args needed to run this notebook as a job. # Provide them via DB widgets or notebook arguments. @@ -56,35 +37,37 @@ f"/dev-mlops_stacks-experiment", label="MLflow experiment name", ) + # Unity Catalog registered model name to use for the trained model. dbutils.widgets.text( - "model_name", "dev.my-mlops-project.mlops_stacks-model", label="Full (Three-Level) Model Name" + "model_name", "dev.mlops_stacks.mlops_stacks-model", label="Full (Three-Level) Model Name" ) + # COMMAND ---------- -# DBTITLE 1,Define input and output variables +# DBTITLE 1,Define input and output variables input_table_path = dbutils.widgets.get("training_data_path") experiment_name = dbutils.widgets.get("experiment_name") model_name = dbutils.widgets.get("model_name") # COMMAND ---------- -# DBTITLE 1, Set experiment +# DBTITLE 1, Set experiment import mlflow mlflow.set_experiment(experiment_name) -mlflow.set_registry_uri('databricks-uc') # COMMAND ---------- -# DBTITLE 1, Load raw data +# DBTITLE 1, Load raw data training_df = spark.read.format("delta").load(input_table_path) training_df.display() # COMMAND ---------- + # DBTITLE 1, Helper function -from mlflow.tracking import MlflowClient +from mlflow import MlflowClient import mlflow.pyfunc @@ -104,8 +87,8 @@ def get_latest_model_version(model_name): # MAGIC Train a LightGBM model on the data, then log and register the model with MLflow. # COMMAND ---------- -# DBTITLE 1, Train model +# DBTITLE 1, Train model import mlflow from sklearn.model_selection import train_test_split import lightgbm as lgb @@ -133,8 +116,8 @@ def get_latest_model_version(model_name): model = lgb.train(param, train_lgb_dataset, num_rounds) # COMMAND ---------- -# DBTITLE 1, Log model and return output. +# DBTITLE 1, Log model and return output. # Take the first row of the training dataset as the model input example. input_example = X_train.iloc[[0]] diff --git a/mlops_stacks/mlops_stacks/training/__init__.py b/mlops_stacks/mlops_stacks/training/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mlops_stacks/mlops_stacks/training/data/sample.parquet b/mlops_stacks/mlops_stacks/training/data/sample.parquet deleted file mode 100644 index 4efd1712..00000000 Binary files a/mlops_stacks/mlops_stacks/training/data/sample.parquet and /dev/null differ diff --git a/mlops_stacks/mlops_stacks/training/steps/__init__.py b/mlops_stacks/mlops_stacks/training/steps/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mlops_stacks/mlops_stacks/training/steps/custom_metrics.py b/mlops_stacks/mlops_stacks/training/steps/custom_metrics.py deleted file mode 100644 index 23d5eb25..00000000 --- a/mlops_stacks/mlops_stacks/training/steps/custom_metrics.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -This module defines custom metric functions that are invoked during the 'train' and 'evaluate' -steps to provide model performance insights. Custom metric functions defined in this module are -referenced in the ``metrics`` section of ``recipe.yaml``, for example: - -.. code-block:: yaml - :caption: Example custom metrics definition in ``recipe.yaml`` - - metrics: - custom: - - name: weighted_mean_squared_error - function: weighted_mean_squared_error - greater_is_better: False -""" - -from typing import Dict - -from pandas import DataFrame -from sklearn.metrics import mean_squared_error - - -def weighted_mean_squared_error( - eval_df: DataFrame, - builtin_metrics: Dict[str, int], # pylint: disable=unused-argument -) -> int: - """ - Computes the weighted mean squared error (MSE) metric. - - :param eval_df: A Pandas DataFrame containing the following columns: - - - ``"prediction"``: Predictions produced by submitting input data to the model. - - ``"target"``: Ground truth values corresponding to the input data. - - :param builtin_metrics: A dictionary containing the built-in metrics that are calculated - automatically during model evaluation. The keys are the names of the - metrics and the values are the scalar values of the metrics. For more - information, see - https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate. - :return: A single-entry dictionary containing the MSE metric. The key is the metric name and - the value is the scalar metric value. Note that custom metric functions can return - dictionaries with multiple metric entries as well. - """ - return mean_squared_error( - eval_df["prediction"], - eval_df["target"], - sample_weight=1 / eval_df["prediction"].values, - ) diff --git a/mlops_stacks/mlops_stacks/training/steps/ingest.py b/mlops_stacks/mlops_stacks/training/steps/ingest.py deleted file mode 100644 index 7dfa89fc..00000000 --- a/mlops_stacks/mlops_stacks/training/steps/ingest.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -This module defines the following routines used by the 'ingest' step of the regression recipe: - -- ``load_file_as_dataframe``: Defines customizable logic for parsing dataset formats that are not - natively parsed by MLflow Recipes (i.e. formats other than Parquet, Delta, and Spark SQL). -""" - -import logging - -from pandas import DataFrame - -_logger = logging.getLogger(__name__) - - -def load_file_as_dataframe(file_path: str, file_format: str) -> DataFrame: - """ - Load content from the specified dataset file as a Pandas DataFrame. - - This method is used to load dataset types that are not natively managed by MLflow Recipes - (datasets that are not in Parquet, Delta Table, or Spark SQL Table format). This method is - called once for each file in the dataset, and MLflow Recipes automatically combines the - resulting DataFrames together. - - :param file_path: The path to the dataset file. - :param file_format: The file format string, such as "csv". - :return: A Pandas DataFrame representing the content of the specified file. - """ - - if file_format == "csv": - import pandas - - _logger.warning( - "Loading dataset CSV using `pandas.read_csv()` with default arguments and assumed index" - " column 0 which may not produce the desired schema. If the schema is not correct, you" - " can adjust it by modifying the `load_file_as_dataframe()` function in" - " `steps/ingest.py`" - ) - return pandas.read_csv(file_path, index_col=0) - else: - raise NotImplementedError diff --git a/mlops_stacks/mlops_stacks/training/steps/split.py b/mlops_stacks/mlops_stacks/training/steps/split.py deleted file mode 100644 index 5fa7f926..00000000 --- a/mlops_stacks/mlops_stacks/training/steps/split.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -This module defines the following routines used by the 'split' step of the regression recipe: - -- ``process_splits``: Defines customizable logic for processing & cleaning the training, validation, - and test datasets produced by the data splitting procedure. -""" - -from pandas import DataFrame - - -def process_splits( - train_df: DataFrame, validation_df: DataFrame, test_df: DataFrame -) -> (DataFrame, DataFrame, DataFrame): - """ - Perform additional processing on the split datasets. - - :param train_df: The training dataset produced by the data splitting procedure. - :param validation_df: The validation dataset produced by the data splitting procedure. - :param test_df: The test dataset produced by the data splitting procedure. - :return: A tuple containing, in order: the processed training dataset, the processed - validation dataset, and the processed test dataset. - """ - - def process(df: DataFrame): - # Drop invalid data points - cleaned = df.dropna() - # Filter out invalid fare amounts and trip distance - cleaned = cleaned[ - (cleaned["fare_amount"] > 0) - & (cleaned["trip_distance"] < 400) - & (cleaned["trip_distance"] > 0) - & (cleaned["fare_amount"] < 1000) - ] - - return cleaned - - return process(train_df), process(validation_df), process(test_df) diff --git a/mlops_stacks/mlops_stacks/training/steps/train.py b/mlops_stacks/mlops_stacks/training/steps/train.py deleted file mode 100644 index c61b2bd8..00000000 --- a/mlops_stacks/mlops_stacks/training/steps/train.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -This module defines the following routines used by the 'train' step of the regression recipe: - -- ``estimator_fn``: Defines the customizable estimator type and parameters that are used - during training to produce a model pipeline. -""" - - -def estimator_fn(): - """ - Returns an *unfitted* estimator that defines ``fit()`` and ``predict()`` methods. - The estimator's input and output signatures should be compatible with scikit-learn - estimators. - """ - from sklearn.linear_model import SGDRegressor - - return SGDRegressor(random_state=42) diff --git a/mlops_stacks/mlops_stacks/training/steps/transform.py b/mlops_stacks/mlops_stacks/training/steps/transform.py deleted file mode 100644 index 7851b899..00000000 --- a/mlops_stacks/mlops_stacks/training/steps/transform.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -This module defines the following routines used by the 'transform' step of the regression recipe: - -- ``transformer_fn``: Defines customizable logic for transforming input data before it is passed - to the estimator during model inference. -""" - -from pandas import DataFrame -from sklearn.compose import ColumnTransformer -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder, StandardScaler, FunctionTransformer - - -def calculate_features(df: DataFrame): - """ - Extend the input dataframe with pickup day of week and hour, and trip duration. - Drop the now-unneeded pickup datetime and dropoff datetime columns. - """ - df["pickup_dow"] = df["tpep_pickup_datetime"].dt.dayofweek - df["pickup_hour"] = df["tpep_pickup_datetime"].dt.hour - trip_duration = df["tpep_dropoff_datetime"] - df["tpep_pickup_datetime"] - df["trip_duration"] = trip_duration.map(lambda x: x.total_seconds() / 60) - df.drop(columns=["tpep_pickup_datetime", "tpep_dropoff_datetime"], inplace=True) - return df - - -def transformer_fn(): - """ - Returns an *unfitted* transformer that defines ``fit()`` and ``transform()`` methods. - The transformer's input and output signatures should be compatible with scikit-learn - transformers. - """ - return Pipeline( - steps=[ - ( - "calculate_time_and_duration_features", - FunctionTransformer(calculate_features, feature_names_out="one-to-one"), - ), - ( - "encoder", - ColumnTransformer( - transformers=[ - ( - "hour_encoder", - OneHotEncoder(categories="auto", sparse=False), - ["pickup_hour"], - ), - ( - "day_encoder", - OneHotEncoder(categories="auto", sparse=False), - ["pickup_dow"], - ), - ( - "std_scaler", - StandardScaler(), - ["trip_distance", "trip_duration"], - ), - ] - ), - ), - ] - ) diff --git a/mlops_stacks/mlops_stacks/validation/notebooks/ModelValidation.py b/mlops_stacks/mlops_stacks/validation/ModelValidation.py similarity index 75% rename from mlops_stacks/mlops_stacks/validation/notebooks/ModelValidation.py rename to mlops_stacks/mlops_stacks/validation/ModelValidation.py index 5e4c9413..ba866d04 100644 --- a/mlops_stacks/mlops_stacks/validation/notebooks/ModelValidation.py +++ b/mlops_stacks/mlops_stacks/validation/ModelValidation.py @@ -3,7 +3,7 @@ # Model Validation Notebook ## # This notebook uses mlflow model validation API to run mode validation after training and registering a model -# in model registry, before deploying it to the "Champion" alias. +# in model registry, before deploying it to the "champion" alias. # # It runs as part of CD and by an automated model training job -> validation -> deployment job defined under ``mlops_stacks/resources/model-workflow-resource.yml`` # @@ -14,18 +14,18 @@ # * `run_mode` - The `run_mode` defines whether model validation is enabled or not. It can be one of the three values: # * `disabled` : Do not run the model validation notebook. # * `dry_run` : Run the model validation notebook. Ignore failed model validation rules and proceed to move -# model to the "Champion" alias. -# * `enabled` : Run the model validation notebook. Move model to the "Champion" alias only if all model validation +# model to the "champion" alias. +# * `enabled` : Run the model validation notebook. Move model to the "champion" alias only if all model validation # rules are passing. -# * enable_baseline_comparison - Whether to load the current registered "Champion" model as baseline. +# * enable_baseline_comparison - Whether to load the current registered "champion" model as baseline. # Baseline model is a requirement for relative change and absolute change validation thresholds. # * validation_input - Validation input. Please refer to data parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate # * model_type - A string describing the model type. The model type can be either "regressor" and "classifier". # Please refer to model_type parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate # * targets - The string name of a column from data that contains evaluation labels. # Please refer to targets parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate -# * custom_metrics_loader_function - Specifies the name of the function in mlops_stacks/validation/validation.py that returns custom metrics. -# * validation_thresholds_loader_function - Specifies the name of the function in mlops_stacks/validation/validation.py that returns model validation thresholds. +# +# Custom metrics, validation thresholds, and evaluator config are defined in validation/validation.py. # # For details on mlflow evaluate API, see doc https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate # For details and examples about performing model validation, see the Model Validation documentation https://mlflow.org/docs/latest/models.html#model-validation @@ -34,32 +34,6 @@ # COMMAND ---------- -# MAGIC %load_ext autoreload -# MAGIC %autoreload 2 - -# COMMAND ---------- - -import os -notebook_path = '/Workspace/' + os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) -%cd $notebook_path - -# COMMAND ---------- - -# MAGIC %pip install -r ../../requirements.txt - -# COMMAND ---------- - -dbutils.library.restartPython() - -# COMMAND ---------- - -import os -notebook_path = '/Workspace/' + os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) -%cd $notebook_path -%cd ../ - -# COMMAND ---------- - dbutils.widgets.text( "experiment_name", "/dev-mlops_stacks-experiment", @@ -68,17 +42,13 @@ dbutils.widgets.dropdown("run_mode", "disabled", ["disabled", "dry_run", "enabled"], "Run Mode") dbutils.widgets.dropdown("enable_baseline_comparison", "false", ["true", "false"], "Enable Baseline Comparison") dbutils.widgets.text("validation_input", "SELECT * FROM delta.`dbfs:/databricks-datasets/nyctaxi-with-zipcodes/subsampled`", "Validation Input") - dbutils.widgets.text("model_type", "regressor", "Model Type") dbutils.widgets.text("targets", "fare_amount", "Targets") -dbutils.widgets.text("custom_metrics_loader_function", "custom_metrics", "Custom Metrics Loader Function") -dbutils.widgets.text("validation_thresholds_loader_function", "validation_thresholds", "Validation Thresholds Loader Function") -dbutils.widgets.text("evaluator_config_loader_function", "evaluator_config", "Evaluator Config Loader Function") -dbutils.widgets.text("model_name", "dev.my-mlops-project.mlops_stacks-model", "Full (Three-Level) Model Name") +dbutils.widgets.text("model_name", "dev.mlops_stacks.mlops_stacks-model", "Full (Three-Level) Model Name") + dbutils.widgets.text("model_version", "", "Candidate Model Version") # COMMAND ---------- - run_mode = dbutils.widgets.get("run_mode").lower() assert run_mode == "disabled" or run_mode == "dry_run" or run_mode == "enabled" @@ -100,20 +70,17 @@ # COMMAND ---------- -import importlib import mlflow import os import tempfile import traceback +from mlflow import MlflowClient -from mlflow.tracking.client import MlflowClient - -client = MlflowClient(registry_uri="databricks-uc") +client = MlflowClient() # set experiment experiment_name = dbutils.widgets.get("experiment_name") mlflow.set_experiment(experiment_name) - # set model evaluation parameters that can be inferred from the job model_uri = dbutils.jobs.taskValues.get("Train", "model_uri", debugValue="") model_name = dbutils.jobs.taskValues.get("Train", "model_name", debugValue="") @@ -124,7 +91,7 @@ model_version = dbutils.widgets.get("model_version") model_uri = "models:/" + model_name + "/" + model_version -baseline_model_uri = "models:/" + model_name + "@Champion" +baseline_model_uri = "models:/" + model_name + "@champion" evaluators = "default" assert model_uri != "", "model_uri notebook parameter must be specified" @@ -135,6 +102,8 @@ # take input enable_baseline_comparison = dbutils.widgets.get("enable_baseline_comparison") + + assert enable_baseline_comparison == "true" or enable_baseline_comparison == "false" enable_baseline_comparison = enable_baseline_comparison == "true" @@ -148,24 +117,10 @@ assert model_type assert targets -custom_metrics_loader_function_name = dbutils.widgets.get("custom_metrics_loader_function") -validation_thresholds_loader_function_name = dbutils.widgets.get("validation_thresholds_loader_function") -evaluator_config_loader_function_name = dbutils.widgets.get("evaluator_config_loader_function") -assert custom_metrics_loader_function_name -assert validation_thresholds_loader_function_name -assert evaluator_config_loader_function_name -custom_metrics_loader_function = getattr( - importlib.import_module("validation"), custom_metrics_loader_function_name -) -validation_thresholds_loader_function = getattr( - importlib.import_module("validation"), validation_thresholds_loader_function_name -) -evaluator_config_loader_function = getattr( - importlib.import_module("validation"), evaluator_config_loader_function_name -) -custom_metrics = custom_metrics_loader_function() -validation_thresholds = validation_thresholds_loader_function() -evaluator_config = evaluator_config_loader_function() +from validation import custom_metrics as _custom_metrics, validation_thresholds as _validation_thresholds, evaluator_config as _evaluator_config +custom_metrics = _custom_metrics() +validation_thresholds = _validation_thresholds() +evaluator_config = _evaluator_config() # COMMAND ---------- @@ -178,7 +133,9 @@ def get_run_link(run_info): def get_training_run(model_name, model_version): version = client.get_model_version(model_name, model_version) - return mlflow.get_run(run_id=version.run_id) + if version.run_id: + return mlflow.get_run(run_id=version.run_id) + return None def generate_run_name(training_run): @@ -206,8 +163,12 @@ def log_to_model_description(run, success): name=model_name, version=model_version, description=description ) + + # COMMAND ---------- + + training_run = get_training_run(model_name, model_version) # run evaluate @@ -228,7 +189,9 @@ def log_to_model_description(run, success): try: eval_result = mlflow.evaluate( + model=model_uri, + data=data, targets=targets, model_type=model_type, @@ -262,11 +225,9 @@ def log_to_model_description(run, success): ) mlflow.log_artifact(metrics_file) log_to_model_description(run, True) - - # Assign "Challenger" alias to indicate model version has passed validation checks - print("Validation checks passed. Assigning 'Challenger' alias to model version.") - client.set_registered_model_alias(model_name, "Challenger", model_version) - + # Assign "challenger" alias to indicate model version has passed validation checks + print("Validation checks passed. Assigning 'challenger' alias to model version.") + client.set_registered_model_alias(model_name, "challenger", model_version) except Exception as err: log_to_model_description(run, False) error_file = os.path.join(tmp_dir, "error.txt") diff --git a/mlops_stacks/mlops_stacks/validation/README.md b/mlops_stacks/mlops_stacks/validation/README.md index 5c06d69b..a8169dd5 100644 --- a/mlops_stacks/mlops_stacks/validation/README.md +++ b/mlops_stacks/mlops_stacks/validation/README.md @@ -1,2 +1,2 @@ # Model Validation -To enable model validation as part of scheduled databricks workflow, please refer to [mlops_stacks/resources/README.md](../resources/README.md) \ No newline at end of file +To enable model validation as part of scheduled databricks workflow, please refer to [mlops_stacks/resources/README.md](../resources/README.md) diff --git a/mlops_stacks/mlops_stacks/validation/validation.py b/mlops_stacks/mlops_stacks/validation/validation.py index ac4f2eac..25f7bc28 100644 --- a/mlops_stacks/mlops_stacks/validation/validation.py +++ b/mlops_stacks/mlops_stacks/validation/validation.py @@ -23,13 +23,13 @@ def squared_diff_plus_one(eval_df, _builtin_metrics): def validation_thresholds(): return { "max_error": MetricThreshold( - threshold=500, higher_is_better=False # max_error should be <= 500 + threshold=500, greater_is_better=False # max_error should be <= 500 ), "mean_squared_error": MetricThreshold( - threshold=20, # mean_squared_error should be <= 20 + threshold=500, # mean_squared_error should be <= 500 # min_absolute_change=0.01, # mean_squared_error should be at least 0.01 greater than baseline model accuracy # min_relative_change=0.01, # mean_squared_error should be at least 1 percent greater than baseline model accuracy - higher_is_better=False, + greater_is_better=False, ), } diff --git a/mlops_stacks/test-requirements.txt b/mlops_stacks/test-requirements.txt index 296b56e0..e160152b 100644 --- a/mlops_stacks/test-requirements.txt +++ b/mlops_stacks/test-requirements.txt @@ -1,2 +1 @@ -pytest>=7.1.2 -pandas==1.5.3 \ No newline at end of file +pytest>=7.1.2 \ No newline at end of file