diff --git a/.gitignore b/.gitignore index 03c790e..da64b5a 100644 --- a/.gitignore +++ b/.gitignore @@ -100,6 +100,7 @@ web_modules/ .yarn-integrity # dotenv environment variable files +.venv/ .env .env.* !.env.example @@ -172,3 +173,11 @@ dist vite.config.js.timestamp-* vite.config.ts.timestamp-* +# dbt build & package artifacts +task-2/target/ +task-2/dbt_packages/ +task-2/logs/ + +# Local configuration & credentials +task-2/profiles.yml +task-2/.user.yml \ No newline at end of file diff --git a/AI_ASSIST.md b/AI_ASSIST.md index b134b96..e8b41d2 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -4,13 +4,13 @@ Record at least one point where you used an AI coding assistant (ChatGPT, Claude ## Interaction 1 -- **Tool used:** (e.g. ChatGPT / Cursor / Claude) -- **Task / Problem:** (e.g. debugging dbt connection profile / writing PySpark join / configuring Job trigger) +- **Tool used:** Gemini +- **Task / Problem:** Fixing errors when running dbt build for the fct_trips model on Databricks. - **Prompt sent:** - > `___` + > [Pasted the dbt build error log output] - **Output provided by AI:** - > `___` + > Explained that two things went wrong: the dbt_utils package was missing, and a table named raw_zones couldn't be found. Suggested running dbt deps first, then using dbt run instead of dbt build to skip test errors and build the model directly. - **What I kept, changed, or rejected, and why:** - > `___` + > I kept the advice to run dbt deps to install missing packages, which fixed the main package error. I also kept using dbt run instead of dbt build so I could build my model without getting stuck on tests for unrelated tables. *(Ensure no personal passwords, Databricks tokens, or unapproved credentials are included in prompts or logged outputs.)* diff --git a/task-1/README.md b/task-1/README.md index 569d9a9..6b9dfe1 100644 --- a/task-1/README.md +++ b/task-1/README.md @@ -2,6 +2,8 @@ Add your Databricks notebook here. +See my pyspark_exploration file [here](c55-data-week-13\task-1\pyspark_exploration.ipynb) + **Filename:** `pyspark_exploration.ipynb` (or export as `pyspark_exploration.py` from Databricks). **Must include:** diff --git a/task-1/pyspark_exploration.ipynb b/task-1/pyspark_exploration.ipynb new file mode 100644 index 0000000..c34ad84 --- /dev/null +++ b/task-1/pyspark_exploration.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1785332881078, + "inputWidgets": {}, + "nuid": "555284dc-e75d-46cb-8999-4d4e691c94a7", + "showTitle": false, + "startTime": 1785332875660, + "submitTime": 1785332616699, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "from pyspark.sql import functions as F" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1785333009408, + "inputWidgets": {}, + "nuid": "87eaedc5-8e6e-4c61-bc60-e208d1f414d8", + "showTitle": false, + "startTime": 1785333008357, + "submitTime": 1785333008321, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "trips_data = spark.read.table(\"hyf.nyc_yellow.raw_trips\")\n", + "zones_data = spark.read.table(\"hyf.nyc_yellow.raw_zones\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1785333012912, + "inputWidgets": {}, + "nuid": "53d9f31e-e20f-4e2d-85a8-385c6a91725f", + "showTitle": false, + "startTime": 1785333012752, + "submitTime": 1785333012712, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "output_type": "stream", + "text": [ + "root\n |-- vendor_id: long (nullable = true)\n |-- pickup_datetime: timestamp_ntz (nullable = true)\n |-- dropoff_datetime: timestamp_ntz (nullable = true)\n |-- passenger_count: long (nullable = true)\n |-- trip_distance: double (nullable = true)\n |-- rate_code_id: long (nullable = true)\n |-- store_and_fwd_flag: string (nullable = true)\n |-- pickup_location_id: long (nullable = true)\n |-- dropoff_location_id: long (nullable = true)\n |-- payment_type: long (nullable = true)\n |-- fare_amount: double (nullable = true)\n |-- extra: double (nullable = true)\n |-- mta_tax: double (nullable = true)\n |-- tip_amount: double (nullable = true)\n |-- tolls_amount: double (nullable = true)\n |-- improvement_surcharge: double (nullable = true)\n |-- total_amount: double (nullable = true)\n |-- congestion_surcharge: double (nullable = true)\n |-- airport_fee: double (nullable = true)\n\nroot\n |-- location_id: integer (nullable = true)\n |-- borough: string (nullable = true)\n |-- zone: string (nullable = true)\n |-- service_zone: string (nullable = true)\n\n" + ] + } + ], + "source": [ + "trips_data.printSchema()\n", + "zones_data.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a40b7610-f9d9-4e1e-ae4d-7ae5fe5cb206", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "Which pickup borough has the most trips?" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1785333179473, + "inputWidgets": {}, + "nuid": "e8cb404e-4131-4cce-a1b3-5797fd2fa358", + "showTitle": false, + "startTime": 1785333122255, + "submitTime": 1785333122205, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------+---------+\n| borough| count|\n+-------------+---------+\n| Manhattan|112028489|\n| Queens| 12292035|\n| Brooklyn| 2648890|\n| Bronx| 570457|\n| Unknown| 559802|\n| N/A| 76659|\n| EWR| 17103|\n|Staten Island| 9113|\n+-------------+---------+\n\n" + ] + } + ], + "source": [ + "pickup_borough=(\n", + " trips_data.join(zones_data, trips_data.pickup_location_id == zones_data.location_id)\n", + " .groupBy(\"borough\")\n", + " .count()\n", + " .orderBy(F.desc(\"count\"))\n", + ")\n", + "pickup_borough.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "dda809a0-32b9-454d-af76-19b0f8845eb3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "An aggregation of average `total_amount` per `payment_type`." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1785333596549, + "inputWidgets": {}, + "nuid": "e93a5ac0-d976-4e12-8082-68ee5dacaa77", + "showTitle": false, + "startTime": 1785333586469, + "submitTime": 1785333586311, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------+------------------+\n|payment_type| avg_payment|\n+------------+------------------+\n| 1|29.999458495737827|\n| 2| 23.7469706922074|\n| 0|23.485032800615247|\n| 5|14.887777777777778|\n| 3| 9.02871399651705|\n| 4| 2.16139900667104|\n+------------+------------------+\n\n" + ] + } + ], + "source": [ + "avg_payment_per_type=(\n", + " trips_data.groupBy(\"payment_type\")\n", + " .agg(F.avg(\"total_amount\").alias(\"avg_payment\"))\n", + " .orderBy(F.desc(\"avg_payment\"))\n", + "\n", + "\n", + ")\n", + "\n", + "avg_payment_per_type.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a6f1789a-cc80-4f33-9dcf-8016d35d9b26", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "when you would choose PySpark versus dbt SQL?\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9cf458f1-b177-4c7c-9afe-228d9a7a45d3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "- Choose PySpark to clean raw or live streaming data using Python before it hits the database. \n", + "- Choose dbt SQL to organize and test that data inside the database so people can use it for reports" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "pyspark_exploration", + "widgets": {} + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/task-2/WRITEUP.md b/task-2/WRITEUP.md index f3d1eb3..26dd9b9 100644 --- a/task-2/WRITEUP.md +++ b/task-2/WRITEUP.md @@ -4,21 +4,21 @@ Fill in after running `dbt build --select fct_trips --full-refresh` baseline fol ## First build (full / initial load with --full-refresh) -- **Wall-clock time:** -- **Notes:** (optional: warehouse size, any errors you fixed) +- **Wall-clock time:** 155.17 seconds (2m 35s) +- **Notes:** Installed missing `dbt_utils` dependency and updated source schema path ## Second build (incremental rerun) -- **Wall-clock time:** +- **Wall-clock time:** 141.41 seconds (2m 21s) ## Why was the second run faster? Write two or three sentences in your own words (see the assignment for the concepts you must name): -`___` +The second run was faster because dbt utilized incremental materialization with the `is_incremental()` macro, fetching only new or updated records instead of reprocessing the entire dataset from scratch. Rather than executing a full `CREATE OR REPLACE TABLE` operation, Delta Lake executed an efficient `MERGE` operation to update and insert only the delta rows into the existing table. ## Delta Table History (DESCRIBE HISTORY) Paste the output or summary of `DESCRIBE HISTORY hyf.dev_yourname.fct_trips` (showing `CREATE OR REPLACE TABLE` and `MERGE` operations) or reference a screenshot: -`___` +![alt text](image.png) diff --git a/task-2/dbt_project.yml b/task-2/dbt_project.yml new file mode 100644 index 0000000..d6cba5a --- /dev/null +++ b/task-2/dbt_project.yml @@ -0,0 +1,25 @@ +name: 'nyc_taxi' +version: '1.0.0' + + +# This project connects to the profile of the same name in profiles.yml. +profile: 'nyc_taxi' + +model-paths: ["models"] +macro-paths: ["macros"] +test-paths: ["tests"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +# Folder-level materialization defaults. Staging models stay as views (cheap, +# always fresh); the mart is built as a table (queried repeatedly by the +# dashboard). You can override per model with {{ config(materialized='...') }}. +models: + nyc_taxi_borough_daily: + staging: + +materialized: view + marts: + +materialized: table diff --git a/task-2/image.png b/task-2/image.png new file mode 100644 index 0000000..e48b4c0 Binary files /dev/null and b/task-2/image.png differ diff --git a/task-2/models/marts/_fct_trips.yml b/task-2/models/marts/_fct_trips.yml new file mode 100644 index 0000000..4f82008 --- /dev/null +++ b/task-2/models/marts/_fct_trips.yml @@ -0,0 +1,53 @@ +version: 2 + +models: + - name: fct_trips + description: | + One row per completed NYC yellow taxi trip (2023-2025), with + pickup/dropoff zone attributes folded in (OBT-style mart). Queried + directly by dashboards and ad-hoc analysis. + + **Grain:** one row per trip (`trip_id` surrogate key). + **Source:** `hyf.nyc_yellow.raw_trips` joined to `raw_zones` on + `pickup_location_id` and `dropoff_location_id`. + **Not included:** trips where `pickup_location_id` is NULL (dropped + in `stg_trips`); duplicate rows from the TLC source are kept as-is + and surfaced by `dbt_utils.unique_combination_of_columns`. + columns: + - name: trip_id + description: Surrogate key generated in `stg_trips` for incremental merge. + tests: [not_null, unique] + - name: pickup_datetime + description: Wall-clock time the trip began (America/New_York, no timezone attached). + tests: [not_null] + - name: dropoff_datetime + description: Wall-clock time the trip ended. + - name: fare_amount + description: Metered fare in USD, not including tip, tolls, or surcharges. + - name: tip_amount + description: Tip in USD. Non-zero only when payment_type is credit card (1). + - name: trip_distance + description: Distance in miles as reported by the taximeter. + - name: tip_pct + description: | + `tip_amount / fare_amount`, rounded to 4 decimals. NULL when + `fare_amount` is 0 (voided trips, no-charge rides). + - name: fare_per_mile + description: | + `fare_amount / trip_distance`, rounded to 4 decimals. NULL when + `trip_distance` is 0 (data-quality anomalies). + - name: payment_type_label + description: | + Human-readable payment method from the TLC code. See the jinja + dictionary in `stg_trips.sql` for the 1-6 → label mapping. + - name: pickup_borough + description: | + NYC borough of the pickup zone, joined from `stg_zones.borough`. + Values: Manhattan, Brooklyn, Queens, Bronx, Staten Island, EWR, + Unknown, NaN, or NULL when `pickup_location_id` did not resolve. + - name: pickup_zone + description: Human-readable pickup-zone name from `stg_zones.zone`. + - name: dropoff_borough + description: NYC borough of the dropoff zone. + - name: dropoff_zone + description: Human-readable dropoff-zone name. diff --git a/task-2/models/marts/fct_trips.sql b/task-2/models/marts/fct_trips.sql new file mode 100644 index 0000000..5beceae --- /dev/null +++ b/task-2/models/marts/fct_trips.sql @@ -0,0 +1,31 @@ +{{ config( + materialized='incremental', + incremental_strategy='merge', + unique_key='trip_id' +) }} + +select + t.trip_id, + t.pickup_datetime, + t.dropoff_datetime, + t.fare_amount, + t.tip_amount, + t.trip_distance, + t.trip_duration_minutes, + t.tip_pct, + t.fare_per_mile, + t.payment_type_label, + pz.borough as pickup_borough, + pz.zone as pickup_zone, + dz.borough as dropoff_borough, + dz.zone as dropoff_zone +from {{ ref('stg_trips') }} t +left join {{ ref('stg_zones') }} pz + on t.pickup_location_id = pz.location_id +left join {{ ref('stg_zones') }} dz + on t.dropoff_location_id = dz.location_id + + +{% if is_incremental() %} + where pickup_datetime > (select max(pickup_datetime) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/task-2/models/marts/fct_trips_docs.md b/task-2/models/marts/fct_trips_docs.md new file mode 100644 index 0000000..2c1db8d --- /dev/null +++ b/task-2/models/marts/fct_trips_docs.md @@ -0,0 +1,10 @@ +{% docs trip_grain %} + +One row per completed taxi trip. "Completed" means the TLC submitted the +trip record to the public dataset; cancellations and trips in progress +are not included. Duplicates exist in the source data (roughly 4 rows in +January 2024 where every column is identical) and are kept as-is; see +the `dbt_utils.unique_combination_of_columns` test results for the +current count. + +{% enddocs %} diff --git a/task-2/models/staging/_sources.yml b/task-2/models/staging/_sources.yml new file mode 100644 index 0000000..89bc789 --- /dev/null +++ b/task-2/models/staging/_sources.yml @@ -0,0 +1,11 @@ +version: 2 + +sources: + - name: nyc_taxi + description: Raw NYC yellow taxi trip records and zone lookup. + schema: nyc_yellow + tables: + - name: raw_trips + description: One row per yellow taxi trip. + - name: raw_zones + description: NYC taxi zone lookup. diff --git a/task-2/models/staging/_stg_trips.yml b/task-2/models/staging/_stg_trips.yml new file mode 100644 index 0000000..f461212 --- /dev/null +++ b/task-2/models/staging/_stg_trips.yml @@ -0,0 +1,52 @@ +version: 2 + +models: + - name: stg_trips + description: Cleaned yellow taxi trips, one row per trip. + tests: + # Chapter 5 teaches this test at the default `error` severity to demonstrate + # how `dbt build` skips downstream models on a test failure. The January 2024 + # raw_trips data contains 4 genuine duplicate rows (a TLC source-data issue), + # so the test always fails. In this reference repo we soften it to `warn` to + # keep CI green while still surfacing the count. When students follow the + # chapter on their own machine, they should leave it at the default. + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [pickup_datetime, dropoff_datetime, pickup_location_id, fare_amount] + config: + severity: warn + columns: + - name: pickup_datetime + description: When the trip started. + tests: + - not_null + - name: pickup_location_id + description: TLC zone ID where the trip started. + tests: + - not_null + - relationships: + to: ref('stg_zones') + field: location_id + config: + severity: warn + - name: payment_type + description: TLC payment code. Yellow 2023-2025 adds code 0 (Flex Fare / unknown) on top of the classic 1-6. + tests: + - not_null: + severity: warn + - accepted_values: + values: [0, 1, 2, 3, 4, 5, 6] + +unit_tests: + - name: payment_type_label_maps_known_codes + model: stg_trips + given: + - input: source('nyc_taxi', 'raw_trips') + rows: + - {payment_type: 1, pickup_datetime: '2024-01-01 08:00:00', pickup_location_id: 100, fare_amount: 10.0, tip_amount: 2.0, trip_distance: 2.0} + - {payment_type: 2, pickup_datetime: '2024-01-01 09:00:00', pickup_location_id: 100, fare_amount: 10.0, tip_amount: 0.0, trip_distance: 2.0} + - {payment_type: 6, pickup_datetime: '2024-01-01 10:00:00', pickup_location_id: 100, fare_amount: 10.0, tip_amount: 0.0, trip_distance: 2.0} + expect: + rows: + - {payment_type: 1, payment_type_label: 'Credit card'} + - {payment_type: 2, payment_type_label: 'Cash'} + - {payment_type: 6, payment_type_label: 'Voided trip'} diff --git a/task-2/models/staging/_stg_zones.yml b/task-2/models/staging/_stg_zones.yml new file mode 100644 index 0000000..14f8227 --- /dev/null +++ b/task-2/models/staging/_stg_zones.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: stg_zones + description: One row per TLC taxi zone (265 zones total). + columns: + - name: location_id + description: TLC zone ID. + tests: + - unique + - not_null + - name: borough + description: NYC borough (Manhattan, Brooklyn, Queens, Bronx, Staten Island, EWR, Unknown). diff --git a/task-2/models/staging/stg_trips.sql b/task-2/models/staging/stg_trips.sql new file mode 100644 index 0000000..b7341f7 --- /dev/null +++ b/task-2/models/staging/stg_trips.sql @@ -0,0 +1,49 @@ +{{ config(materialized='view') }} + +{% set payment_types = { + 1: 'Credit card', + 2: 'Cash', + 3: 'No charge', + 4: 'Dispute', + 5: 'Unknown', + 6: 'Voided trip' +} %} + +select + {{ dbt_utils.generate_surrogate_key([ + 'vendor_id', 'pickup_datetime', 'dropoff_datetime', 'pickup_location_id', + 'dropoff_location_id', 'fare_amount', 'trip_distance', 'total_amount', 'passenger_count' + ]) }} as trip_id, + pickup_datetime, + dropoff_datetime, + pickup_location_id, + dropoff_location_id, + fare_amount, + tip_amount, + trip_distance, + payment_type, + case + when fare_amount > 0 then round(tip_amount / fare_amount, 4) + else null + end as tip_pct, + case + when trip_distance > 0 then round(fare_amount / trip_distance, 4) + else null + end as fare_per_mile, + case payment_type + {% for code, label in payment_types.items() %} + when {{ code }} then '{{ label }}' + {% endfor %} + else 'Other' + end as payment_type_label, + round((unix_timestamp(dropoff_datetime) - unix_timestamp(pickup_datetime)) / 60.0, 2) as trip_duration_minutes +from {{ source('nyc_taxi', 'raw_trips') }} +where pickup_location_id is not null + and fare_amount >= 0 +qualify row_number() over ( + partition by {{ dbt_utils.generate_surrogate_key([ + 'vendor_id', 'pickup_datetime', 'dropoff_datetime', 'pickup_location_id', + 'dropoff_location_id', 'fare_amount', 'trip_distance', 'total_amount', 'passenger_count' + ]) }} + order by pickup_datetime +) = 1 diff --git a/task-2/models/staging/stg_zones.sql b/task-2/models/staging/stg_zones.sql new file mode 100644 index 0000000..1e98897 --- /dev/null +++ b/task-2/models/staging/stg_zones.sql @@ -0,0 +1,6 @@ +select + location_id, + borough, + zone, + service_zone +from {{ source('nyc_taxi', 'raw_zones') }} diff --git a/task-2/package-lock.yml b/task-2/package-lock.yml new file mode 100644 index 0000000..3b1f04e --- /dev/null +++ b/task-2/package-lock.yml @@ -0,0 +1,5 @@ +packages: + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 diff --git a/task-2/packages.yml b/task-2/packages.yml new file mode 100644 index 0000000..39f82d4 --- /dev/null +++ b/task-2/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/task-3/SCHEDULING.md b/task-3/SCHEDULING.md index 2780686..172bc5e 100644 --- a/task-3/SCHEDULING.md +++ b/task-3/SCHEDULING.md @@ -4,20 +4,22 @@ Paste the URL of your successful Job run from the Databricks UI address bar: -`___` +https://adb-7405619530719547.7.azuredatabricks.net/jobs/34361952891076/runs/436009695970757?o=7405619530719547 ## Screenshots Ensure the following screenshot files exist in `task-3/screenshots/`: -1. `job_config.png` — Showing the dbt task configuration with Git repository URL, branch `main`, path `task-2`, and warehouse `hyf-dbt-warehouse`. +1. `job_config.png` — Showing the dbt task configuration with Git repository URL, branch `main`, path `task-2`, and warehouse `hyf-dbt-warehouse`. ![You can see the image from here](task-3\screenshots\job_config.png) 2. `job_run_success.png` — Showing a successful run log with a green checkmark and stdout execution output. -3. `job_schedule_paused.png` — Showing the scheduled trigger set to **Paused**. - +![You can see the image from here](task-3\screenshots\job_run_success.png) +3. `job_schedule_paused.png` — Showing the scheduled trigger set to **Paused**. ![You can see the image from here](c55-data-week-13\task-3\screenshots\job_schedule_paused.png) ## Orchestration Comparison ### When would you choose Databricks Jobs versus Apache Airflow for pipeline orchestration? Write two to three sentences comparing Databricks Jobs and Apache Airflow in your own words: +I choose Databricks Jobs if all my tasks run entirely within Databricks and I want a simple, ready-to-use tool without managing extra setup. + +I choose Apache Airflow if I need to orchestrate complex workflows that connect multiple different systems and tools outside of Databricks. -`___` diff --git a/task-3/screenshots/job_config.png b/task-3/screenshots/job_config.png new file mode 100644 index 0000000..5af857a Binary files /dev/null and b/task-3/screenshots/job_config.png differ diff --git a/task-3/screenshots/job_run_success.png b/task-3/screenshots/job_run_success.png new file mode 100644 index 0000000..22ed0ce Binary files /dev/null and b/task-3/screenshots/job_run_success.png differ diff --git a/task-3/screenshots/job_schedule_paused.png b/task-3/screenshots/job_schedule_paused.png new file mode 100644 index 0000000..85b53e3 Binary files /dev/null and b/task-3/screenshots/job_schedule_paused.png differ