diff --git a/CHANGELOG.md b/CHANGELOG.md index a7979ae..df95ca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## v1.13.10 + +### Bug Fixes + +- Fixed `on_schema_change: sync_all_columns` and `sync_remove_only` on Delta + incremental models. Removed columns are now dropped with `ALTER TABLE ... DROP COLUMNS` + when the table already uses Delta column mapping, or when the new + `enable_column_mapping_on_drop` model config lets dbt enable it. Otherwise dbt + raises a compile error that explains both options, instead of the misleading + "Delta tables does not support dropping columns" error or a Spark syntax error. + +### Dependencies + +- Consolidated Dependabot dependency bumps: `nx` 23.1.2 → 23.3.0 (drops the transitive + `follow-redirects`) and `handlebars` 4.7.9 → 4.7.10 (#313, #312) + ## v1.13.9 ### Bug Fixes diff --git a/README.md b/README.md index 1af0943..059fd4b 100644 --- a/README.md +++ b/README.md @@ -532,6 +532,30 @@ require Fabric Runtime 1.3 or newer. ) }} ``` +#### Dropping columns with `on_schema_change` + +`on_schema_change: sync_all_columns` and `sync_remove_only` drop target columns that +are no longer in the model. Delta Lake can only drop columns when +[column mapping](https://docs.delta.io/latest/delta-column-mapping.html) is enabled on +the table (`delta.columnMapping.mode` = `name` or `id`): + +- If the table already uses column mapping, dbt drops the columns with `ALTER TABLE ... DROP COLUMNS`. +- If it does not, set `enable_column_mapping_on_drop: true` to let dbt enable it + (`delta.columnMapping.mode = 'name'`) before the drop. This upgrades the table + protocol to at least reader version 2 / writer version 5, so older Delta readers can no + longer read the table; existing higher protocol versions are never downgraded. +- Otherwise dbt raises a compile error that names both options. + +```sql +{{ config( + materialized='incremental', + incremental_strategy='merge', + unique_key='id', + file_format='delta', + on_schema_change='sync_all_columns', + enable_column_mapping_on_drop=true +) }} +``` ### Automatic OPTIMIZE Every write leaves small Parquet files behind inside a Delta table, and small-file diff --git a/package-lock.json b/package-lock.json index a1f3982..d9683d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@types/node": "^20.4.4", "commander": "^11.0.0", "jest": "^30.2.0", - "nx": "23.1.2", + "nx": "23.3.0", "ts-jest": "^29.4.6", "ts-node": "^10.9.2", "tsx": "^4.7.1", @@ -516,6 +516,36 @@ "dev": true, "license": "MIT" }, + "node_modules/@clack/core": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/@clack/core/-/core-1.4.3.tgz", + "integrity": "sha512-/kr3UWNtdJfxZtPgDqUOmG2pvwlmcLGheex5yiZKdwbzZJxhV+HMNR9QNmyY5cGwTNV6LrR7Jtp+KjhUAP1qBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-wrap-ansi": "^0.2.0", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 20.12.0" + } + }, + "node_modules/@clack/prompts": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@clack/prompts/-/prompts-1.7.0.tgz", + "integrity": "sha512-y7/yvZ2TPAnR9+jnc00klvNNLkJiXFFrQA/hlLCcxA9a2A4zQIOimyFQ9XfwYKiGD1fb5GY8vbKIIgO8d5Tb2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@clack/core": "1.4.3", + "fast-string-width": "^3.0.2", + "fast-wrap-ansi": "^0.2.0", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 20.12.0" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1474,9 +1504,9 @@ } }, "node_modules/@nx/nx-darwin-arm64": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-darwin-arm64/-/nx-darwin-arm64-23.1.2.tgz", - "integrity": "sha512-g4Y5okERFcTiFlFS8vkSv65p81XleajClGROLA0CqtIjkwJTInH/pEMfKx9VlnFYIJxk7FIOeFbEpvdqL1VPGg==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-darwin-arm64/-/nx-darwin-arm64-23.3.0.tgz", + "integrity": "sha512-g06RId2Tn99ICwDFisZfDZ2vpM2bXvoCZeNtTWgwLm5mpJ4A7+akI7iXCe50CbzgE4waVRQCOAYpD1zPsKoUVQ==", "cpu": [ "arm64" ], @@ -1488,9 +1518,9 @@ ] }, "node_modules/@nx/nx-darwin-x64": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-darwin-x64/-/nx-darwin-x64-23.1.2.tgz", - "integrity": "sha512-33i1eMuUyXJ2723aQA8mZhuEYoObjVWB8lz56ofXCGDsmiOnKf+s3gPIZi5admD6otLBO+fhnwfPLstHSiqOXw==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-darwin-x64/-/nx-darwin-x64-23.3.0.tgz", + "integrity": "sha512-6t9P5ZQ3LPuCf+HBDMTGtpKHjVvINL9/blpY4PucKjdC+vnNSL7uTZwng1ePenlNF0WFtLHqxUx/H1XgohRq0w==", "cpu": [ "x64" ], @@ -1502,9 +1532,9 @@ ] }, "node_modules/@nx/nx-freebsd-x64": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-freebsd-x64/-/nx-freebsd-x64-23.1.2.tgz", - "integrity": "sha512-7l36xPKFBvcYKbHrafCbJ5ueJfm9Mu0XjjOonPlSc6ys95bum9fUiwar4kqsXGsqgj4iyDMoyQYjTU1wg7TmVA==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-freebsd-x64/-/nx-freebsd-x64-23.3.0.tgz", + "integrity": "sha512-B8Xuh36ZdMJ2SxpFfUITUvCZQTKoGOFHpSactxCx3ymFBUc24Z0B/hQRIYPv43icugcnp71dIwu6IvI66/aqtQ==", "cpu": [ "x64" ], @@ -1516,9 +1546,9 @@ ] }, "node_modules/@nx/nx-linux-arm-gnueabihf": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm-gnueabihf/-/nx-linux-arm-gnueabihf-23.1.2.tgz", - "integrity": "sha512-zMgO2U/MxJEt04ShfqQuTX5k5uNl3pAYTOrRv9wDvfh+iqZb5SkIJdHLDRUKaoI3TFCECg6fIzo60akc5WEdcw==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm-gnueabihf/-/nx-linux-arm-gnueabihf-23.3.0.tgz", + "integrity": "sha512-piMlCEI0tbHt1+QSgL8J1szlqBSGKV27nax8uvhTxleR2Evl3Kxa80BDIEObrUU3lz+AS4oKrcUHUeyJGPdCsQ==", "cpu": [ "arm" ], @@ -1530,9 +1560,9 @@ ] }, "node_modules/@nx/nx-linux-arm64-gnu": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-23.1.2.tgz", - "integrity": "sha512-XN92hnz3cZwx7+ePicGPHshP3h5rloeh57p4A+xHIo3nym500Hsx8iS+jnUmnMWl8xOG8Jj/hlJwRUvKxMCukQ==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-23.3.0.tgz", + "integrity": "sha512-SljVcZYwD4qpwseAUsEQmrXxE0vcPdDMEtvKlpkTProofAMeQFNEh4imw458TylOq7YeFVnNQdZcBnTO+1FeIw==", "cpu": [ "arm64" ], @@ -1547,9 +1577,9 @@ ] }, "node_modules/@nx/nx-linux-arm64-musl": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-musl/-/nx-linux-arm64-musl-23.1.2.tgz", - "integrity": "sha512-ntuKRyPrm/aEOusGsp0eQ3cP+OicBF7/w9G18puFRu0lEwQ2aqBXwXC3GkVUsLjYsOr7SnVmY+MJnVH5NWMQDw==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-arm64-musl/-/nx-linux-arm64-musl-23.3.0.tgz", + "integrity": "sha512-UwzWPtcWxkAE8IOZl4M1eiqsa1yci+zlZ7Vv5ogozhm9jD9EHZVsgeWLrftW22OtU2u+ude/fzBeacuqUDOMtg==", "cpu": [ "arm64" ], @@ -1564,9 +1594,9 @@ ] }, "node_modules/@nx/nx-linux-x64-gnu": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-gnu/-/nx-linux-x64-gnu-23.1.2.tgz", - "integrity": "sha512-e+sLHpmzrvh1qauc9PauV01h2fJlQoJUuaokTZMKTFWaHKwC7sDPYc3X6lEVcO1TZ7lGpIjiiofdu0Fu2PufDw==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-gnu/-/nx-linux-x64-gnu-23.3.0.tgz", + "integrity": "sha512-PJerI1Ag/wkPh8XMBKO4Cx6Ao01QJsSk/ycFNmM7vzW19Db3OvsTSKqiql/z96zINI69MqLDZH2iwHMZXbJupQ==", "cpu": [ "x64" ], @@ -1581,9 +1611,9 @@ ] }, "node_modules/@nx/nx-linux-x64-musl": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-musl/-/nx-linux-x64-musl-23.1.2.tgz", - "integrity": "sha512-9a0SbUhK6/q9ficxtZ40ywqd5u/iggGicmxeMIgW30v7zu+WozrfQK0b+jJa2etCgJCJSwR2eeGQ4vGAIoLclw==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-linux-x64-musl/-/nx-linux-x64-musl-23.3.0.tgz", + "integrity": "sha512-CGete35FhqkdlBZM/Oq6Ddu+yQ7UHMRV/ZYCoKeNU4/YIDIWze8VblF7PGKLU4k5JN4bRY1MmDmryHqVIMXMBA==", "cpu": [ "x64" ], @@ -1598,9 +1628,9 @@ ] }, "node_modules/@nx/nx-win32-arm64-msvc": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-23.1.2.tgz", - "integrity": "sha512-zpDIRceGxkE6ZD12nz6NaMvG54SBSPb7v+SnKuaS6YP8uKtMuR0MsHMrTIYThmmrvtzrZNUmn6EGAlffulr8pg==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-23.3.0.tgz", + "integrity": "sha512-T/ChrK9uzt9/29TQzc8FGkRGDJC130ltM77eKI3Z67OjyD4SKHhsiTu5GameH1BU72jcF4QEKFf74PoI9JzbuA==", "cpu": [ "arm64" ], @@ -1612,9 +1642,9 @@ ] }, "node_modules/@nx/nx-win32-x64-msvc": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/@nx/nx-win32-x64-msvc/-/nx-win32-x64-msvc-23.1.2.tgz", - "integrity": "sha512-7fAteEO6NSWLzrzy+QQoG6n78vKAcbOMgM/f7P1kIXzUWNzPNgCg4Svz2OuxkWEnZs8KGDcqw0YILggdQlqd8g==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/@nx/nx-win32-x64-msvc/-/nx-win32-x64-msvc-23.3.0.tgz", + "integrity": "sha512-ermx8Qy8qf0ZIEDiGPqIXxQ0OPEL8VVUOyvAsxDec8aft3pCFjX8LrFhlfnAfM6MBFs5Bnc9Xl9D8TMzFBgo9w==", "cpu": [ "x64" ], @@ -2191,29 +2221,6 @@ "node": ">=0.4.0" } }, - "node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/ansi-colors": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", - "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -2303,26 +2310,6 @@ "sprintf-js": "~1.0.2" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz", - "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", - "https-proxy-agent": "^5.0.1", - "proxy-from-env": "^2.1.0" - } - }, "node_modules/babel-jest": { "version": "30.3.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.3.0.tgz", @@ -2590,20 +2577,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -2847,19 +2820,6 @@ "dev": true, "license": "MIT" }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", @@ -3005,16 +2965,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -3064,21 +3014,6 @@ "url": "https://dotenvx.com" } }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -3136,19 +3071,6 @@ "once": "^1.4.0" } }, - "node_modules/enquirer": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", - "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-colors": "^4.1.1" - }, - "engines": { - "node": ">=8.6" - } - }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", @@ -3159,55 +3081,6 @@ "is-arrayish": "^0.2.1" } }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/esbuild": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", @@ -3350,6 +3223,33 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.0.tgz", + "integrity": "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -3390,37 +3290,6 @@ "node": ">=8" } }, - "node_modules/flat": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", - "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", - "dev": true, - "license": "BSD-3-Clause", - "bin": { - "flat": "cli.js" - } - }, - "node_modules/follow-redirects": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", - "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/foreground-child": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", @@ -3438,23 +3307,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -3484,16 +3336,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -3514,31 +3356,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -3549,20 +3366,6 @@ "node": ">=8.0.0" } }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -3611,19 +3414,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -3632,13 +3422,13 @@ "license": "ISC" }, "node_modules/handlebars": { - "version": "4.7.9", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", - "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", + "version": "4.7.10", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.10.tgz", + "integrity": "sha512-P5VJMVM7qgBn6vjXMw8WG9uVI+ncf2pi72j4de4yz5ZULLj2RGqLYaKOYGsgyrViQ0tePOVlN1tDCCXXtFqXKg==", "dev": true, "license": "MIT", "dependencies": { - "minimist": "^1.2.5", + "minimist": "^1.2.8", "neo-async": "^2.6.2", "source-map": "^0.6.1", "wordwrap": "^1.0.0" @@ -3663,48 +3453,6 @@ "node": ">=8" } }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -3712,20 +3460,6 @@ "dev": true, "license": "MIT" }, - "node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -3758,9 +3492,9 @@ "license": "BSD-3-Clause" }, "node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", "dev": true, "license": "MIT", "engines": { @@ -4803,16 +4537,6 @@ "tmpl": "1.0.5" } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -4820,29 +4544,6 @@ "dev": true, "license": "MIT" }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -4967,13 +4668,15 @@ } }, "node_modules/nx": { - "version": "23.1.2", - "resolved": "https://registry.npmjs.org/nx/-/nx-23.1.2.tgz", - "integrity": "sha512-Qn0D3QOjBqEVH2abVCjkrs15m3OmsU7gOz7PYMhSbmJECAO1PHSKDyMJTaxIaP10E7R3JEdgehkqIBMPVOIYiw==", + "version": "23.3.0", + "resolved": "https://registry.npmjs.org/nx/-/nx-23.3.0.tgz", + "integrity": "sha512-R7Kl+hURSGiG9uEjc19Cg3o+vGCDHqCVp6mioiBMEsOSol/LgzP+K8TVF06TnnSC8uzxMeB0ymu+LGMJGXnbbw==", "dev": true, "hasInstallScript": true, "license": "MIT", "dependencies": { + "@clack/core": "1.4.3", + "@clack/prompts": "1.7.0", "@emnapi/core": "1.4.5", "@emnapi/runtime": "1.4.5", "@emnapi/wasi-threads": "1.0.4", @@ -4982,20 +4685,15 @@ "@tybys/wasm-util": "0.9.0", "@yarnpkg/lockfile": "1.1.0", "@zkochan/js-yaml": "0.0.7", - "agent-base": "6.0.2", - "ansi-colors": "4.1.3", "ansi-regex": "5.0.1", "ansi-styles": "4.3.0", "argparse": "2.0.1", - "asynckit": "0.4.0", - "axios": "1.18.1", "balanced-match": "4.0.3", "base64-js": "1.5.1", "bl": "4.1.0", - "brace-expansion": "5.0.9", + "brace-expansion": "5.0.12", "buffer": "5.7.1", "bundle-name": "4.1.0", - "call-bind-apply-helpers": "1.0.2", "chalk": "4.1.2", "cli-cursor": "3.1.0", "cli-spinners": "2.6.1", @@ -5003,43 +4701,26 @@ "clone": "1.0.4", "color-convert": "2.0.1", "color-name": "1.1.4", - "combined-stream": "1.0.8", - "debug": "4.4.3", "default-browser": "5.2.1", "default-browser-id": "5.0.0", "defaults": "1.0.4", "define-lazy-prop": "3.0.0", - "delayed-stream": "1.0.0", "dotenv": "16.4.7", "dotenv-expand": "12.0.3", - "dunder-proto": "1.0.1", "ejs": "5.0.1", "emoji-regex": "8.0.0", "end-of-stream": "1.4.5", - "enquirer": "2.3.6", - "es-define-property": "1.0.1", - "es-errors": "1.3.0", - "es-object-atoms": "1.1.1", - "es-set-tostringtag": "2.1.0", "escalade": "3.2.0", "escape-string-regexp": "1.0.5", + "fast-string-truncated-width": "3.0.3", + "fast-string-width": "3.0.2", + "fast-wrap-ansi": "0.2.0", "figures": "3.2.0", - "flat": "5.0.2", - "follow-redirects": "1.16.0", - "form-data": "4.0.6", "fs-constants": "1.0.0", - "function-bind": "1.1.2", "get-caller-file": "2.0.5", - "get-intrinsic": "1.3.0", - "get-proto": "1.0.1", - "gopd": "1.2.0", "has-flag": "4.0.0", - "has-symbols": "1.1.0", - "has-tostringtag": "1.0.2", - "hasown": "2.0.4", - "https-proxy-agent": "5.0.1", "ieee754": "1.2.1", - "ignore": "7.0.5", + "ignore": "7.0.6", "inherits": "2.0.4", "is-docker": "3.0.0", "is-fullwidth-code-point": "3.0.0", @@ -5052,13 +4733,9 @@ "jsonc-parser": "3.3.1", "lines-and-columns": "2.0.3", "log-symbols": "4.1.0", - "math-intrinsics": "1.1.0", - "mime-db": "1.52.0", - "mime-types": "2.1.35", "mimic-fn": "2.1.0", "minimatch": "10.2.5", "minimist": "1.2.8", - "ms": "2.1.3", "npm-run-path": "4.0.1", "once": "1.4.0", "onetime": "5.1.2", @@ -5066,16 +4743,16 @@ "ora": "5.4.1", "path-key": "3.1.1", "picocolors": "1.1.1", - "proxy-from-env": "2.1.0", "readable-stream": "3.6.2", "require-directory": "2.1.1", "resolve.exports": "2.0.3", "restore-cursor": "3.1.0", "run-applescript": "7.0.0", "safe-buffer": "5.2.1", - "semver": "7.8.4", + "semver": "7.8.5", "signal-exit": "3.0.7", - "smol-toml": "1.6.1", + "sisteransi": "1.0.5", + "smol-toml": "1.9.0", "string_decoder": "1.3.0", "string-width": "4.2.3", "strip-ansi": "6.0.1", @@ -5085,6 +4762,7 @@ "tmp": "0.2.7", "tsconfig-paths": "4.2.0", "tslib": "2.8.1", + "undici": "7.29.0", "util-deprecate": "1.0.2", "wcwidth": "1.0.1", "which": "3.0.1", @@ -5100,16 +4778,16 @@ "nx-cloud": "dist/bin/nx-cloud.js" }, "optionalDependencies": { - "@nx/nx-darwin-arm64": "23.1.2", - "@nx/nx-darwin-x64": "23.1.2", - "@nx/nx-freebsd-x64": "23.1.2", - "@nx/nx-linux-arm-gnueabihf": "23.1.2", - "@nx/nx-linux-arm64-gnu": "23.1.2", - "@nx/nx-linux-arm64-musl": "23.1.2", - "@nx/nx-linux-x64-gnu": "23.1.2", - "@nx/nx-linux-x64-musl": "23.1.2", - "@nx/nx-win32-arm64-msvc": "23.1.2", - "@nx/nx-win32-x64-msvc": "23.1.2" + "@nx/nx-darwin-arm64": "23.3.0", + "@nx/nx-darwin-x64": "23.3.0", + "@nx/nx-freebsd-x64": "23.3.0", + "@nx/nx-linux-arm-gnueabihf": "23.3.0", + "@nx/nx-linux-arm64-gnu": "23.3.0", + "@nx/nx-linux-arm64-musl": "23.3.0", + "@nx/nx-linux-x64-gnu": "23.3.0", + "@nx/nx-linux-x64-musl": "23.3.0", + "@nx/nx-win32-arm64-msvc": "23.3.0", + "@nx/nx-win32-x64-msvc": "23.3.0" }, "peerDependencies": { "@swc-node/register": "^1.11.1", @@ -5193,9 +4871,9 @@ } }, "node_modules/nx/node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "dev": true, "license": "MIT", "dependencies": { @@ -5229,9 +4907,9 @@ } }, "node_modules/nx/node_modules/semver": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz", - "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { @@ -5625,16 +5303,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/proxy-from-env": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", - "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, "node_modules/pure-rand": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", @@ -5828,6 +5496,13 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -5839,9 +5514,9 @@ } }, "node_modules/smol-toml": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", - "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz", + "integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -6436,6 +6111,16 @@ "node": ">=0.8.0" } }, + "node_modules/undici": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 273fb6c..2a3ec39 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "@types/node": "^20.4.4", "commander": "^11.0.0", "jest": "^30.2.0", - "nx": "23.1.2", + "nx": "23.3.0", "ts-jest": "^29.4.6", "ts-node": "^10.9.2", "tsx": "^4.7.1", diff --git a/src/dbt/adapters/fabricspark/__version__.py b/src/dbt/adapters/fabricspark/__version__.py index 6efb1fc..9dbac01 100644 --- a/src/dbt/adapters/fabricspark/__version__.py +++ b/src/dbt/adapters/fabricspark/__version__.py @@ -1 +1 @@ -version = "1.13.9" +version = "1.13.10" diff --git a/src/dbt/include/fabricspark/macros/adapters/columns.sql b/src/dbt/include/fabricspark/macros/adapters/columns.sql index f557057..b5cf415 100644 --- a/src/dbt/include/fabricspark/macros/adapters/columns.sql +++ b/src/dbt/include/fabricspark/macros/adapters/columns.sql @@ -25,28 +25,60 @@ {% macro fabricspark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} {% if remove_columns %} - {% if relation.is_delta %} - {{ exceptions.raise_compiler_error('Delta tables does not support dropping columns from tables') }} + {% if relation.is_delta is false %} + {{ exceptions.raise_compiler_error('Dropping columns is only supported on Delta tables; ' ~ relation ~ ' is not a Delta table.') }} {% endif %} + {% do fabricspark__ensure_delta_column_mapping(relation) %} {% endif %} - {% if add_columns is none %} - {% set add_columns = [] %} + {% if add_columns %} + {% set sql -%} + alter {{ relation.type }} {{ relation }} add columns + {% for column in add_columns %} + {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + {% endfor %} + {%- endset -%} + {% do run_query(sql) %} {% endif %} - {% set sql -%} + {% if remove_columns %} + {% set sql -%} + alter {{ relation.type }} {{ relation }} drop columns ( + {% for column in remove_columns %} + {{ adapter.quote(column.name) }}{{ ',' if not loop.last }} + {% endfor %} + ) + {%- endset -%} + {% do run_query(sql) %} + {% endif %} - alter {{ relation.type }} {{ relation }} +{% endmacro %} - {% if add_columns %} add columns {% endif %} - {% for column in add_columns %} - {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} - {% endfor %} - - {%- endset -%} - - {% do run_query(sql) %} +{#- Delta only drops columns when column mapping is on; enabling it upgrades the table protocol, so it is opt-in. -#} +{% macro fabricspark__ensure_delta_column_mapping(relation) %} + {% set properties = {} %} + {% for row in run_query('show tblproperties ' ~ relation) %} + {% do properties.update({row[0]: row[1]}) %} + {% endfor %} + {% if properties.get('delta.columnMapping.mode', 'none') | lower in ['name', 'id'] %} + {% elif config.get('enable_column_mapping_on_drop', false) %} + {% set upgrades = ["'delta.columnMapping.mode' = 'name'"] %} + {% if properties.get('delta.minReaderVersion', '1') | int < 2 %} + {% do upgrades.append("'delta.minReaderVersion' = '2'") %} + {% endif %} + {% if properties.get('delta.minWriterVersion', '2') | int < 5 %} + {% do upgrades.append("'delta.minWriterVersion' = '5'") %} + {% endif %} + {% do run_query('alter table ' ~ relation ~ ' set tblproperties (' ~ upgrades | join(', ') ~ ')') %} + {% else %} + {{ exceptions.raise_compiler_error( + 'Dropping columns from Delta table ' ~ relation ~ ' requires Delta column mapping. ' + ~ "Set enable_column_mapping_on_drop=true on the model to let dbt set delta.columnMapping.mode = 'name' " + ~ '(this upgrades the table protocol to at least reader version 2 / writer version 5, so older Delta readers can no longer read it), ' + ~ "or run: ALTER TABLE " ~ relation ~ " SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name', 'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5')" + ) }} + {% endif %} {% endmacro %} {% macro fabricspark__alter_column_comment(relation, column_dict) %} diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index 49e82f8..ce46131 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -3,7 +3,7 @@ import pytest from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChangeSetup, ) -from dbt.tests.util import run_dbt +from dbt.tests.util import relation_from_name, run_dbt class IncrementalOnSchemaChangeIgnoreFail(BaseIncrementalOnSchemaChangeSetup): @@ -56,22 +56,67 @@ class TestDeltaOnSchemaChange(BaseIncrementalOnSchemaChangeSetup): def run_incremental_sync_all_columns(self, project): select = "model_a incremental_sync_all_columns incremental_sync_all_columns_target" run_dbt(["run", "--models", select, "--full-refresh"]) - # Delta Lake doesn"t support removing columns -- show a nice compilation error results = run_dbt(["run", "--models", select], expect_pass=False) assert "Compilation Error" in results[1].message + assert "enable_column_mapping_on_drop" in results[1].message def run_incremental_sync_remove_only(self, project): select = "model_a incremental_sync_remove_only incremental_sync_remove_only_target" run_dbt(["run", "--models", select, "--full-refresh"]) - # Delta Lake doesn"t support removing columns -- show a nice compilation error results = run_dbt(["run", "--models", select], expect_pass=False) assert "Compilation Error" in results[1].message + assert "enable_column_mapping_on_drop" in results[1].message def test_run_incremental_append_new_columns(self, project): - # only adding new columns in supported self.run_incremental_append_new_columns(project) - # handling columns that have been removed doesn"t work on Delta Lake today def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_all_columns(project) self.run_incremental_sync_remove_only(project) + + +def _column_mapping_mode(project, name): + relation = relation_from_name(project.adapter, name) + rows = project.run_sql(f"show tblproperties {relation}", fetch="all") + return {row[0]: row[1] for row in rows}.get("delta.columnMapping.mode") + + +class TestDeltaOnSchemaChangeEnableColumnMappingOnDrop(BaseIncrementalOnSchemaChangeSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + "+incremental_strategy": "merge", + "+unique_key": "id", + "+enable_column_mapping_on_drop": True, + } + } + + def test_run_incremental_sync_all_columns(self, project): + self.run_incremental_sync_all_columns(project) + assert _column_mapping_mode(project, "incremental_sync_all_columns") == "name" + + def test_run_incremental_sync_remove_only(self, project): + self.run_incremental_sync_remove_only(project) + assert _column_mapping_mode(project, "incremental_sync_remove_only") == "name" + + +class TestDeltaOnSchemaChangeExistingColumnMapping(BaseIncrementalOnSchemaChangeSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + "+incremental_strategy": "merge", + "+unique_key": "id", + "+tblproperties": { + "delta.columnMapping.mode": "name", + "delta.minReaderVersion": "2", + "delta.minWriterVersion": "5", + }, + } + } + + def test_run_incremental_sync_remove_only(self, project): + self.run_incremental_sync_remove_only(project) diff --git a/tests/unit/test_columns_macros.py b/tests/unit/test_columns_macros.py new file mode 100644 index 0000000..380a4e8 --- /dev/null +++ b/tests/unit/test_columns_macros.py @@ -0,0 +1,133 @@ +import unittest +from unittest import mock + +from jinja2 import Environment, FileSystemLoader + + +class CompilerError(Exception): + pass + + +class FakeRelation: + type = "table" + + def __init__(self, is_delta=True): + self.is_delta = is_delta + + def __str__(self): + return "lh.dbo.orders" + + +class Column: + def __init__(self, name, data_type="string"): + self.name = name + self.data_type = data_type + + +class TestAlterRelationAddRemoveColumns(unittest.TestCase): + def setUp(self): + self.queries = [] + self.tblproperties = {} + self.config = {} + + def run_query(sql): + sql = " ".join(str(sql).split()) + self.queries.append(sql) + if sql.startswith("show tblproperties"): + return list(self.tblproperties.items()) + return [] + + def raise_compiler_error(message): + raise CompilerError(message) + + adapter = mock.Mock() + adapter.quote = lambda name: f"`{name}`" + config = mock.Mock() + config.get = lambda key, default=None: self.config.get(key, default) + + env = Environment( + loader=FileSystemLoader("src/dbt/include/fabricspark/macros/adapters"), + extensions=["jinja2.ext.do"], + ) + self.template = env.get_template( + "columns.sql", + globals={ + "adapter": adapter, + "config": config, + "run_query": run_query, + "exceptions": mock.Mock(raise_compiler_error=raise_compiler_error), + "return": lambda value: value, + }, + ) + + def _alter(self, add_columns=None, remove_columns=None, relation=None): + self.template.module.fabricspark__alter_relation_add_remove_columns( + relation or FakeRelation(), add_columns, remove_columns + ) + return [q for q in self.queries if not q.startswith("show tblproperties")] + + def test_add_only_emits_add_columns(self): + self.assertEqual( + self._alter(add_columns=[Column("a"), Column("b", "int")]), + ["alter table lh.dbo.orders add columns a string, b int"], + ) + + def test_nothing_to_do_emits_no_statement(self): + self.assertEqual(self._alter(add_columns=[], remove_columns=[]), []) + self.assertEqual(self._alter(add_columns=None, remove_columns=None), []) + + def test_remove_without_column_mapping_or_opt_in_raises(self): + with self.assertRaises(CompilerError) as ctx: + self._alter(remove_columns=[Column("field2")]) + self.assertIn("enable_column_mapping_on_drop", str(ctx.exception)) + self.assertIn("lh.dbo.orders", str(ctx.exception)) + self.assertEqual(self.queries, ["show tblproperties lh.dbo.orders"]) + + def test_remove_with_opt_in_enables_column_mapping_then_drops(self): + self.config["enable_column_mapping_on_drop"] = True + self.tblproperties = {"delta.minReaderVersion": "1", "delta.minWriterVersion": "2"} + self.assertEqual( + self._alter(add_columns=[Column("field5")], remove_columns=[Column("field2")]), + [ + "alter table lh.dbo.orders set tblproperties ('delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5')", + "alter table lh.dbo.orders add columns field5 string", + "alter table lh.dbo.orders drop columns ( `field2` )", + ], + ) + + def test_opt_in_never_downgrades_the_protocol(self): + self.config["enable_column_mapping_on_drop"] = True + self.tblproperties = {"delta.minReaderVersion": "3", "delta.minWriterVersion": "7"} + self.assertEqual( + self._alter(remove_columns=[Column("field2"), Column("field3")]), + [ + "alter table lh.dbo.orders set tblproperties ('delta.columnMapping.mode' = 'name')", + "alter table lh.dbo.orders drop columns ( `field2`, `field3` )", + ], + ) + + def test_existing_column_mapping_drops_without_opt_in(self): + for mode in ("name", "id", "NAME"): + with self.subTest(mode=mode): + self.queries.clear() + self.tblproperties = {"delta.columnMapping.mode": mode} + self.assertEqual( + self._alter(remove_columns=[Column("field2")]), + ["alter table lh.dbo.orders drop columns ( `field2` )"], + ) + + def test_column_mapping_mode_none_still_requires_opt_in(self): + self.tblproperties = {"delta.columnMapping.mode": "none"} + with self.assertRaises(CompilerError): + self._alter(remove_columns=[Column("field2")]) + + def test_remove_on_non_delta_relation_raises(self): + with self.assertRaises(CompilerError) as ctx: + self._alter(remove_columns=[Column("field2")], relation=FakeRelation(is_delta=False)) + self.assertIn("only supported on Delta tables", str(ctx.exception)) + self.assertEqual(self.queries, []) + + +if __name__ == "__main__": + unittest.main()