mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Give Dag bundles their own read permission (#76329)
A bundle is deployed as a whole, so its pages should show every file and the full counts rather than whatever slice of Dags the caller can read. Filtering by readable Dags made the counts differ between users and hid a bundle whose only file failed before defining a Dag, which is the case the page most needs to show. Since that exposes every deployed file, the bundle pages need operator-level access instead of Dag read.
This commit is contained in:
1 parent
6f7cb34ea4
commit
5040171c3e
49 files changed
+1184
-1000
No files matched your search
@@ -238,14 +238,16 @@ when a Dag processor last refreshed it, its import-error count, and whether it i
|
||||
refreshes itself while open, and the same data is available at ``GET /api/v2/dagBundles`` for a
|
||||
deployment pipeline to poll.
|
||||
|
||||
.. note::
|
||||
Bundles have their own permission rather than following Dag access, so a user who holds it sees
|
||||
every file and count in a bundle, whatever Dags they can read. By default it is held by:
|
||||
|
||||
Under the FAB auth manager the *Browse* entry needs a ``apache-airflow-providers-fab`` release
|
||||
that maps the new menu item onto the ``DAGs`` resource; upgrade to the latest one if the entry is
|
||||
missing. On an older release the lookup falls back to a FAB resource named after the menu item,
|
||||
no such resource exists, and the entry is hidden for every role including ``Admin``. The page is
|
||||
still reachable at ``/dag_bundles`` and the API is unaffected, so a missing *Browse* entry means
|
||||
the provider needs upgrading, not that the feature is off.
|
||||
* Simple auth manager: ``op`` and ``admin``.
|
||||
* FAB auth manager: ``Op`` and ``Admin``. This needs an ``apache-airflow-providers-fab`` release with
|
||||
the ``Dag Bundles`` resource; on an older release the page and the API are denied to every role,
|
||||
including ``Admin``.
|
||||
* Keycloak auth manager: every role, ``Viewer`` included. In multi-team mode, ``SuperAdmin`` only.
|
||||
This needs an ``apache-airflow-providers-keycloak`` release that knows the view; on an older
|
||||
release every team role can read it.
|
||||
|
||||
``version`` is whatever the bundle reports; for a Git Dag bundle it is the commit SHA of the tracking
|
||||
ref. Bundles that do not support versioning, such as ``LocalDagBundle``, report ``null``.
|
||||
@@ -263,17 +265,12 @@ Some caveats:
|
||||
Dags are parsed.
|
||||
* The version and the import-error count together cannot confirm that a particular commit parsed,
|
||||
so do not treat the pair as a deployment-health check. Import errors are not recorded against a
|
||||
version, a file whose parse exceeds
|
||||
version, and a file whose parse exceeds
|
||||
:ref:`config:dag_processor__dag_file_processor_timeout` is killed without writing an import-error
|
||||
row at all, and the count only covers errors you are allowed to read. A count of zero therefore
|
||||
means "nothing I can show you", not "this commit is healthy". To tie a running Dag to a commit,
|
||||
read the Dag's version in ``/api/v2/dags/{dag_id}/dagVersions``.
|
||||
row at all. A count of zero therefore does not mean "this commit is healthy". To tie a running Dag
|
||||
to a commit, read the Dag's version in ``/api/v2/dags/{dag_id}/dagVersions``.
|
||||
* With several Dag processors the row reflects whichever refreshed last, as they share one row per
|
||||
bundle.
|
||||
* A bundle is listed for users who can read at least one Dag recorded against it. A bundle holding
|
||||
no registered Dag at all -- a first deploy whose only file fails to import, for instance -- has no
|
||||
Dag to authorize against, so it is listed only for users who may read import errors for files with
|
||||
no registered Dag (``Admin`` by default under the FAB auth manager).
|
||||
|
||||
``active`` is false when the bundle was missing from the configuration of whatever last ran a full
|
||||
sync, which also means processors that disagree about ``dag_bundle_config_list`` will deactivate
|
||||
|
||||
@@ -204,16 +204,16 @@ source code so it stays up to date as endpoints are added or changed.
|
||||
- ``PUT``
|
||||
* - ``GET``
|
||||
- ``/api/v2/dagBundles``
|
||||
- ``DAG``
|
||||
- ``GET``
|
||||
- ``View.DAG_BUNDLES``
|
||||
- ``DAG_BUNDLES``
|
||||
* - ``GET``
|
||||
- ``/api/v2/dagBundles/{bundle_name}``
|
||||
- ``DAG``
|
||||
- ``GET``
|
||||
- ``View.DAG_BUNDLES``
|
||||
- ``DAG_BUNDLES``
|
||||
* - ``GET``
|
||||
- ``/api/v2/dagBundles/{bundle_name}/files``
|
||||
- ``DAG``
|
||||
- ``GET``
|
||||
- ``View.DAG_BUNDLES``
|
||||
- ``DAG_BUNDLES``
|
||||
* - ``GET``
|
||||
- ``/api/v2/dagSources/{dag_id}``
|
||||
- ``DAG.CODE``
|
||||
|
||||
@@ -105,6 +105,9 @@ class AccessView(Enum):
|
||||
# view rather than riding on Dag-level ``DagAccessEntity.AUDIT_LOG`` access.
|
||||
AUDIT_LOGS_ALL = "AUDIT_LOGS_ALL"
|
||||
CLUSTER_ACTIVITY = "CLUSTER_ACTIVITY"
|
||||
# Dag bundles, the files in them and their parse state. A bundle is deployed as a whole, so it
|
||||
# is authorized on its own rather than through the Dags it happens to hold.
|
||||
DAG_BUNDLES = "DAG_BUNDLES"
|
||||
DOCS = "DOCS"
|
||||
IMPORT_ERRORS = "IMPORT_ERRORS"
|
||||
# Import errors for files with no registered Dag: there is no per-Dag key to
|
||||
|
||||
@@ -386,13 +386,15 @@ class SimpleAuthManager(BaseAuthManager[SimpleAuthManagerUser]):
|
||||
) -> bool:
|
||||
# Views covering records that have no per-Dag key to authorize on are admin-only --
|
||||
# import errors for files with no registered Dag, audit log rows not tied to a Dag, and
|
||||
# reparsing a file with no registered Dag. Every other view stays readable by viewers.
|
||||
allow_role = (
|
||||
SimpleAuthManagerRole.ADMIN
|
||||
if access_view
|
||||
in (AccessView.IMPORT_ERRORS_ALL, AccessView.AUDIT_LOGS_ALL, AccessView.REPARSE_ALL)
|
||||
else SimpleAuthManagerRole.VIEWER
|
||||
)
|
||||
# reparsing a file with no registered Dag. Dag bundles show every file deployed, whatever
|
||||
# Dags the caller may read, so they are left to operators. Every other view stays
|
||||
# readable by viewers.
|
||||
if access_view in (AccessView.IMPORT_ERRORS_ALL, AccessView.AUDIT_LOGS_ALL, AccessView.REPARSE_ALL):
|
||||
allow_role = SimpleAuthManagerRole.ADMIN
|
||||
elif access_view is AccessView.DAG_BUNDLES:
|
||||
allow_role = SimpleAuthManagerRole.OP
|
||||
else:
|
||||
allow_role = SimpleAuthManagerRole.VIEWER
|
||||
return self._is_authorized(method="GET", allow_role=allow_role, user=user, team_name=team_name)
|
||||
|
||||
def is_authorized_custom_view(
|
||||
|
||||
@@ -45,16 +45,16 @@ class DagBundleResponse(BaseModel):
|
||||
)
|
||||
bundle_url: str | None = Field(
|
||||
description=(
|
||||
"A link to view the bundle at ``version``, when one is configured and the caller may "
|
||||
"read Dag versions."
|
||||
"A link to view the bundle at ``version``. Null when the bundle has no URL template, when "
|
||||
"the template needs a version and the bundle has not been refreshed yet, or when the "
|
||||
"template cannot be rendered."
|
||||
)
|
||||
)
|
||||
team_name: str | None = Field(description="The team owning the bundle, in a multi-team deployment.")
|
||||
import_error_count: int | None = Field(
|
||||
import_error_count: int = Field(
|
||||
description=(
|
||||
"Number of Dag import errors recorded against this bundle that the caller is permitted "
|
||||
"to see, counted on the same terms as ``GET /importErrors``. Null when the caller may "
|
||||
"not read import errors."
|
||||
"Number of Dag import errors recorded against the bundle. Reading the errors themselves "
|
||||
"is authorized separately, through ``GET /importErrors``."
|
||||
)
|
||||
)
|
||||
|
||||
@@ -71,8 +71,7 @@ class DagBundleDetailResponse(DagBundleResponse):
|
||||
|
||||
dag_count: int = Field(
|
||||
description=(
|
||||
"Number of live Dags recorded against the bundle that the caller is permitted to see, "
|
||||
"counted on the same terms as ``GET /dags``."
|
||||
"Number of live Dags recorded against the bundle, including Dags the caller may not read."
|
||||
)
|
||||
)
|
||||
|
||||
@@ -81,18 +80,19 @@ class DagBundleFileResponse(BaseModel):
|
||||
"""A file in a Dag bundle, as the Dag processor last saw it."""
|
||||
|
||||
relative_fileloc: str
|
||||
dag_count: int = Field(description="Number of live Dags the file defines that the caller may read.")
|
||||
dag_count: int = Field(
|
||||
description="Number of live Dags the file defines, including Dags the caller may not read."
|
||||
)
|
||||
last_parsed_time: datetime | None = Field(
|
||||
description="When the file was last parsed, or null if it has never parsed successfully."
|
||||
)
|
||||
last_parse_duration: float | None = Field(
|
||||
description="How long the last successful parse of the file took, in seconds."
|
||||
)
|
||||
import_error_count: int | None = Field(
|
||||
import_error_count: int = Field(
|
||||
description=(
|
||||
"Number of import errors recorded against the file, which is at most one. Null when the "
|
||||
"caller may not read import errors -- deliberately not zero, which would read as a "
|
||||
"file with nothing wrong."
|
||||
"Number of import errors recorded against the file, which is at most one. Reading the "
|
||||
"error itself is authorized separately, through ``GET /importErrors``."
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from airflow.api_fastapi.core_api.base import BaseModel
|
||||
|
||||
|
||||
class DagBundleCountResponse(BaseModel):
|
||||
"""Number of Dag bundles holding a Dag or an import error the caller may read."""
|
||||
|
||||
total_entries: int
|
||||
@@ -631,6 +631,39 @@ paths:
|
||||
security:
|
||||
- OAuth2PasswordBearer: []
|
||||
- HTTPBearer: []
|
||||
/ui/dagBundles/count:
|
||||
get:
|
||||
tags:
|
||||
- Dag Bundle
|
||||
summary: Get Dag Bundle Count
|
||||
description: 'Count the Dag bundles holding a Dag or an import error the caller
|
||||
may read.
|
||||
|
||||
|
||||
Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI
|
||||
can tell whether a
|
||||
|
||||
bundle name is worth showing next to an import error. Stale Dags count, and
|
||||
so do import errors
|
||||
|
||||
in files that never registered a Dag, which ``GET /importErrors`` shows to
|
||||
holders of
|
||||
|
||||
``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import
|
||||
is where the name
|
||||
|
||||
matters most.'
|
||||
operationId: get_dag_bundle_count
|
||||
responses:
|
||||
'200':
|
||||
description: Successful Response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/DagBundleCountResponse'
|
||||
security:
|
||||
- OAuth2PasswordBearer: []
|
||||
- HTTPBearer: []
|
||||
/ui/dags:
|
||||
get:
|
||||
tags:
|
||||
@@ -3966,6 +3999,17 @@ components:
|
||||
- state_count_limit
|
||||
title: DAGsRunStateCountsCollectionResponse
|
||||
description: Collection of per-Dag DagRun-state counts for the Dag list page.
|
||||
DagBundleCountResponse:
|
||||
properties:
|
||||
total_entries:
|
||||
type: integer
|
||||
title: Total Entries
|
||||
type: object
|
||||
required:
|
||||
- total_entries
|
||||
title: DagBundleCountResponse
|
||||
description: Number of Dag bundles holding a Dag or an import error the caller
|
||||
may read.
|
||||
DagFolderCollectionResponse:
|
||||
properties:
|
||||
folders:
|
||||
|
||||
@@ -3380,19 +3380,10 @@ paths:
|
||||
up yet.
|
||||
|
||||
|
||||
A bundle is visible to a user who can read at least one Dag recorded against
|
||||
it. A bundle
|
||||
Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked
|
||||
against each
|
||||
|
||||
holding no registered Dag at all has nothing to authorize against, so it is
|
||||
listed only for a
|
||||
|
||||
user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that
|
||||
bundle''s team --
|
||||
|
||||
the same view, scoped the same way, that governs import errors for a file
|
||||
that never
|
||||
|
||||
registered a Dag.'
|
||||
bundle''s team, and bundles of a team the caller cannot see are left out.'
|
||||
operationId: get_dag_bundles
|
||||
security:
|
||||
- OAuth2PasswordBearer: []
|
||||
@@ -3511,16 +3502,14 @@ paths:
|
||||
description: 'List the files in a Dag bundle, ordered by path.
|
||||
|
||||
|
||||
A file is listed when the caller can read at least one Dag it defines. A file
|
||||
that recorded an
|
||||
Every file the bundle holds is listed, with counts covering the whole file:
|
||||
the bundle is
|
||||
|
||||
import error without registering any Dag is listed too, on the same admin-by-default
|
||||
terms as
|
||||
authorized as a unit, so its contents are not narrowed to the Dags the caller
|
||||
may read. What a
|
||||
|
||||
``GET /importErrors`` -- a file that fails before defining a Dag has no Dag
|
||||
to authorize on,
|
||||
|
||||
and it is the case this page most needs to show.
|
||||
file''s Dags contain, and the text of its import error, stay behind their
|
||||
own permissions.
|
||||
|
||||
|
||||
``dag_count`` counts live Dags only. Dag rows are never deleted: a file that
|
||||
@@ -15713,8 +15702,9 @@ components:
|
||||
- type: string
|
||||
- type: 'null'
|
||||
title: Bundle Url
|
||||
description: A link to view the bundle at ``version``, when one is configured
|
||||
and the caller may read Dag versions.
|
||||
description: A link to view the bundle at ``version``. Null when the bundle
|
||||
has no URL template, when the template needs a version and the bundle
|
||||
has not been refreshed yet, or when the template cannot be rendered.
|
||||
team_name:
|
||||
anyOf:
|
||||
- type: string
|
||||
@@ -15722,18 +15712,15 @@ components:
|
||||
title: Team Name
|
||||
description: The team owning the bundle, in a multi-team deployment.
|
||||
import_error_count:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: 'null'
|
||||
type: integer
|
||||
title: Import Error Count
|
||||
description: Number of Dag import errors recorded against this bundle that
|
||||
the caller is permitted to see, counted on the same terms as ``GET /importErrors``.
|
||||
Null when the caller may not read import errors.
|
||||
description: Number of Dag import errors recorded against the bundle. Reading
|
||||
the errors themselves is authorized separately, through ``GET /importErrors``.
|
||||
dag_count:
|
||||
type: integer
|
||||
title: Dag Count
|
||||
description: Number of live Dags recorded against the bundle that the caller
|
||||
is permitted to see, counted on the same terms as ``GET /dags``.
|
||||
description: Number of live Dags recorded against the bundle, including
|
||||
Dags the caller may not read.
|
||||
type: object
|
||||
required:
|
||||
- name
|
||||
@@ -15770,7 +15757,8 @@ components:
|
||||
dag_count:
|
||||
type: integer
|
||||
title: Dag Count
|
||||
description: Number of live Dags the file defines that the caller may read.
|
||||
description: Number of live Dags the file defines, including Dags the caller
|
||||
may not read.
|
||||
last_parsed_time:
|
||||
anyOf:
|
||||
- type: string
|
||||
@@ -15786,13 +15774,11 @@ components:
|
||||
title: Last Parse Duration
|
||||
description: How long the last successful parse of the file took, in seconds.
|
||||
import_error_count:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: 'null'
|
||||
type: integer
|
||||
title: Import Error Count
|
||||
description: Number of import errors recorded against the file, which is
|
||||
at most one. Null when the caller may not read import errors -- deliberately
|
||||
not zero, which would read as a file with nothing wrong.
|
||||
at most one. Reading the error itself is authorized separately, through
|
||||
``GET /importErrors``.
|
||||
type: object
|
||||
required:
|
||||
- relative_fileloc
|
||||
@@ -15835,8 +15821,9 @@ components:
|
||||
- type: string
|
||||
- type: 'null'
|
||||
title: Bundle Url
|
||||
description: A link to view the bundle at ``version``, when one is configured
|
||||
and the caller may read Dag versions.
|
||||
description: A link to view the bundle at ``version``. Null when the bundle
|
||||
has no URL template, when the template needs a version and the bundle
|
||||
has not been refreshed yet, or when the template cannot be rendered.
|
||||
team_name:
|
||||
anyOf:
|
||||
- type: string
|
||||
@@ -15844,13 +15831,10 @@ components:
|
||||
title: Team Name
|
||||
description: The team owning the bundle, in a multi-team deployment.
|
||||
import_error_count:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: 'null'
|
||||
type: integer
|
||||
title: Import Error Count
|
||||
description: Number of Dag import errors recorded against this bundle that
|
||||
the caller is permitted to see, counted on the same terms as ``GET /importErrors``.
|
||||
Null when the caller may not read import errors.
|
||||
description: Number of Dag import errors recorded against the bundle. Reading
|
||||
the errors themselves is authorized separately, through ``GET /importErrors``.
|
||||
type: object
|
||||
required:
|
||||
- name
|
||||
|
||||
@@ -20,9 +20,8 @@ from operator import itemgetter
|
||||
from typing import TYPE_CHECKING, Annotated, Any, TypeVar
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from sqlalchemy import and_, case, false, func, select
|
||||
from sqlalchemy import case, false, func, select
|
||||
|
||||
from airflow.api_fastapi.auth.managers.models.resource_details import AccessView, DagAccessEntity
|
||||
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
|
||||
from airflow.api_fastapi.common.parameters import QueryLimit, QueryOffset, SortParam
|
||||
from airflow.api_fastapi.common.router import AirflowRouter
|
||||
@@ -35,11 +34,9 @@ from airflow.api_fastapi.core_api.datamodels.dag_bundles import (
|
||||
)
|
||||
from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc
|
||||
from airflow.api_fastapi.core_api.security import (
|
||||
AuthManagerDep,
|
||||
GetUserDep,
|
||||
PermittedDagBundleFilter,
|
||||
ReadableDagBundlesFilterDep,
|
||||
requires_access_dag,
|
||||
requires_access_dag_bundles,
|
||||
)
|
||||
from airflow.configuration import conf
|
||||
from airflow.models import DagModel
|
||||
@@ -51,104 +48,29 @@ if TYPE_CHECKING:
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from airflow.api_fastapi.auth.managers.base_auth_manager import BaseAuthManager
|
||||
from airflow.api_fastapi.auth.managers.models.base_user import BaseUser
|
||||
|
||||
dag_bundles_router = AirflowRouter(tags=["Dag Bundle"], prefix="/dagBundles")
|
||||
|
||||
_BundleResponseT = TypeVar("_BundleResponseT", bound=DagBundleResponse)
|
||||
|
||||
|
||||
def _import_error_counts(
|
||||
*,
|
||||
bundle_names: Sequence[str],
|
||||
readable_dag_ids: set[str],
|
||||
auth_manager: BaseAuthManager,
|
||||
user: BaseUser,
|
||||
session: Session,
|
||||
team_names: dict[str, str | None] | None = None,
|
||||
) -> dict[str, int] | None:
|
||||
"""
|
||||
Count the import errors per bundle that this caller is allowed to know about.
|
||||
|
||||
Reproduces the two-part authorization of ``GET /importErrors`` rather than counting every row
|
||||
for the bundle: an error in a file the caller can read no Dag in stays hidden, and an error in
|
||||
a file with no registered Dag needs the admin-by-default ``IMPORT_ERRORS_ALL``, since the
|
||||
file's existence would otherwise leak. Returns ``None`` when the caller may not read import
|
||||
errors at all.
|
||||
"""
|
||||
if not auth_manager.authorize_view(access_view=AccessView.IMPORT_ERRORS, user=user):
|
||||
return None
|
||||
def _count_import_errors(bundle_names: Sequence[str], *, session: Session) -> dict[str, int]:
|
||||
if not bundle_names:
|
||||
return {}
|
||||
|
||||
# Files -- keyed ``(relative_fileloc, bundle_name)`` -- in which the caller can read a Dag.
|
||||
readable_files = (
|
||||
select(DagModel.relative_fileloc, DagModel.bundle_name)
|
||||
.where(
|
||||
DagModel.dag_id.in_(readable_dag_ids),
|
||||
DagModel.bundle_name.in_(bundle_names),
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
counts: dict[str, int] = {
|
||||
return {
|
||||
bundle_name: count
|
||||
for bundle_name, count in session.execute(
|
||||
select(ParseImportError.bundle_name, func.count())
|
||||
.join(
|
||||
readable_files,
|
||||
and_(
|
||||
ParseImportError.filename == readable_files.c.relative_fileloc,
|
||||
ParseImportError.bundle_name == readable_files.c.bundle_name,
|
||||
),
|
||||
)
|
||||
.where(ParseImportError.bundle_name.in_(bundle_names))
|
||||
.group_by(ParseImportError.bundle_name)
|
||||
).all()
|
||||
if bundle_name is not None
|
||||
)
|
||||
}
|
||||
|
||||
# Errors for files that never registered a Dag, added only where the caller holds the
|
||||
# admin-by-default view for that bundle's team. Narrowed to this page's bundles, unlike the
|
||||
# equivalent in ``import_error.py``: this endpoint polls, so an unbounded ``SELECT DISTINCT``
|
||||
# over ``dag`` every few seconds would be a real cost on a large deployment.
|
||||
files_with_any_dags = (
|
||||
select(DagModel.relative_fileloc, DagModel.bundle_name)
|
||||
.where(DagModel.bundle_name.in_(bundle_names))
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
unregistered = session.execute(
|
||||
select(ParseImportError.bundle_name, func.count())
|
||||
.outerjoin(
|
||||
files_with_any_dags,
|
||||
and_(
|
||||
ParseImportError.filename == files_with_any_dags.c.relative_fileloc,
|
||||
ParseImportError.bundle_name == files_with_any_dags.c.bundle_name,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
files_with_any_dags.c.relative_fileloc.is_(None),
|
||||
ParseImportError.bundle_name.in_(bundle_names),
|
||||
)
|
||||
.group_by(ParseImportError.bundle_name)
|
||||
).all()
|
||||
if unregistered:
|
||||
if team_names is None:
|
||||
team_names = DagBundleModel.get_team_names(
|
||||
[bundle_name for bundle_name, _ in unregistered if bundle_name], session=session
|
||||
)
|
||||
for bundle_name, count in unregistered:
|
||||
if bundle_name is None:
|
||||
continue
|
||||
if auth_manager.authorize_view(
|
||||
access_view=AccessView.IMPORT_ERRORS_ALL,
|
||||
user=user,
|
||||
team_name=team_names.get(bundle_name),
|
||||
):
|
||||
counts[bundle_name] = counts.get(bundle_name, 0) + count
|
||||
|
||||
return counts
|
||||
def _get_team_names(bundle_names: Sequence[str], *, session: Session) -> dict[str, str | None]:
|
||||
# ``DagBundleModel.team_name`` walks a lazily loaded relationship, so read them in one query.
|
||||
if not bundle_names or not conf.getboolean("core", "multi_team"):
|
||||
return {}
|
||||
return DagBundleModel.get_team_names(bundle_names, session=session)
|
||||
|
||||
|
||||
def _resolve_readable_bundle(
|
||||
@@ -157,8 +79,8 @@ def _resolve_readable_bundle(
|
||||
session: Session,
|
||||
) -> DagBundleModel:
|
||||
"""Load a bundle the caller is allowed to see, or raise 404 if there is no such bundle."""
|
||||
# 404 rather than 403 for a bundle that exists but holds no readable Dag, so that the response
|
||||
# does not confirm which bundle names a deployment has.
|
||||
# 404 rather than 403 for a bundle of a team the caller cannot see, so that the response does
|
||||
# not confirm which bundle names a deployment has.
|
||||
bundle = session.scalar(
|
||||
readable_dag_bundles_filter.to_orm(select(DagBundleModel).where(DagBundleModel.name == bundle_name))
|
||||
)
|
||||
@@ -172,8 +94,7 @@ def _serialize_bundle(
|
||||
bundle: DagBundleModel,
|
||||
*,
|
||||
team_name: str | None,
|
||||
import_error_count: int | None,
|
||||
show_bundle_url: bool,
|
||||
import_error_count: int,
|
||||
**extra: Any,
|
||||
) -> _BundleResponseT:
|
||||
"""Build a bundle response, so the column-to-field mapping lives in one place."""
|
||||
@@ -182,20 +103,14 @@ def _serialize_bundle(
|
||||
active=bundle.active,
|
||||
version=bundle.version,
|
||||
last_refreshed=bundle.last_refreshed,
|
||||
bundle_url=bundle.render_url(bundle.version) if show_bundle_url else None,
|
||||
bundle_url=bundle.render_url(bundle.version),
|
||||
team_name=team_name,
|
||||
import_error_count=import_error_count,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def _may_show_bundle_url(auth_manager: BaseAuthManager, user: BaseUser) -> bool:
|
||||
# A rendered bundle url is otherwise only reachable through ``GET /dags/{dag_id}/dagVersions``,
|
||||
# which additionally requires Dag *version* read, so withhold it from a role without that.
|
||||
return auth_manager.is_authorized_dag(method="GET", access_entity=DagAccessEntity.VERSION, user=user)
|
||||
|
||||
|
||||
@dag_bundles_router.get("", dependencies=[Depends(requires_access_dag(method="GET"))])
|
||||
@dag_bundles_router.get("", dependencies=[Depends(requires_access_dag_bundles())])
|
||||
def get_dag_bundles(
|
||||
limit: QueryLimit,
|
||||
offset: QueryOffset,
|
||||
@@ -204,9 +119,7 @@ def get_dag_bundles(
|
||||
Depends(SortParam(["name", "version", "last_refreshed", "active"], DagBundleModel).dynamic_depends()),
|
||||
],
|
||||
readable_dag_bundles_filter: ReadableDagBundlesFilterDep,
|
||||
auth_manager: AuthManagerDep,
|
||||
session: SessionDep,
|
||||
user: GetUserDep,
|
||||
) -> DagBundleCollectionResponse:
|
||||
"""
|
||||
List the Dag bundles Airflow knows about.
|
||||
@@ -214,11 +127,8 @@ def get_dag_bundles(
|
||||
Reports the version of each bundle Airflow currently holds and when a Dag processor last
|
||||
refreshed it, so whoever deployed the code can tell whether it has been picked up yet.
|
||||
|
||||
A bundle is visible to a user who can read at least one Dag recorded against it. A bundle
|
||||
holding no registered Dag at all has nothing to authorize against, so it is listed only for a
|
||||
user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that bundle's team --
|
||||
the same view, scoped the same way, that governs import errors for a file that never
|
||||
registered a Dag.
|
||||
Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked against each
|
||||
bundle's team, and bundles of a team the caller cannot see are left out.
|
||||
"""
|
||||
bundles_select, total_entries = paginated_select(
|
||||
statement=select(DagBundleModel),
|
||||
@@ -230,23 +140,8 @@ def get_dag_bundles(
|
||||
)
|
||||
bundles = session.scalars(bundles_select).all()
|
||||
bundle_names = [bundle.name for bundle in bundles]
|
||||
|
||||
# ``DagBundleModel.team_name`` walks a lazily loaded relationship, so read them in one query.
|
||||
team_names = (
|
||||
DagBundleModel.get_team_names(bundle_names, session=session)
|
||||
if bundle_names and conf.getboolean("core", "multi_team")
|
||||
else {}
|
||||
)
|
||||
|
||||
import_error_counts = _import_error_counts(
|
||||
bundle_names=bundle_names,
|
||||
readable_dag_ids=readable_dag_bundles_filter.value or set(),
|
||||
auth_manager=auth_manager,
|
||||
user=user,
|
||||
session=session,
|
||||
)
|
||||
|
||||
show_bundle_url = _may_show_bundle_url(auth_manager, user)
|
||||
team_names = _get_team_names(bundle_names, session=session)
|
||||
import_error_counts = _count_import_errors(bundle_names, session=session)
|
||||
|
||||
return DagBundleCollectionResponse(
|
||||
dag_bundles=[
|
||||
@@ -254,10 +149,7 @@ def get_dag_bundles(
|
||||
DagBundleResponse,
|
||||
bundle,
|
||||
team_name=team_names.get(bundle.name),
|
||||
import_error_count=(
|
||||
None if import_error_counts is None else import_error_counts.get(bundle.name, 0)
|
||||
),
|
||||
show_bundle_url=show_bundle_url,
|
||||
import_error_count=import_error_counts.get(bundle.name, 0),
|
||||
)
|
||||
for bundle in bundles
|
||||
],
|
||||
@@ -268,49 +160,29 @@ def get_dag_bundles(
|
||||
@dag_bundles_router.get(
|
||||
"/{bundle_name}",
|
||||
responses=create_openapi_http_exception_doc([status.HTTP_404_NOT_FOUND]),
|
||||
dependencies=[Depends(requires_access_dag(method="GET"))],
|
||||
dependencies=[Depends(requires_access_dag_bundles())],
|
||||
)
|
||||
def get_dag_bundle(
|
||||
bundle_name: str,
|
||||
readable_dag_bundles_filter: ReadableDagBundlesFilterDep,
|
||||
auth_manager: AuthManagerDep,
|
||||
session: SessionDep,
|
||||
user: GetUserDep,
|
||||
) -> DagBundleDetailResponse:
|
||||
"""Get a Dag bundle."""
|
||||
bundle = _resolve_readable_bundle(bundle_name, readable_dag_bundles_filter, session)
|
||||
readable_dag_ids = readable_dag_bundles_filter.value or set()
|
||||
|
||||
# Stale rows are excluded, matching ``GET /dags``; see ``get_dag_bundle_files`` for why.
|
||||
dag_count = session.scalar(
|
||||
select(func.count(DagModel.dag_id)).where(
|
||||
DagModel.bundle_name == bundle_name,
|
||||
DagModel.dag_id.in_(readable_dag_ids),
|
||||
DagModel.is_stale == false(),
|
||||
)
|
||||
)
|
||||
# Resolved once and threaded through: ``_import_error_counts`` needs the team to authorize
|
||||
# the admin-gated view, and the response needs it again.
|
||||
team_names = (
|
||||
DagBundleModel.get_team_names([bundle_name], session=session)
|
||||
if conf.getboolean("core", "multi_team")
|
||||
else {}
|
||||
)
|
||||
import_error_counts = _import_error_counts(
|
||||
bundle_names=[bundle_name],
|
||||
readable_dag_ids=readable_dag_ids,
|
||||
auth_manager=auth_manager,
|
||||
user=user,
|
||||
session=session,
|
||||
team_names=team_names,
|
||||
)
|
||||
|
||||
return _serialize_bundle(
|
||||
DagBundleDetailResponse,
|
||||
bundle,
|
||||
team_name=team_names.get(bundle_name),
|
||||
import_error_count=(None if import_error_counts is None else import_error_counts.get(bundle_name, 0)),
|
||||
show_bundle_url=_may_show_bundle_url(auth_manager, user),
|
||||
team_name=_get_team_names([bundle_name], session=session).get(bundle_name),
|
||||
import_error_count=_count_import_errors([bundle_name], session=session).get(bundle_name, 0),
|
||||
dag_count=dag_count or 0,
|
||||
)
|
||||
|
||||
@@ -318,24 +190,21 @@ def get_dag_bundle(
|
||||
@dag_bundles_router.get(
|
||||
"/{bundle_name}/files",
|
||||
responses=create_openapi_http_exception_doc([status.HTTP_404_NOT_FOUND]),
|
||||
dependencies=[Depends(requires_access_dag(method="GET"))],
|
||||
dependencies=[Depends(requires_access_dag_bundles())],
|
||||
)
|
||||
def get_dag_bundle_files(
|
||||
bundle_name: str,
|
||||
limit: QueryLimit,
|
||||
offset: QueryOffset,
|
||||
readable_dag_bundles_filter: ReadableDagBundlesFilterDep,
|
||||
auth_manager: AuthManagerDep,
|
||||
session: SessionDep,
|
||||
user: GetUserDep,
|
||||
) -> DagBundleFileCollectionResponse:
|
||||
"""
|
||||
List the files in a Dag bundle, ordered by path.
|
||||
|
||||
A file is listed when the caller can read at least one Dag it defines. A file that recorded an
|
||||
import error without registering any Dag is listed too, on the same admin-by-default terms as
|
||||
``GET /importErrors`` -- a file that fails before defining a Dag has no Dag to authorize on,
|
||||
and it is the case this page most needs to show.
|
||||
Every file the bundle holds is listed, with counts covering the whole file: the bundle is
|
||||
authorized as a unit, so its contents are not narrowed to the Dags the caller may read. What a
|
||||
file's Dags contain, and the text of its import error, stay behind their own permissions.
|
||||
|
||||
``dag_count`` counts live Dags only. Dag rows are never deleted: a file that fails to import
|
||||
has every Dag in it marked stale, and so does a file dropped from the bundle. A file therefore
|
||||
@@ -347,7 +216,6 @@ def get_dag_bundle_files(
|
||||
record Airflow keeps: a file whose every Dag was removed keeps no parse time of its own.
|
||||
"""
|
||||
_resolve_readable_bundle(bundle_name, readable_dag_bundles_filter, session)
|
||||
readable_dag_ids = readable_dag_bundles_filter.value or set()
|
||||
|
||||
# The duration is aggregated over live rows only. A Dag removed from a file keeps its row,
|
||||
# frozen at an older parse, so an unrestricted ``MAX`` would pair that stale duration with the
|
||||
@@ -358,9 +226,7 @@ def get_dag_bundle_files(
|
||||
"dag_count": live_dag_count,
|
||||
"last_parsed_time": last_parsed_time,
|
||||
"last_parse_duration": last_parse_duration,
|
||||
# Zero would read as "nothing wrong with this file"; the caller's permission is
|
||||
# decided below, and until then the count is unknown rather than clean.
|
||||
"import_error_count": None,
|
||||
"import_error_count": 0,
|
||||
}
|
||||
for fileloc, live_dag_count, last_parsed_time, last_parse_duration in session.execute(
|
||||
select(
|
||||
@@ -371,56 +237,32 @@ def get_dag_bundle_files(
|
||||
)
|
||||
.where(
|
||||
DagModel.bundle_name == bundle_name,
|
||||
DagModel.dag_id.in_(readable_dag_ids),
|
||||
DagModel.relative_fileloc.is_not(None),
|
||||
)
|
||||
.group_by(DagModel.relative_fileloc)
|
||||
)
|
||||
}
|
||||
|
||||
error_filenames: set[str] = set()
|
||||
if auth_manager.authorize_view(access_view=AccessView.IMPORT_ERRORS, user=user):
|
||||
for row in files.values():
|
||||
row["import_error_count"] = 0
|
||||
error_filenames = {
|
||||
filename
|
||||
for filename in session.scalars(
|
||||
select(ParseImportError.filename).where(ParseImportError.bundle_name == bundle_name)
|
||||
)
|
||||
if filename is not None
|
||||
}
|
||||
for filename in error_filenames & files.keys():
|
||||
# ``import_error`` holds one row per file, so this is 0 or 1.
|
||||
files[filename]["import_error_count"] = 1
|
||||
error_filenames = {
|
||||
filename
|
||||
for filename in session.scalars(
|
||||
select(ParseImportError.filename).where(ParseImportError.bundle_name == bundle_name)
|
||||
)
|
||||
if filename is not None
|
||||
}
|
||||
for filename in error_filenames:
|
||||
# ``import_error`` holds one row per file, so this is 0 or 1.
|
||||
files.setdefault(
|
||||
filename,
|
||||
{
|
||||
"relative_fileloc": filename,
|
||||
"dag_count": 0,
|
||||
"last_parsed_time": None,
|
||||
"last_parse_duration": None,
|
||||
},
|
||||
)["import_error_count"] = 1
|
||||
|
||||
unregistered = error_filenames - files.keys()
|
||||
if unregistered and auth_manager.authorize_view(
|
||||
access_view=AccessView.IMPORT_ERRORS_ALL,
|
||||
user=user,
|
||||
team_name=(
|
||||
DagBundleModel.get_team_names([bundle_name], session=session).get(bundle_name)
|
||||
if conf.getboolean("core", "multi_team")
|
||||
else None
|
||||
),
|
||||
):
|
||||
# Queried only behind the permission that consumes it. A file with no Dag row at all
|
||||
# has no Dag to authorize on, so it needs the admin-by-default view; a file whose Dags
|
||||
# exist but are unreadable is excluded by being in this set.
|
||||
files_with_any_dags = set(
|
||||
session.scalars(
|
||||
select(DagModel.relative_fileloc).where(DagModel.bundle_name == bundle_name).distinct()
|
||||
)
|
||||
)
|
||||
for filename in unregistered - files_with_any_dags:
|
||||
files[filename] = {
|
||||
"relative_fileloc": filename,
|
||||
"dag_count": 0,
|
||||
"last_parsed_time": None,
|
||||
"last_parse_duration": None,
|
||||
"import_error_count": 1,
|
||||
}
|
||||
|
||||
# A file with no live Dag and no error the caller can see is one the bundle no longer has.
|
||||
# A file with no live Dag and no error is one the bundle no longer has.
|
||||
ordered = sorted(
|
||||
(row for path, row in files.items() if row["dag_count"] > 0 or path in error_filenames),
|
||||
key=itemgetter("relative_fileloc"),
|
||||
|
||||
@@ -25,6 +25,7 @@ from airflow.api_fastapi.core_api.routes.ui.backfills import backfills_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.calendar import calendar_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.config import config_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.connections import connections_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.dag_bundles import dag_bundles_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.dag_runs import dag_runs_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.dags import dags_router
|
||||
from airflow.api_fastapi.core_api.routes.ui.dashboard import dashboard_router
|
||||
@@ -49,6 +50,7 @@ ui_router.include_router(execution_router)
|
||||
ui_router.include_router(partitioned_dag_runs_router)
|
||||
ui_router.include_router(config_router)
|
||||
ui_router.include_router(connections_router)
|
||||
ui_router.include_router(dag_bundles_router)
|
||||
ui_router.include_router(dags_router)
|
||||
ui_router.include_router(dependencies_router)
|
||||
ui_router.include_router(dashboard_router)
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import exists, select
|
||||
|
||||
from airflow.api_fastapi.common.db.common import SessionDep
|
||||
from airflow.api_fastapi.common.router import AirflowRouter
|
||||
from airflow.api_fastapi.core_api.datamodels.ui.dag_bundles import DagBundleCountResponse
|
||||
from airflow.api_fastapi.core_api.security import (
|
||||
AccessView,
|
||||
AuthManagerDep,
|
||||
GetUserDep,
|
||||
ReadableDagsFilterDep,
|
||||
requires_access_dag,
|
||||
)
|
||||
from airflow.models import DagModel
|
||||
from airflow.models.dagbundle import DagBundleModel
|
||||
from airflow.models.errors import ParseImportError
|
||||
|
||||
dag_bundles_router = AirflowRouter(tags=["Dag Bundle"], prefix="/dagBundles")
|
||||
|
||||
|
||||
@dag_bundles_router.get("/count", dependencies=[Depends(requires_access_dag(method="GET"))])
|
||||
def get_dag_bundle_count(
|
||||
readable_dags_filter: ReadableDagsFilterDep,
|
||||
user: GetUserDep,
|
||||
auth_manager: AuthManagerDep,
|
||||
session: SessionDep,
|
||||
) -> DagBundleCountResponse:
|
||||
"""
|
||||
Count the Dag bundles holding a Dag or an import error the caller may read.
|
||||
|
||||
Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI can tell whether a
|
||||
bundle name is worth showing next to an import error. Stale Dags count, and so do import errors
|
||||
in files that never registered a Dag, which ``GET /importErrors`` shows to holders of
|
||||
``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import is where the name
|
||||
matters most.
|
||||
"""
|
||||
bundle_names = set(session.scalars(readable_dags_filter.to_orm(select(DagModel.bundle_name).distinct())))
|
||||
dagless_error_bundle_names: set[str] = {
|
||||
bundle_name
|
||||
for bundle_name in session.scalars(
|
||||
select(ParseImportError.bundle_name)
|
||||
.distinct()
|
||||
.where(
|
||||
ParseImportError.bundle_name.is_not(None),
|
||||
~exists().where(
|
||||
DagModel.bundle_name == ParseImportError.bundle_name,
|
||||
DagModel.relative_fileloc == ParseImportError.filename,
|
||||
),
|
||||
)
|
||||
)
|
||||
if bundle_name is not None and bundle_name not in bundle_names
|
||||
}
|
||||
team_name_by_bundle = DagBundleModel.get_team_names(dagless_error_bundle_names, session=session)
|
||||
bundle_names.update(
|
||||
bundle_name
|
||||
for bundle_name in dagless_error_bundle_names
|
||||
if auth_manager.authorize_view(
|
||||
access_view=AccessView.IMPORT_ERRORS_ALL,
|
||||
user=user,
|
||||
team_name=team_name_by_bundle.get(bundle_name),
|
||||
)
|
||||
)
|
||||
return DagBundleCountResponse(total_entries=len(bundle_names))
|
||||
@@ -27,7 +27,7 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer, OAuth2Pas
|
||||
from itsdangerous import BadSignature, URLSafeSerializer
|
||||
from jwt import ExpiredSignatureError, InvalidTokenError
|
||||
from pydantic import NonNegativeInt, TypeAdapter, ValidationError
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy import false, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from airflow.api_fastapi.app import get_auth_manager
|
||||
@@ -80,7 +80,7 @@ from airflow.models.dagwarning import DagWarning
|
||||
from airflow.models.errors import ParseImportError
|
||||
from airflow.models.log import Log
|
||||
from airflow.models.taskinstance import TaskInstance as TI
|
||||
from airflow.models.team import Team
|
||||
from airflow.models.team import Team, dag_bundle_team_association_table
|
||||
from airflow.models.xcom import XComModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -394,33 +394,28 @@ class PermittedBackfillFilter(PermittedDagFilter):
|
||||
return select.where(Backfill.dag_id.in_(self.value or set()))
|
||||
|
||||
|
||||
class PermittedDagBundleFilter(PermittedDagFilter):
|
||||
"""A parameter that filters Dag bundles to the ones holding a Dag the user may read."""
|
||||
class PermittedDagBundleFilter(OrmClause[set[str | None]]):
|
||||
"""
|
||||
A parameter that filters Dag bundles to the teams the user may read them for.
|
||||
|
||||
def __init__(self, value: set[str] | None = None, *, permitted_dagless_bundles: set[str] | None = None):
|
||||
super().__init__(value)
|
||||
self.permitted_dagless_bundles = permitted_dagless_bundles or set()
|
||||
``value`` holds those team names, with ``None`` standing for bundles with no team. A ``None``
|
||||
value leaves every bundle in.
|
||||
"""
|
||||
|
||||
def to_orm(self, statement: Select) -> Select:
|
||||
# A bundle carries no per-Dag key to authorize on, so it is scoped by the Dags inside it.
|
||||
# Filtering in the query keeps unauthorized rows out of the count and pagination as well.
|
||||
# A bundle whose Dags have since been removed stays visible while a stale ``DagModel`` row
|
||||
# names it.
|
||||
holds_a_readable_dag = DagBundleModel.name.in_(
|
||||
select(DagModel.bundle_name).where(DagModel.dag_id.in_(self.value or set()))
|
||||
)
|
||||
if not self.permitted_dagless_bundles:
|
||||
return statement.where(holds_a_readable_dag)
|
||||
# A bundle with no registered Dag at all -- a first deploy whose only file fails to import,
|
||||
# say -- has no Dag to authorize against, so scoping by Dags alone would hide the bundle
|
||||
# precisely when its import-error count is the thing worth reading. Those bundles are
|
||||
# authorized one at a time against their own team in
|
||||
# ``readable_dag_bundles_filter_factory`` and arrive here as an explicit allow-list, so a
|
||||
# team-aware policy decides each one rather than a single unscoped check standing in for
|
||||
# all of them.
|
||||
return statement.where(
|
||||
or_(holds_a_readable_dag, DagBundleModel.name.in_(self.permitted_dagless_bundles))
|
||||
)
|
||||
if self.value is None:
|
||||
return statement
|
||||
team_bundle_names = select(dag_bundle_team_association_table.c.dag_bundle_name)
|
||||
conditions = []
|
||||
if team_names := {team_name for team_name in self.value if team_name is not None}:
|
||||
conditions.append(
|
||||
DagBundleModel.name.in_(
|
||||
team_bundle_names.where(dag_bundle_team_association_table.c.team_name.in_(team_names))
|
||||
)
|
||||
)
|
||||
if None in self.value:
|
||||
conditions.append(DagBundleModel.name.not_in(team_bundle_names))
|
||||
return statement.where(or_(false(), *conditions))
|
||||
|
||||
|
||||
def permitted_dag_filter_factory(
|
||||
@@ -465,9 +460,10 @@ def readable_dag_bundles_filter_factory() -> Callable[
|
||||
"""
|
||||
Create a callable for Depends in FastAPI that returns the Dag bundle filter for the user.
|
||||
|
||||
Dag bundles need their own factory rather than ``permitted_dag_filter_factory``: besides the
|
||||
readable Dag ids, the filter needs the set of bundles holding no registered Dag that this
|
||||
caller may see, which is a separate authorization decision taken per bundle.
|
||||
A bundle is authorized on ``AccessView.DAG_BUNDLES`` and membership of the bundle's team, not
|
||||
on the Dags inside it: a bundle is the unit code is deployed in, so what it holds is the
|
||||
bundle's to show. Outside multi-team mode ``requires_access_dag_bundles`` has decided it for
|
||||
every bundle.
|
||||
"""
|
||||
|
||||
def depends_readable_dag_bundles_filter(
|
||||
@@ -475,34 +471,24 @@ def readable_dag_bundles_filter_factory() -> Callable[
|
||||
auth_manager: AuthManagerDep,
|
||||
session: SessionDep,
|
||||
) -> PermittedDagBundleFilter:
|
||||
# A bundle with no registered Dag has nothing to authorize against, so it is gated on the
|
||||
# admin-by-default ``IMPORT_ERRORS_ALL`` -- the same view ``GET /importErrors`` uses for a
|
||||
# file that never registered a Dag. Resolved here, one bundle at a time against its own
|
||||
# team, so the result can go into the query: authorizing after the fact would leave
|
||||
# unauthorized bundles in ``total_entries`` and in the page.
|
||||
dagless_bundles = session.scalars(
|
||||
select(DagBundleModel.name).where(
|
||||
DagBundleModel.name.notin_(
|
||||
# ``dag.bundle_name`` is non-nullable, but the guard keeps a stray NULL from
|
||||
# emptying the whole ``NOT IN``.
|
||||
select(DagModel.bundle_name).where(DagModel.bundle_name.is_not(None))
|
||||
)
|
||||
)
|
||||
).all()
|
||||
team_name_by_bundle = (
|
||||
DagBundleModel.get_team_names(dagless_bundles, session=session) if dagless_bundles else {}
|
||||
)
|
||||
if not conf.getboolean("core", "multi_team"):
|
||||
# ``requires_access_dag_bundles`` already decided the unscoped view for the route.
|
||||
return PermittedDagBundleFilter()
|
||||
|
||||
# Team membership is checked on its own because some auth managers ignore ``team_name``
|
||||
# when authorizing a view, which would let one team read another team's bundles.
|
||||
member_teams: set[str | None] = {
|
||||
*auth_manager.get_authorized_teams(user=user, session=session),
|
||||
None,
|
||||
}
|
||||
return PermittedDagBundleFilter(
|
||||
auth_manager.get_authorized_dag_ids(user=user, method="GET"),
|
||||
permitted_dagless_bundles={
|
||||
bundle_name
|
||||
for bundle_name in dagless_bundles
|
||||
{
|
||||
team_name
|
||||
for team_name in member_teams
|
||||
if auth_manager.authorize_view(
|
||||
access_view=AccessView.IMPORT_ERRORS_ALL,
|
||||
user=user,
|
||||
team_name=team_name_by_bundle.get(bundle_name),
|
||||
access_view=AccessView.DAG_BUNDLES, user=user, team_name=team_name
|
||||
)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return depends_readable_dag_bundles_filter
|
||||
@@ -1284,6 +1270,34 @@ def requires_access_view(access_view: AccessView) -> Callable[[Request, BaseUser
|
||||
return inner
|
||||
|
||||
|
||||
def requires_access_dag_bundles() -> Callable[[BaseUser, BaseAuthManager, PermittedDagBundleFilter], None]:
|
||||
"""
|
||||
Require ``AccessView.DAG_BUNDLES``.
|
||||
|
||||
In a multi-team deployment it is enough to hold it for one of the caller's teams or without a
|
||||
team, the same set ``ReadableDagBundlesFilterDep`` narrows the bundles to.
|
||||
|
||||
Goes through ``authorize_view`` rather than ``requires_access_view`` so that an auth manager
|
||||
that cannot map the view (raises ``KeyError``) denies it instead of failing the request.
|
||||
"""
|
||||
|
||||
def inner(
|
||||
user: GetUserDep,
|
||||
auth_manager: AuthManagerDep,
|
||||
readable_dag_bundles_filter: ReadableDagBundlesFilterDep,
|
||||
) -> None:
|
||||
if (permitted_teams := readable_dag_bundles_filter.value) is not None:
|
||||
_requires_access(is_authorized_callback=lambda: bool(permitted_teams))
|
||||
return
|
||||
_requires_access(
|
||||
is_authorized_callback=lambda: auth_manager.authorize_view(
|
||||
access_view=AccessView.DAG_BUNDLES, user=user
|
||||
),
|
||||
)
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
def requires_access_asset_alias(method: ResourceMethod) -> Callable[[Request, BaseUser], None]:
|
||||
def inner(
|
||||
request: Request,
|
||||
|
||||
@@ -342,6 +342,10 @@ export const UseDagBundleServiceGetDagBundleFilesKeyFn = ({ bundleName, limit, o
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}, queryKey?: Array<unknown>) => [useDagBundleServiceGetDagBundleFilesKey, ...(queryKey ?? [{ bundleName, limit, offset }])];
|
||||
export type DagBundleServiceGetDagBundleCountDefaultResponse = Awaited<ReturnType<typeof DagBundleService.getDagBundleCount>>;
|
||||
export type DagBundleServiceGetDagBundleCountQueryResult<TData = DagBundleServiceGetDagBundleCountDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>;
|
||||
export const useDagBundleServiceGetDagBundleCountKey = "DagBundleServiceGetDagBundleCount";
|
||||
export const UseDagBundleServiceGetDagBundleCountKeyFn = (queryKey?: Array<unknown>) => [useDagBundleServiceGetDagBundleCountKey, ...(queryKey ?? [])];
|
||||
export type DagStatsServiceGetDagStatsDefaultResponse = Awaited<ReturnType<typeof DagStatsService.getDagStats>>;
|
||||
export type DagStatsServiceGetDagStatsQueryResult<TData = DagStatsServiceGetDagStatsDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>;
|
||||
export const useDagStatsServiceGetDagStatsKey = "DagStatsServiceGetDagStats";
|
||||
|
||||
@@ -630,11 +630,8 @@ export const ensureUseDagSourceServiceGetDagSourceData = (queryClient: QueryClie
|
||||
* Reports the version of each bundle Airflow currently holds and when a Dag processor last
|
||||
* refreshed it, so whoever deployed the code can tell whether it has been picked up yet.
|
||||
*
|
||||
* A bundle is visible to a user who can read at least one Dag recorded against it. A bundle
|
||||
* holding no registered Dag at all has nothing to authorize against, so it is listed only for a
|
||||
* user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that bundle's team --
|
||||
* the same view, scoped the same way, that governs import errors for a file that never
|
||||
* registered a Dag.
|
||||
* Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked against each
|
||||
* bundle's team, and bundles of a team the caller cannot see are left out.
|
||||
* @param data The data for the request.
|
||||
* @param data.limit
|
||||
* @param data.offset
|
||||
@@ -662,10 +659,9 @@ export const ensureUseDagBundleServiceGetDagBundleData = (queryClient: QueryClie
|
||||
* Get Dag Bundle Files
|
||||
* List the files in a Dag bundle, ordered by path.
|
||||
*
|
||||
* A file is listed when the caller can read at least one Dag it defines. A file that recorded an
|
||||
* import error without registering any Dag is listed too, on the same admin-by-default terms as
|
||||
* ``GET /importErrors`` -- a file that fails before defining a Dag has no Dag to authorize on,
|
||||
* and it is the case this page most needs to show.
|
||||
* Every file the bundle holds is listed, with counts covering the whole file: the bundle is
|
||||
* authorized as a unit, so its contents are not narrowed to the Dags the caller may read. What a
|
||||
* file's Dags contain, and the text of its import error, stay behind their own permissions.
|
||||
*
|
||||
* ``dag_count`` counts live Dags only. Dag rows are never deleted: a file that fails to import
|
||||
* has every Dag in it marked stale, and so does a file dropped from the bundle. A file therefore
|
||||
@@ -688,6 +684,19 @@ export const ensureUseDagBundleServiceGetDagBundleFilesData = (queryClient: Quer
|
||||
offset?: number;
|
||||
}) => queryClient.ensureQueryData({ queryKey: Common.UseDagBundleServiceGetDagBundleFilesKeyFn({ bundleName, limit, offset }), queryFn: () => DagBundleService.getDagBundleFiles({ bundleName, limit, offset }) });
|
||||
/**
|
||||
* Get Dag Bundle Count
|
||||
* Count the Dag bundles holding a Dag or an import error the caller may read.
|
||||
*
|
||||
* Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI can tell whether a
|
||||
* bundle name is worth showing next to an import error. Stale Dags count, and so do import errors
|
||||
* in files that never registered a Dag, which ``GET /importErrors`` shows to holders of
|
||||
* ``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import is where the name
|
||||
* matters most.
|
||||
* @returns DagBundleCountResponse Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
export const ensureUseDagBundleServiceGetDagBundleCountData = (queryClient: QueryClient) => queryClient.ensureQueryData({ queryKey: Common.UseDagBundleServiceGetDagBundleCountKeyFn(), queryFn: () => DagBundleService.getDagBundleCount() });
|
||||
/**
|
||||
* Get Dag Stats
|
||||
* Get Dag statistics.
|
||||
* @param data The data for the request.
|
||||
|
||||
@@ -630,11 +630,8 @@ export const prefetchUseDagSourceServiceGetDagSource = (queryClient: QueryClient
|
||||
* Reports the version of each bundle Airflow currently holds and when a Dag processor last
|
||||
* refreshed it, so whoever deployed the code can tell whether it has been picked up yet.
|
||||
*
|
||||
* A bundle is visible to a user who can read at least one Dag recorded against it. A bundle
|
||||
* holding no registered Dag at all has nothing to authorize against, so it is listed only for a
|
||||
* user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that bundle's team --
|
||||
* the same view, scoped the same way, that governs import errors for a file that never
|
||||
* registered a Dag.
|
||||
* Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked against each
|
||||
* bundle's team, and bundles of a team the caller cannot see are left out.
|
||||
* @param data The data for the request.
|
||||
* @param data.limit
|
||||
* @param data.offset
|
||||
@@ -662,10 +659,9 @@ export const prefetchUseDagBundleServiceGetDagBundle = (queryClient: QueryClient
|
||||
* Get Dag Bundle Files
|
||||
* List the files in a Dag bundle, ordered by path.
|
||||
*
|
||||
* A file is listed when the caller can read at least one Dag it defines. A file that recorded an
|
||||
* import error without registering any Dag is listed too, on the same admin-by-default terms as
|
||||
* ``GET /importErrors`` -- a file that fails before defining a Dag has no Dag to authorize on,
|
||||
* and it is the case this page most needs to show.
|
||||
* Every file the bundle holds is listed, with counts covering the whole file: the bundle is
|
||||
* authorized as a unit, so its contents are not narrowed to the Dags the caller may read. What a
|
||||
* file's Dags contain, and the text of its import error, stay behind their own permissions.
|
||||
*
|
||||
* ``dag_count`` counts live Dags only. Dag rows are never deleted: a file that fails to import
|
||||
* has every Dag in it marked stale, and so does a file dropped from the bundle. A file therefore
|
||||
@@ -688,6 +684,19 @@ export const prefetchUseDagBundleServiceGetDagBundleFiles = (queryClient: QueryC
|
||||
offset?: number;
|
||||
}) => queryClient.prefetchQuery({ queryKey: Common.UseDagBundleServiceGetDagBundleFilesKeyFn({ bundleName, limit, offset }), queryFn: () => DagBundleService.getDagBundleFiles({ bundleName, limit, offset }) });
|
||||
/**
|
||||
* Get Dag Bundle Count
|
||||
* Count the Dag bundles holding a Dag or an import error the caller may read.
|
||||
*
|
||||
* Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI can tell whether a
|
||||
* bundle name is worth showing next to an import error. Stale Dags count, and so do import errors
|
||||
* in files that never registered a Dag, which ``GET /importErrors`` shows to holders of
|
||||
* ``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import is where the name
|
||||
* matters most.
|
||||
* @returns DagBundleCountResponse Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
export const prefetchUseDagBundleServiceGetDagBundleCount = (queryClient: QueryClient) => queryClient.prefetchQuery({ queryKey: Common.UseDagBundleServiceGetDagBundleCountKeyFn(), queryFn: () => DagBundleService.getDagBundleCount() });
|
||||
/**
|
||||
* Get Dag Stats
|
||||
* Get Dag statistics.
|
||||
* @param data The data for the request.
|
||||
|
||||
@@ -630,11 +630,8 @@ export const useDagSourceServiceGetDagSource = <TData = Common.DagSourceServiceG
|
||||
* Reports the version of each bundle Airflow currently holds and when a Dag processor last
|
||||
* refreshed it, so whoever deployed the code can tell whether it has been picked up yet.
|
||||
*
|
||||
* A bundle is visible to a user who can read at least one Dag recorded against it. A bundle
|
||||
* holding no registered Dag at all has nothing to authorize against, so it is listed only for a
|
||||
* user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that bundle's team --
|
||||
* the same view, scoped the same way, that governs import errors for a file that never
|
||||
* registered a Dag.
|
||||
* Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked against each
|
||||
* bundle's team, and bundles of a team the caller cannot see are left out.
|
||||
* @param data The data for the request.
|
||||
* @param data.limit
|
||||
* @param data.offset
|
||||
@@ -662,10 +659,9 @@ export const useDagBundleServiceGetDagBundle = <TData = Common.DagBundleServiceG
|
||||
* Get Dag Bundle Files
|
||||
* List the files in a Dag bundle, ordered by path.
|
||||
*
|
||||
* A file is listed when the caller can read at least one Dag it defines. A file that recorded an
|
||||
* import error without registering any Dag is listed too, on the same admin-by-default terms as
|
||||
* ``GET /importErrors`` -- a file that fails before defining a Dag has no Dag to authorize on,
|
||||
* and it is the case this page most needs to show.
|
||||
* Every file the bundle holds is listed, with counts covering the whole file: the bundle is
|
||||
* authorized as a unit, so its contents are not narrowed to the Dags the caller may read. What a
|
||||
* file's Dags contain, and the text of its import error, stay behind their own permissions.
|
||||
*
|
||||
* ``dag_count`` counts live Dags only. Dag rows are never deleted: a file that fails to import
|
||||
* has every Dag in it marked stale, and so does a file dropped from the bundle. A file therefore
|
||||
@@ -688,6 +684,19 @@ export const useDagBundleServiceGetDagBundleFiles = <TData = Common.DagBundleSer
|
||||
offset?: number;
|
||||
}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseDagBundleServiceGetDagBundleFilesKeyFn({ bundleName, limit, offset }, queryKey), queryFn: () => DagBundleService.getDagBundleFiles({ bundleName, limit, offset }) as TData, ...options });
|
||||
/**
|
||||
* Get Dag Bundle Count
|
||||
* Count the Dag bundles holding a Dag or an import error the caller may read.
|
||||
*
|
||||
* Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI can tell whether a
|
||||
* bundle name is worth showing next to an import error. Stale Dags count, and so do import errors
|
||||
* in files that never registered a Dag, which ``GET /importErrors`` shows to holders of
|
||||
* ``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import is where the name
|
||||
* matters most.
|
||||
* @returns DagBundleCountResponse Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
export const useDagBundleServiceGetDagBundleCount = <TData = Common.DagBundleServiceGetDagBundleCountDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>(queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseDagBundleServiceGetDagBundleCountKeyFn(queryKey), queryFn: () => DagBundleService.getDagBundleCount() as TData, ...options });
|
||||
/**
|
||||
* Get Dag Stats
|
||||
* Get Dag statistics.
|
||||
* @param data The data for the request.
|
||||
|
||||
@@ -630,11 +630,8 @@ export const useDagSourceServiceGetDagSourceSuspense = <TData = Common.DagSource
|
||||
* Reports the version of each bundle Airflow currently holds and when a Dag processor last
|
||||
* refreshed it, so whoever deployed the code can tell whether it has been picked up yet.
|
||||
*
|
||||
* A bundle is visible to a user who can read at least one Dag recorded against it. A bundle
|
||||
* holding no registered Dag at all has nothing to authorize against, so it is listed only for a
|
||||
* user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that bundle's team --
|
||||
* the same view, scoped the same way, that governs import errors for a file that never
|
||||
* registered a Dag.
|
||||
* Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked against each
|
||||
* bundle's team, and bundles of a team the caller cannot see are left out.
|
||||
* @param data The data for the request.
|
||||
* @param data.limit
|
||||
* @param data.offset
|
||||
@@ -662,10 +659,9 @@ export const useDagBundleServiceGetDagBundleSuspense = <TData = Common.DagBundle
|
||||
* Get Dag Bundle Files
|
||||
* List the files in a Dag bundle, ordered by path.
|
||||
*
|
||||
* A file is listed when the caller can read at least one Dag it defines. A file that recorded an
|
||||
* import error without registering any Dag is listed too, on the same admin-by-default terms as
|
||||
* ``GET /importErrors`` -- a file that fails before defining a Dag has no Dag to authorize on,
|
||||
* and it is the case this page most needs to show.
|
||||
* Every file the bundle holds is listed, with counts covering the whole file: the bundle is
|
||||
* authorized as a unit, so its contents are not narrowed to the Dags the caller may read. What a
|
||||
* file's Dags contain, and the text of its import error, stay behind their own permissions.
|
||||
*
|
||||
* ``dag_count`` counts live Dags only. Dag rows are never deleted: a file that fails to import
|
||||
* has every Dag in it marked stale, and so does a file dropped from the bundle. A file therefore
|
||||
@@ -688,6 +684,19 @@ export const useDagBundleServiceGetDagBundleFilesSuspense = <TData = Common.DagB
|
||||
offset?: number;
|
||||
}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseDagBundleServiceGetDagBundleFilesKeyFn({ bundleName, limit, offset }, queryKey), queryFn: () => DagBundleService.getDagBundleFiles({ bundleName, limit, offset }) as TData, ...options });
|
||||
/**
|
||||
* Get Dag Bundle Count
|
||||
* Count the Dag bundles holding a Dag or an import error the caller may read.
|
||||
*
|
||||
* Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI can tell whether a
|
||||
* bundle name is worth showing next to an import error. Stale Dags count, and so do import errors
|
||||
* in files that never registered a Dag, which ``GET /importErrors`` shows to holders of
|
||||
* ``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import is where the name
|
||||
* matters most.
|
||||
* @returns DagBundleCountResponse Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
export const useDagBundleServiceGetDagBundleCountSuspense = <TData = Common.DagBundleServiceGetDagBundleCountDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>(queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseDagBundleServiceGetDagBundleCountKeyFn(queryKey), queryFn: () => DagBundleService.getDagBundleCount() as TData, ...options });
|
||||
/**
|
||||
* Get Dag Stats
|
||||
* Get Dag statistics.
|
||||
* @param data The data for the request.
|
||||
|
||||
@@ -4867,7 +4867,7 @@ export const $DagBundleDetailResponse = {
|
||||
}
|
||||
],
|
||||
title: 'Bundle Url',
|
||||
description: 'A link to view the bundle at ``version``, when one is configured and the caller may read Dag versions.'
|
||||
description: 'A link to view the bundle at ``version``. Null when the bundle has no URL template, when the template needs a version and the bundle has not been refreshed yet, or when the template cannot be rendered.'
|
||||
},
|
||||
team_name: {
|
||||
anyOf: [
|
||||
@@ -4882,21 +4882,14 @@ export const $DagBundleDetailResponse = {
|
||||
description: 'The team owning the bundle, in a multi-team deployment.'
|
||||
},
|
||||
import_error_count: {
|
||||
anyOf: [
|
||||
{
|
||||
type: 'integer'
|
||||
},
|
||||
{
|
||||
type: 'null'
|
||||
}
|
||||
],
|
||||
type: 'integer',
|
||||
title: 'Import Error Count',
|
||||
description: 'Number of Dag import errors recorded against this bundle that the caller is permitted to see, counted on the same terms as ``GET /importErrors``. Null when the caller may not read import errors.'
|
||||
description: 'Number of Dag import errors recorded against the bundle. Reading the errors themselves is authorized separately, through ``GET /importErrors``.'
|
||||
},
|
||||
dag_count: {
|
||||
type: 'integer',
|
||||
title: 'Dag Count',
|
||||
description: 'Number of live Dags recorded against the bundle that the caller is permitted to see, counted on the same terms as ``GET /dags``.'
|
||||
description: 'Number of live Dags recorded against the bundle, including Dags the caller may not read.'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
@@ -4934,7 +4927,7 @@ export const $DagBundleFileResponse = {
|
||||
dag_count: {
|
||||
type: 'integer',
|
||||
title: 'Dag Count',
|
||||
description: 'Number of live Dags the file defines that the caller may read.'
|
||||
description: 'Number of live Dags the file defines, including Dags the caller may not read.'
|
||||
},
|
||||
last_parsed_time: {
|
||||
anyOf: [
|
||||
@@ -4962,16 +4955,9 @@ export const $DagBundleFileResponse = {
|
||||
description: 'How long the last successful parse of the file took, in seconds.'
|
||||
},
|
||||
import_error_count: {
|
||||
anyOf: [
|
||||
{
|
||||
type: 'integer'
|
||||
},
|
||||
{
|
||||
type: 'null'
|
||||
}
|
||||
],
|
||||
type: 'integer',
|
||||
title: 'Import Error Count',
|
||||
description: 'Number of import errors recorded against the file, which is at most one. Null when the caller may not read import errors -- deliberately not zero, which would read as a file with nothing wrong.'
|
||||
description: 'Number of import errors recorded against the file, which is at most one. Reading the error itself is authorized separately, through ``GET /importErrors``.'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
@@ -5033,7 +5019,7 @@ export const $DagBundleResponse = {
|
||||
}
|
||||
],
|
||||
title: 'Bundle Url',
|
||||
description: 'A link to view the bundle at ``version``, when one is configured and the caller may read Dag versions.'
|
||||
description: 'A link to view the bundle at ``version``. Null when the bundle has no URL template, when the template needs a version and the bundle has not been refreshed yet, or when the template cannot be rendered.'
|
||||
},
|
||||
team_name: {
|
||||
anyOf: [
|
||||
@@ -5048,16 +5034,9 @@ export const $DagBundleResponse = {
|
||||
description: 'The team owning the bundle, in a multi-team deployment.'
|
||||
},
|
||||
import_error_count: {
|
||||
anyOf: [
|
||||
{
|
||||
type: 'integer'
|
||||
},
|
||||
{
|
||||
type: 'null'
|
||||
}
|
||||
],
|
||||
type: 'integer',
|
||||
title: 'Import Error Count',
|
||||
description: 'Number of Dag import errors recorded against this bundle that the caller is permitted to see, counted on the same terms as ``GET /importErrors``. Null when the caller may not read import errors.'
|
||||
description: 'Number of Dag import errors recorded against the bundle. Reading the errors themselves is authorized separately, through ``GET /importErrors``.'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
@@ -12271,6 +12250,19 @@ export const $DAGsRunStateCountsCollectionResponse = {
|
||||
description: 'Collection of per-Dag DagRun-state counts for the Dag list page.'
|
||||
} as const;
|
||||
|
||||
export const $DagBundleCountResponse = {
|
||||
properties: {
|
||||
total_entries: {
|
||||
type: 'integer',
|
||||
title: 'Total Entries'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['total_entries'],
|
||||
title: 'DagBundleCountResponse',
|
||||
description: 'Number of Dag bundles holding a Dag or an import error the caller may read.'
|
||||
} as const;
|
||||
|
||||
export const $DagFolderCollectionResponse = {
|
||||
properties: {
|
||||
folders: {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import type { CancelablePromise } from './core/CancelablePromise';
|
||||
import { OpenAPI } from './core/OpenAPI';
|
||||
import { request as __request } from './core/request';
|
||||
import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, DeleteDagAssetQueuedEventsData, DeleteDagAssetQueuedEventsResponse, GetDagAssetQueuedEventData, GetDagAssetQueuedEventResponse, DeleteDagAssetQueuedEventData, DeleteDagAssetQueuedEventResponse, GetAssetsUiData, GetAssetsUiResponse, NextRunAssetsData, NextRunAssetsResponse2, ListBackfillsData, ListBackfillsResponse, CreateBackfillData, CreateBackfillResponse, GetBackfillData, GetBackfillResponse, ListBackfillDagRunsData, ListBackfillDagRunsResponse, PauseBackfillData, PauseBackfillResponse, UnpauseBackfillData, UnpauseBackfillResponse, CancelBackfillData, CancelBackfillResponse, CreateBackfillDryRunData, CreateBackfillDryRunResponse, ListBackfillsUiData, ListBackfillsUiResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, PatchConnectionData, PatchConnectionResponse, GetConnectionTestData, GetConnectionTestResponse, EnqueueConnectionTestData, EnqueueConnectionTestResponse, GetConnectionsData, GetConnectionsResponse, PostConnectionData, PostConnectionResponse, BulkConnectionsData, BulkConnectionsResponse, TestConnectionData, TestConnectionResponse, CreateDefaultConnectionsResponse, HookMetaDataResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, PatchDagRunData, PatchDagRunResponse, BulkDagRunsData, BulkDagRunsResponse, GetDagRunsData, GetDagRunsResponse, TriggerDagRunData, TriggerDagRunResponse, GetUpstreamAssetEventsData, GetUpstreamAssetEventsResponse, ClearDagRunData, ClearDagRunResponse, WaitDagRunUntilFinishedData, WaitDagRunUntilFinishedResponse, GetListDagRunsBatchData, GetListDagRunsBatchResponse, ClearDagRunsData, ClearDagRunsResponse, ClearDagRunPartitionsData, ClearDagRunPartitionsResponse, GetDagRunStatsData, GetDagRunStatsResponse, GetExecutionData, GetExecutionResponse, GetDagVersionDiffData, GetDagVersionDiffResponse, GetDagSourceData, GetDagSourceResponse, GetDagBundlesData, GetDagBundlesResponse, GetDagBundleData, GetDagBundleResponse, GetDagBundleFilesData, GetDagBundleFilesResponse, GetDagStatsData, GetDagStatsResponse, GetConfigData, GetConfigResponse, GetConfigValueData, GetConfigValueResponse, GetConfigsResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, GetDagsResponse, PatchDagsData, PatchDagsResponse, GetDagData, GetDagResponse, PatchDagData, PatchDagResponse, DeleteDagData, DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, BulkDagsData, BulkDagsResponse, FavoriteDagData, FavoriteDagResponse, UnfavoriteDagData, UnfavoriteDagResponse, GetDagTagsData, GetDagTagsResponse, GetDagsUiData, GetDagsUiResponse, GetDagTimetableTypesUiData, GetDagTimetableTypesUiResponse, GetDagFoldersResponse, GetLatestRunInfoData, GetLatestRunInfoResponse, GetDagRunStateCountsUiData, GetDagRunStateCountsUiResponse, GetRecentTaskInstanceStateCountsUiData, GetRecentTaskInstanceStateCountsUiResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, GetEventLogsResponse, GetExtraLinksData, GetExtraLinksResponse, GetTaskInstanceData, GetTaskInstanceResponse, PatchTaskInstanceData, PatchTaskInstanceResponse, DeleteTaskInstanceData, DeleteTaskInstanceResponse, GetMappedTaskInstancesData, GetMappedTaskInstancesResponse, GetTaskInstanceDependenciesByMapIndexData, GetTaskInstanceDependenciesByMapIndexResponse, GetTaskInstanceDependenciesData, GetTaskInstanceDependenciesResponse, GetTaskInstanceTriesData, GetTaskInstanceTriesResponse, GetMappedTaskInstanceTriesData, GetMappedTaskInstanceTriesResponse, GetMappedTaskInstanceData, GetMappedTaskInstanceResponse, PatchTaskInstanceByMapIndexData, PatchTaskInstanceByMapIndexResponse, GetTaskInstancesData, GetTaskInstancesResponse, BulkTaskInstancesData, BulkTaskInstancesResponse, GetTaskInstancesBatchData, GetTaskInstancesBatchResponse, GetTaskInstanceTryDetailsData, GetTaskInstanceTryDetailsResponse, GetMappedTaskInstanceTryDetailsData, GetMappedTaskInstanceTryDetailsResponse, PostClearTaskInstancesData, PostClearTaskInstancesResponse, PatchTaskGroupInstancesData, PatchTaskGroupInstancesResponse, PatchTaskGroupInstancesDryRunData, PatchTaskGroupInstancesDryRunResponse, PatchTaskInstanceDryRunByMapIndexData, PatchTaskInstanceDryRunByMapIndexResponse, PatchTaskInstanceDryRunData, PatchTaskInstanceDryRunResponse, GetLogData, GetLogResponse, GetExternalLogUrlData, GetExternalLogUrlResponse, UpdateHitlDetailData, UpdateHitlDetailResponse, GetHitlDetailData, GetHitlDetailResponse, GetHitlDetailTryDetailData, GetHitlDetailTryDetailResponse, GetHitlDetailsData, GetHitlDetailsResponse, GetImportLine truncated
|
||||
import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, DeleteDagAssetQueuedEventsData, DeleteDagAssetQueuedEventsResponse, GetDagAssetQueuedEventData, GetDagAssetQueuedEventResponse, DeleteDagAssetQueuedEventData, DeleteDagAssetQueuedEventResponse, GetAssetsUiData, GetAssetsUiResponse, NextRunAssetsData, NextRunAssetsResponse2, ListBackfillsData, ListBackfillsResponse, CreateBackfillData, CreateBackfillResponse, GetBackfillData, GetBackfillResponse, ListBackfillDagRunsData, ListBackfillDagRunsResponse, PauseBackfillData, PauseBackfillResponse, UnpauseBackfillData, UnpauseBackfillResponse, CancelBackfillData, CancelBackfillResponse, CreateBackfillDryRunData, CreateBackfillDryRunResponse, ListBackfillsUiData, ListBackfillsUiResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, PatchConnectionData, PatchConnectionResponse, GetConnectionTestData, GetConnectionTestResponse, EnqueueConnectionTestData, EnqueueConnectionTestResponse, GetConnectionsData, GetConnectionsResponse, PostConnectionData, PostConnectionResponse, BulkConnectionsData, BulkConnectionsResponse, TestConnectionData, TestConnectionResponse, CreateDefaultConnectionsResponse, HookMetaDataResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, PatchDagRunData, PatchDagRunResponse, BulkDagRunsData, BulkDagRunsResponse, GetDagRunsData, GetDagRunsResponse, TriggerDagRunData, TriggerDagRunResponse, GetUpstreamAssetEventsData, GetUpstreamAssetEventsResponse, ClearDagRunData, ClearDagRunResponse, WaitDagRunUntilFinishedData, WaitDagRunUntilFinishedResponse, GetListDagRunsBatchData, GetListDagRunsBatchResponse, ClearDagRunsData, ClearDagRunsResponse, ClearDagRunPartitionsData, ClearDagRunPartitionsResponse, GetDagRunStatsData, GetDagRunStatsResponse, GetExecutionData, GetExecutionResponse, GetDagVersionDiffData, GetDagVersionDiffResponse, GetDagSourceData, GetDagSourceResponse, GetDagBundlesData, GetDagBundlesResponse, GetDagBundleData, GetDagBundleResponse, GetDagBundleFilesData, GetDagBundleFilesResponse, GetDagBundleCountResponse, GetDagStatsData, GetDagStatsResponse, GetConfigData, GetConfigResponse, GetConfigValueData, GetConfigValueResponse, GetConfigsResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, GetDagsResponse, PatchDagsData, PatchDagsResponse, GetDagData, GetDagResponse, PatchDagData, PatchDagResponse, DeleteDagData, DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, BulkDagsData, BulkDagsResponse, FavoriteDagData, FavoriteDagResponse, UnfavoriteDagData, UnfavoriteDagResponse, GetDagTagsData, GetDagTagsResponse, GetDagsUiData, GetDagsUiResponse, GetDagTimetableTypesUiData, GetDagTimetableTypesUiResponse, GetDagFoldersResponse, GetLatestRunInfoData, GetLatestRunInfoResponse, GetDagRunStateCountsUiData, GetDagRunStateCountsUiResponse, GetRecentTaskInstanceStateCountsUiData, GetRecentTaskInstanceStateCountsUiResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, GetEventLogsResponse, GetExtraLinksData, GetExtraLinksResponse, GetTaskInstanceData, GetTaskInstanceResponse, PatchTaskInstanceData, PatchTaskInstanceResponse, DeleteTaskInstanceData, DeleteTaskInstanceResponse, GetMappedTaskInstancesData, GetMappedTaskInstancesResponse, GetTaskInstanceDependenciesByMapIndexData, GetTaskInstanceDependenciesByMapIndexResponse, GetTaskInstanceDependenciesData, GetTaskInstanceDependenciesResponse, GetTaskInstanceTriesData, GetTaskInstanceTriesResponse, GetMappedTaskInstanceTriesData, GetMappedTaskInstanceTriesResponse, GetMappedTaskInstanceData, GetMappedTaskInstanceResponse, PatchTaskInstanceByMapIndexData, PatchTaskInstanceByMapIndexResponse, GetTaskInstancesData, GetTaskInstancesResponse, BulkTaskInstancesData, BulkTaskInstancesResponse, GetTaskInstancesBatchData, GetTaskInstancesBatchResponse, GetTaskInstanceTryDetailsData, GetTaskInstanceTryDetailsResponse, GetMappedTaskInstanceTryDetailsData, GetMappedTaskInstanceTryDetailsResponse, PostClearTaskInstancesData, PostClearTaskInstancesResponse, PatchTaskGroupInstancesData, PatchTaskGroupInstancesResponse, PatchTaskGroupInstancesDryRunData, PatchTaskGroupInstancesDryRunResponse, PatchTaskInstanceDryRunByMapIndexData, PatchTaskInstanceDryRunByMapIndexResponse, PatchTaskInstanceDryRunData, PatchTaskInstanceDryRunResponse, GetLogData, GetLogResponse, GetExternalLogUrlData, GetExternalLogUrlResponse, UpdateHitlDetailData, UpdateHitlDetailResponse, GetHitlDetailData, GetHitlDetailResponse, GetHitlDetailTryDetailData, GetHitlDetailTryDetailResponse, GetHitlDetailsData, GetHitLine truncated
|
||||
|
||||
export class AssetService {
|
||||
/**
|
||||
@@ -1666,11 +1666,8 @@ export class DagBundleService {
|
||||
* Reports the version of each bundle Airflow currently holds and when a Dag processor last
|
||||
* refreshed it, so whoever deployed the code can tell whether it has been picked up yet.
|
||||
*
|
||||
* A bundle is visible to a user who can read at least one Dag recorded against it. A bundle
|
||||
* holding no registered Dag at all has nothing to authorize against, so it is listed only for a
|
||||
* user holding the admin-by-default ``AccessView.IMPORT_ERRORS_ALL`` for that bundle's team --
|
||||
* the same view, scoped the same way, that governs import errors for a file that never
|
||||
* registered a Dag.
|
||||
* Needs ``AccessView.DAG_BUNDLES``. In a multi-team deployment it is checked against each
|
||||
* bundle's team, and bundles of a team the caller cannot see are left out.
|
||||
* @param data The data for the request.
|
||||
* @param data.limit
|
||||
* @param data.offset
|
||||
@@ -1723,10 +1720,9 @@ export class DagBundleService {
|
||||
* Get Dag Bundle Files
|
||||
* List the files in a Dag bundle, ordered by path.
|
||||
*
|
||||
* A file is listed when the caller can read at least one Dag it defines. A file that recorded an
|
||||
* import error without registering any Dag is listed too, on the same admin-by-default terms as
|
||||
* ``GET /importErrors`` -- a file that fails before defining a Dag has no Dag to authorize on,
|
||||
* and it is the case this page most needs to show.
|
||||
* Every file the bundle holds is listed, with counts covering the whole file: the bundle is
|
||||
* authorized as a unit, so its contents are not narrowed to the Dags the caller may read. What a
|
||||
* file's Dags contain, and the text of its import error, stay behind their own permissions.
|
||||
*
|
||||
* ``dag_count`` counts live Dags only. Dag rows are never deleted: a file that fails to import
|
||||
* has every Dag in it marked stale, and so does a file dropped from the bundle. A file therefore
|
||||
@@ -1763,6 +1759,25 @@ export class DagBundleService {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Dag Bundle Count
|
||||
* Count the Dag bundles holding a Dag or an import error the caller may read.
|
||||
*
|
||||
* Readable by anyone who can read Dags, unlike ``GET /dagBundles``, so the UI can tell whether a
|
||||
* bundle name is worth showing next to an import error. Stale Dags count, and so do import errors
|
||||
* in files that never registered a Dag, which ``GET /importErrors`` shows to holders of
|
||||
* ``AccessView.IMPORT_ERRORS_ALL``: a bundle whose files all fail to import is where the name
|
||||
* matters most.
|
||||
* @returns DagBundleCountResponse Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static getDagBundleCount(): CancelablePromise<GetDagBundleCountResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/ui/dagBundles/count'
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class DagStatsService {
|
||||
|
||||
@@ -1262,7 +1262,7 @@ export type DagBundleDetailResponse = {
|
||||
*/
|
||||
last_refreshed: string | null;
|
||||
/**
|
||||
* A link to view the bundle at ``version``, when one is configured and the caller may read Dag versions.
|
||||
* A link to view the bundle at ``version``. Null when the bundle has no URL template, when the template needs a version and the bundle has not been refreshed yet, or when the template cannot be rendered.
|
||||
*/
|
||||
bundle_url: string | null;
|
||||
/**
|
||||
@@ -1270,11 +1270,11 @@ export type DagBundleDetailResponse = {
|
||||
*/
|
||||
team_name: string | null;
|
||||
/**
|
||||
* Number of Dag import errors recorded against this bundle that the caller is permitted to see, counted on the same terms as ``GET /importErrors``. Null when the caller may not read import errors.
|
||||
* Number of Dag import errors recorded against the bundle. Reading the errors themselves is authorized separately, through ``GET /importErrors``.
|
||||
*/
|
||||
import_error_count: number | null;
|
||||
import_error_count: number;
|
||||
/**
|
||||
* Number of live Dags recorded against the bundle that the caller is permitted to see, counted on the same terms as ``GET /dags``.
|
||||
* Number of live Dags recorded against the bundle, including Dags the caller may not read.
|
||||
*/
|
||||
dag_count: number;
|
||||
};
|
||||
@@ -1293,7 +1293,7 @@ export type DagBundleFileCollectionResponse = {
|
||||
export type DagBundleFileResponse = {
|
||||
relative_fileloc: string;
|
||||
/**
|
||||
* Number of live Dags the file defines that the caller may read.
|
||||
* Number of live Dags the file defines, including Dags the caller may not read.
|
||||
*/
|
||||
dag_count: number;
|
||||
/**
|
||||
@@ -1305,9 +1305,9 @@ export type DagBundleFileResponse = {
|
||||
*/
|
||||
last_parse_duration: number | null;
|
||||
/**
|
||||
* Number of import errors recorded against the file, which is at most one. Null when the caller may not read import errors -- deliberately not zero, which would read as a file with nothing wrong.
|
||||
* Number of import errors recorded against the file, which is at most one. Reading the error itself is authorized separately, through ``GET /importErrors``.
|
||||
*/
|
||||
import_error_count: number | null;
|
||||
import_error_count: number;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1328,7 +1328,7 @@ export type DagBundleResponse = {
|
||||
*/
|
||||
last_refreshed: string | null;
|
||||
/**
|
||||
* A link to view the bundle at ``version``, when one is configured and the caller may read Dag versions.
|
||||
* A link to view the bundle at ``version``. Null when the bundle has no URL template, when the template needs a version and the bundle has not been refreshed yet, or when the template cannot be rendered.
|
||||
*/
|
||||
bundle_url: string | null;
|
||||
/**
|
||||
@@ -1336,9 +1336,9 @@ export type DagBundleResponse = {
|
||||
*/
|
||||
team_name: string | null;
|
||||
/**
|
||||
* Number of Dag import errors recorded against this bundle that the caller is permitted to see, counted on the same terms as ``GET /importErrors``. Null when the caller may not read import errors.
|
||||
* Number of Dag import errors recorded against the bundle. Reading the errors themselves is authorized separately, through ``GET /importErrors``.
|
||||
*/
|
||||
import_error_count: number | null;
|
||||
import_error_count: number;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -3101,6 +3101,13 @@ export type DAGsRunStateCountsCollectionResponse = {
|
||||
state_count_limit: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Number of Dag bundles holding a Dag or an import error the caller may read.
|
||||
*/
|
||||
export type DagBundleCountResponse = {
|
||||
total_entries: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Collection of distinct Dag folders, each scoped to the bundle it belongs to.
|
||||
*/
|
||||
@@ -4318,6 +4325,8 @@ export type GetDagBundleFilesData = {
|
||||
|
||||
export type GetDagBundleFilesResponse = DagBundleFileCollectionResponse;
|
||||
|
||||
export type GetDagBundleCountResponse = DagBundleCountResponse;
|
||||
|
||||
export type GetDagStatsData = {
|
||||
dagIds?: Array<(string)>;
|
||||
};
|
||||
@@ -7357,6 +7366,16 @@ export type $OpenApiTs = {
|
||||
};
|
||||
};
|
||||
};
|
||||
'/ui/dagBundles/count': {
|
||||
get: {
|
||||
res: {
|
||||
/**
|
||||
* Successful Response
|
||||
*/
|
||||
200: DagBundleCountResponse;
|
||||
};
|
||||
};
|
||||
};
|
||||
'/api/v2/dagStats': {
|
||||
get: {
|
||||
req: GetDagStatsData;
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
},
|
||||
"file_one": "File",
|
||||
"file_other": "Files",
|
||||
"importErrorForbidden": "You don't have permission to read this import error.",
|
||||
"importErrorGone": "This import error is no longer recorded. It was probably fixed by a reparse.",
|
||||
"neverParsed": "Never",
|
||||
"showImportError_one": "View error",
|
||||
|
||||
@@ -53,11 +53,7 @@ describe("DagImportErrorModal", () => {
|
||||
{ shown: false, totalEntries: 1 },
|
||||
{ shown: true, totalEntries: 2 },
|
||||
])("shows the bundle name only with several bundles ($totalEntries)", async ({ shown, totalEntries }) => {
|
||||
server.use(
|
||||
http.get("/api/v2/dagBundles", () =>
|
||||
HttpResponse.json({ dag_bundles: [], total_entries: totalEntries }),
|
||||
),
|
||||
);
|
||||
server.use(http.get("/ui/dagBundles/count", () => HttpResponse.json({ total_entries: totalEntries })));
|
||||
|
||||
render(<DagImportErrorModal importError={importError} onClose={vi.fn()} open />, { wrapper: Wrapper });
|
||||
|
||||
|
||||
@@ -22,22 +22,13 @@ import { useTranslation } from "react-i18next";
|
||||
import { formatNumber } from "src/utils";
|
||||
|
||||
type Props = {
|
||||
readonly count: number | null;
|
||||
readonly count: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* An import-error count, flagged red only when there is something to flag.
|
||||
*
|
||||
* Null is not zero: it means the caller may not read import errors, so it renders as unknown
|
||||
* rather than as a clean bill of health.
|
||||
*/
|
||||
/** An import-error count, flagged red only when there is something to flag. */
|
||||
export const ImportErrorCount = ({ count }: Props) => {
|
||||
const { i18n } = useTranslation();
|
||||
|
||||
if (count === null) {
|
||||
return <Text color="fg.muted">-</Text>;
|
||||
}
|
||||
|
||||
return count === 0 ? (
|
||||
<Text color="fg.muted">0</Text>
|
||||
) : (
|
||||
|
||||
@@ -24,6 +24,7 @@ export const handlers: Array<HttpHandler> = [
|
||||
http.get("/api/v2/connections", () => HttpResponse.json({ connections: [], total_entries: 0 })),
|
||||
http.get("/ui/connections/hook_meta", () => HttpResponse.json([])),
|
||||
http.get("/api/v2/dagBundles", () => HttpResponse.json({ dag_bundles: [], total_entries: 0 })),
|
||||
http.get("/ui/dagBundles/count", () => HttpResponse.json({ total_entries: 0 })),
|
||||
http.get("/api/v2/eventLogs", () => HttpResponse.json({ event_logs: [], total_entries: 0 })),
|
||||
http.get("/api/v2/jobs", () => HttpResponse.json({ jobs: [], total_entries: 0 })),
|
||||
http.get("/api/v2/pools", () => HttpResponse.json({ pools: [], total_entries: 0 })),
|
||||
|
||||
@@ -83,7 +83,7 @@ const createColumns = (
|
||||
cell: ({ row: { original } }: FileRow) =>
|
||||
// Only worth clicking when there is an error behind the count. A bare badge reads as a
|
||||
// status rather than a control, so the clickable form carries an icon and its own label.
|
||||
original.import_error_count === null || original.import_error_count === 0 ? (
|
||||
original.import_error_count === 0 ? (
|
||||
<ImportErrorCount count={original.import_error_count} />
|
||||
) : (
|
||||
<Button
|
||||
@@ -113,7 +113,7 @@ export const BundleFiles = ({ bundleName }: { readonly bundleName: string }) =>
|
||||
const { pagination } = tableURLState;
|
||||
const [errorFileloc, setErrorFileloc] = useState<string | undefined>(undefined);
|
||||
|
||||
const { data, error, isLoading } = useDagBundleServiceGetDagBundleFiles(
|
||||
const { data, error, isLoading, refetch } = useDagBundleServiceGetDagBundleFiles(
|
||||
{
|
||||
bundleName,
|
||||
limit: pagination.pageSize,
|
||||
@@ -127,9 +127,14 @@ export const BundleFiles = ({ bundleName }: { readonly bundleName: string }) =>
|
||||
<Box pt={2}>
|
||||
<FileImportError
|
||||
bundleName={bundleName}
|
||||
hasImportError={
|
||||
(data?.dag_bundle_files.find((file) => file.relative_fileloc === errorFileloc)
|
||||
?.import_error_count ?? 0) > 0
|
||||
}
|
||||
onClose={() => {
|
||||
setErrorFileloc(undefined);
|
||||
}}
|
||||
recheckImportError={refetch}
|
||||
relativeFileloc={errorFileloc}
|
||||
/>
|
||||
<DataTable
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
/*!
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { useState } from "react";
|
||||
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import "@testing-library/jest-dom/vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { http, HttpResponse } from "msw";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { server } from "src/mocks/server";
|
||||
import { Wrapper } from "src/utils/Wrapper";
|
||||
|
||||
import { FileImportError } from "./FileImportError";
|
||||
|
||||
const FORBIDDEN = "dagBundles.files.importErrorForbidden";
|
||||
const GONE = "dagBundles.files.importErrorGone";
|
||||
|
||||
/** Renders the dialog for a file whose row shows an error until a recheck, which a fixed file clears. */
|
||||
const Harness = ({
|
||||
fixedOnRecheck,
|
||||
hasImportError,
|
||||
onRecheck,
|
||||
}: {
|
||||
readonly fixedOnRecheck: boolean;
|
||||
readonly hasImportError: boolean;
|
||||
readonly onRecheck: () => void;
|
||||
}) => {
|
||||
const [rowHasError, setRowHasError] = useState(hasImportError);
|
||||
|
||||
return (
|
||||
<FileImportError
|
||||
bundleName="dags-folder"
|
||||
hasImportError={rowHasError}
|
||||
onClose={vi.fn()}
|
||||
recheckImportError={() => {
|
||||
onRecheck();
|
||||
if (fixedOnRecheck) {
|
||||
setRowHasError(false);
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
}}
|
||||
relativeFileloc="broken.py"
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const IMPORT_ERROR = {
|
||||
bundle_name: "dags-folder",
|
||||
filename: "broken.py",
|
||||
import_error_id: 1,
|
||||
stack_trace: "Traceback: broken again",
|
||||
timestamp: "2026-11-02T00:00:00Z",
|
||||
};
|
||||
|
||||
const renderWith = ({
|
||||
fixedOnRecheck = false,
|
||||
hasImportError,
|
||||
reappearsOnRecheck = false,
|
||||
status,
|
||||
}: {
|
||||
readonly fixedOnRecheck?: boolean;
|
||||
readonly hasImportError: boolean;
|
||||
readonly reappearsOnRecheck?: boolean;
|
||||
readonly status: number;
|
||||
}) => {
|
||||
let requests = 0;
|
||||
|
||||
server.use(
|
||||
http.get("/api/v2/importErrors", () => {
|
||||
requests += 1;
|
||||
if (status !== 200) {
|
||||
return HttpResponse.json({ detail: "boom" }, { status });
|
||||
}
|
||||
const importErrors = reappearsOnRecheck && requests > 1 ? [IMPORT_ERROR] : [];
|
||||
|
||||
return HttpResponse.json({ import_errors: importErrors, total_entries: importErrors.length });
|
||||
}),
|
||||
);
|
||||
const onRecheck = vi.fn();
|
||||
|
||||
render(
|
||||
// The app does not retry a 4xx; the shared wrapper's client would, and outlast the assertion.
|
||||
<QueryClientProvider client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}>
|
||||
<Harness fixedOnRecheck={fixedOnRecheck} hasImportError={hasImportError} onRecheck={onRecheck} />
|
||||
</QueryClientProvider>,
|
||||
{ wrapper: Wrapper },
|
||||
);
|
||||
|
||||
return onRecheck;
|
||||
};
|
||||
|
||||
// i18n resources are not loaded in unit tests, so messages render as their keys.
|
||||
describe("FileImportError", () => {
|
||||
it("says forbidden on a 403 without rechecking the file list", async () => {
|
||||
const onRecheck = renderWith({ hasImportError: true, status: 403 });
|
||||
|
||||
expect(await screen.findByText(FORBIDDEN)).toBeInTheDocument();
|
||||
expect(onRecheck).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("says forbidden when the file still counts the error after a recheck", async () => {
|
||||
const onRecheck = renderWith({ hasImportError: true, status: 200 });
|
||||
|
||||
expect(await screen.findByText(FORBIDDEN)).toBeInTheDocument();
|
||||
expect(onRecheck).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("says gone when the recheck shows the error was fixed since the last poll", async () => {
|
||||
const onRecheck = renderWith({ fixedOnRecheck: true, hasImportError: true, status: 200 });
|
||||
|
||||
expect(await screen.findByText(GONE)).toBeInTheDocument();
|
||||
expect(onRecheck).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows an error that came back since an empty result was fetched", async () => {
|
||||
const onRecheck = renderWith({ hasImportError: true, reappearsOnRecheck: true, status: 200 });
|
||||
|
||||
expect(await screen.findByText(IMPORT_ERROR.stack_trace)).toBeInTheDocument();
|
||||
expect(onRecheck).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows other failures as errors rather than as a missing permission", async () => {
|
||||
renderWith({ hasImportError: true, status: 500 });
|
||||
|
||||
expect(await screen.findByRole("dialog")).toBeInTheDocument();
|
||||
expect(screen.queryByText(FORBIDDEN)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(GONE)).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,8 @@
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import { Heading, HStack, Text } from "@chakra-ui/react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { LuFileWarning } from "react-icons/lu";
|
||||
@@ -25,10 +27,15 @@ import { useImportErrorServiceGetImportErrors } from "openapi/queries";
|
||||
import { Modal } from "src/system-components";
|
||||
|
||||
import { DagImportErrorModal } from "src/components/DagImportErrorModal";
|
||||
import { ErrorAlert } from "src/components/ErrorAlert";
|
||||
|
||||
import { getErrorStatus } from "src/utils/errorHandling";
|
||||
|
||||
type Props = {
|
||||
readonly bundleName: string;
|
||||
readonly hasImportError: boolean;
|
||||
readonly onClose: () => void;
|
||||
readonly recheckImportError: () => Promise<unknown>;
|
||||
readonly relativeFileloc: string | undefined;
|
||||
};
|
||||
|
||||
@@ -40,10 +47,17 @@ type Props = {
|
||||
* error. A dialog rather than a page of its own, because a single stack trace is not worth an
|
||||
* addressable route.
|
||||
*/
|
||||
export const FileImportError = ({ bundleName, onClose, relativeFileloc }: Props) => {
|
||||
export const FileImportError = ({
|
||||
bundleName,
|
||||
hasImportError,
|
||||
onClose,
|
||||
recheckImportError,
|
||||
relativeFileloc,
|
||||
}: Props) => {
|
||||
const { t: translate } = useTranslation("browse");
|
||||
const [recheckedFileloc, setRecheckedFileloc] = useState<string | undefined>(undefined);
|
||||
|
||||
const { data, isLoading } = useImportErrorServiceGetImportErrors(
|
||||
const { data, error, isLoading, refetch } = useImportErrorServiceGetImportErrors(
|
||||
{ bundleName, filename: relativeFileloc ?? "" },
|
||||
undefined,
|
||||
{ enabled: relativeFileloc !== undefined },
|
||||
@@ -51,14 +65,38 @@ export const FileImportError = ({ bundleName, onClose, relativeFileloc }: Props)
|
||||
|
||||
const [importError] = data?.import_errors ?? [];
|
||||
|
||||
// ``GET /importErrors`` leaves out errors the caller may not read, so an empty result is told
|
||||
// apart by the file list, whose count is not filtered by caller. The badge was clicked on a
|
||||
// count that may predate a fix, and the empty result may be cached from before the error came
|
||||
// back, so both are refetched once before the count is trusted.
|
||||
const needsRecheck =
|
||||
relativeFileloc !== undefined &&
|
||||
data !== undefined &&
|
||||
importError === undefined &&
|
||||
hasImportError &&
|
||||
recheckedFileloc !== relativeFileloc;
|
||||
|
||||
useEffect(() => {
|
||||
if (needsRecheck) {
|
||||
void Promise.all([recheckImportError(), refetch()]).finally(() => {
|
||||
setRecheckedFileloc(relativeFileloc);
|
||||
});
|
||||
}
|
||||
}, [needsRecheck, recheckImportError, refetch, relativeFileloc]);
|
||||
|
||||
const close = () => {
|
||||
setRecheckedFileloc(undefined);
|
||||
onClose();
|
||||
};
|
||||
|
||||
if (importError !== undefined) {
|
||||
return (
|
||||
<DagImportErrorModal importError={importError} onClose={onClose} open={relativeFileloc !== undefined} />
|
||||
<DagImportErrorModal importError={importError} onClose={close} open={relativeFileloc !== undefined} />
|
||||
);
|
||||
}
|
||||
|
||||
// The table polls, so the error can be gone by the time its badge is clicked. Say so rather
|
||||
// than leaving the click with no visible effect.
|
||||
const isForbidden = getErrorStatus(error) === 403 || (data !== undefined && hasImportError);
|
||||
|
||||
return (
|
||||
<Modal
|
||||
headerProps={{
|
||||
@@ -69,10 +107,18 @@ export const FileImportError = ({ bundleName, onClose, relativeFileloc }: Props)
|
||||
</HStack>
|
||||
),
|
||||
}}
|
||||
onOpenChange={onClose}
|
||||
open={relativeFileloc !== undefined && !isLoading}
|
||||
onOpenChange={close}
|
||||
open={relativeFileloc !== undefined && !isLoading && !needsRecheck}
|
||||
>
|
||||
<Text color="fg.muted">{translate("dagBundles.files.importErrorGone")}</Text>
|
||||
{error !== null && !isForbidden ? (
|
||||
<ErrorAlert error={error} />
|
||||
) : (
|
||||
<Text color="fg.muted">
|
||||
{translate(
|
||||
isForbidden ? "dagBundles.files.importErrorForbidden" : "dagBundles.files.importErrorGone",
|
||||
)}
|
||||
</Text>
|
||||
)}
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
@@ -16,11 +16,11 @@
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { useDagBundleServiceGetDagBundles } from "openapi/queries";
|
||||
import { useDagBundleServiceGetDagBundleCount } from "openapi/queries";
|
||||
|
||||
/** Whether the user can see more than one Dag bundle, i.e. whether a bundle name tells anything apart. */
|
||||
/** Whether the user can read Dags or import errors in more than one bundle, i.e. whether a bundle name tells anything apart. */
|
||||
export const useHasMultipleDagBundles = (): boolean => {
|
||||
const { data } = useDagBundleServiceGetDagBundles({ limit: 1 });
|
||||
const { data } = useDagBundleServiceGetDagBundleCount();
|
||||
|
||||
return (data?.total_entries ?? 0) > 1;
|
||||
};
|
||||
@@ -47,8 +47,6 @@ test.describe("Dag Bundles Page", () => {
|
||||
.toHaveText(/^([\da-f]{7}|not versioned|not refreshed yet)$/i);
|
||||
// Either a relative time or the "Never" fallback, but never blank.
|
||||
await expect.soft(dagBundlesPage.lastRefreshedCellAt(0)).not.toBeEmpty();
|
||||
// A number when the caller may read import errors, "-" when they may not. Admin runs these,
|
||||
// so a count is expected -- but never a blank cell either way.
|
||||
await expect.soft(dagBundlesPage.importErrorsCellAt(0)).toHaveText(/^(\d+|-)$/);
|
||||
await expect.soft(dagBundlesPage.importErrorsCellAt(0)).toHaveText(/^\d+$/);
|
||||
});
|
||||
});
|
||||
@@ -388,6 +388,23 @@ class TestSimpleAuthManager:
|
||||
is result
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("role", "result"),
|
||||
[
|
||||
("ADMIN", True),
|
||||
("OP", True),
|
||||
("USER", False),
|
||||
("VIEWER", False),
|
||||
],
|
||||
)
|
||||
def test_is_authorized_view_dag_bundles_needs_op(self, auth_manager, role, result):
|
||||
assert (
|
||||
auth_manager.is_authorized_view(
|
||||
access_view=AccessView.DAG_BUNDLES, user=SimpleAuthManagerUser(username="test", role=role)
|
||||
)
|
||||
is result
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api",
|
||||
[
|
||||
|
||||
@@ -24,7 +24,7 @@ import pytest
|
||||
from itsdangerous import URLSafeSerializer
|
||||
from sqlalchemy import insert, update
|
||||
|
||||
from airflow.api_fastapi.auth.managers.models.resource_details import AccessView, DagAccessEntity
|
||||
from airflow.api_fastapi.auth.managers.models.resource_details import AccessView
|
||||
from airflow.api_fastapi.auth.managers.simple.user import SimpleAuthManagerUser
|
||||
from airflow.configuration import conf
|
||||
from airflow.models import DagModel
|
||||
@@ -55,9 +55,8 @@ TEAM_NAME = "team_a"
|
||||
# ``dag_in_git_bundle`` is seeded at ``dag_0.py``, so this is a file with a registered Dag.
|
||||
REGISTERED_FILE = "dag_0.py"
|
||||
UNREGISTERED_FILE = "broken.py"
|
||||
# A file inside GIT_BUNDLE whose Dag the caller may NOT read. Registered, so the admin-gated
|
||||
# unregistered path cannot hide it -- only the readable-Dag filter excludes it.
|
||||
UNREADABLE_FILE = "secret.py"
|
||||
# A second registered file in GIT_BUNDLE, which also has an import error.
|
||||
SECOND_REGISTERED_FILE = "second.py"
|
||||
|
||||
GIT_VERSION = "8f0e5b1c9a2d4e6f8a0b1c2d3e4f5a6b7c8d9e0f"
|
||||
REFRESHED_AT = datetime(2026, 9, 10, 12, 0, tzinfo=UTC)
|
||||
@@ -66,11 +65,10 @@ PARSE_DURATION = 0.125
|
||||
GONE_REFRESHED_AT = datetime(2026, 9, 1, 8, 30, tzinfo=UTC)
|
||||
|
||||
WITH_DAGS = (GIT_BUNDLE, LOCAL_BUNDLE, GONE_BUNDLE, OTHER_TEAM_BUNDLE)
|
||||
# Everything except the Dag in OTHER_TEAM_BUNDLE.
|
||||
REMOVED_DAG_ID = "removed_from_dag_0"
|
||||
READABLE_DAG_IDS = {f"dag_in_{name}" for name in (GIT_BUNDLE, LOCAL_BUNDLE, GONE_BUNDLE)} | {REMOVED_DAG_ID}
|
||||
# Sorted by name, which is the endpoint's default order.
|
||||
READABLE_BUNDLES = [GIT_BUNDLE, GONE_BUNDLE, LOCAL_BUNDLE]
|
||||
ALL_BUNDLES = [DAGLESS_BUNDLE, GIT_BUNDLE, GONE_BUNDLE, LOCAL_BUNDLE, OTHER_TEAM_BUNDLE]
|
||||
GIT_BUNDLE_FILES = [UNREGISTERED_FILE, REGISTERED_FILE, SECOND_REGISTERED_FILE]
|
||||
|
||||
|
||||
def _sign(url_template: str, bundle_name: str) -> str:
|
||||
@@ -134,8 +132,7 @@ def bundles() -> Generator[None, None, None]:
|
||||
_make_bundle(LOCAL_BUNDLE, version=None, last_refreshed=REFRESHED_AT),
|
||||
_make_bundle(GONE_BUNDLE, version="deadbeef", last_refreshed=GONE_REFRESHED_AT, active=False),
|
||||
_make_bundle(OTHER_TEAM_BUNDLE, version="cafed00d", last_refreshed=REFRESHED_AT),
|
||||
# No Dag has ever parsed from this one -- the documented blind spot of deriving
|
||||
# visibility from readable Dags.
|
||||
# No Dag has ever parsed from this one, as on a first deploy whose only file fails.
|
||||
_make_bundle(DAGLESS_BUNDLE, version="0badcafe", last_refreshed=REFRESHED_AT),
|
||||
]
|
||||
)
|
||||
@@ -172,13 +169,12 @@ def bundles() -> Generator[None, None, None]:
|
||||
last_parse_duration=PARSE_DURATION * 100,
|
||||
)
|
||||
)
|
||||
# A co-located Dag in a visible bundle that the caller cannot read -- deliberately absent
|
||||
# from READABLE_DAG_IDS.
|
||||
# A second live Dag in GIT_BUNDLE, in a file of its own.
|
||||
session.add(
|
||||
DagModel(
|
||||
dag_id="unreadable_in_git_bundle",
|
||||
fileloc=UNREADABLE_FILE,
|
||||
relative_fileloc=UNREADABLE_FILE,
|
||||
dag_id="second_in_git_bundle",
|
||||
fileloc=SECOND_REGISTERED_FILE,
|
||||
relative_fileloc=SECOND_REGISTERED_FILE,
|
||||
bundle_name=GIT_BUNDLE,
|
||||
is_paused=False,
|
||||
is_stale=False,
|
||||
@@ -186,28 +182,24 @@ def bundles() -> Generator[None, None, None]:
|
||||
)
|
||||
session.add_all(
|
||||
[
|
||||
# An error in the file the bundle's readable Dag comes from: visible to anyone who
|
||||
# can read that Dag.
|
||||
# An error in a file that also registered a Dag.
|
||||
ParseImportError(
|
||||
bundle_name=GIT_BUNDLE,
|
||||
filename=REGISTERED_FILE,
|
||||
stacktrace="boom",
|
||||
timestamp=REFRESHED_AT,
|
||||
),
|
||||
# An error in a file that never registered a Dag: admin-gated, because the file's
|
||||
# existence is itself the disclosure.
|
||||
# An error in a file that never registered a Dag.
|
||||
ParseImportError(
|
||||
bundle_name=GIT_BUNDLE,
|
||||
filename=UNREGISTERED_FILE,
|
||||
stacktrace="boom before any dag",
|
||||
timestamp=REFRESHED_AT,
|
||||
),
|
||||
# Registered to a Dag the caller cannot read: excluded by the readable-Dag filter
|
||||
# alone, so it is what makes that half of the authorization testable.
|
||||
ParseImportError(
|
||||
bundle_name=GIT_BUNDLE,
|
||||
filename=UNREADABLE_FILE,
|
||||
stacktrace="not yours",
|
||||
filename=SECOND_REGISTERED_FILE,
|
||||
stacktrace="boom in a second file",
|
||||
timestamp=REFRESHED_AT,
|
||||
),
|
||||
]
|
||||
@@ -219,75 +211,54 @@ def bundles() -> Generator[None, None, None]:
|
||||
_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def readable_dag_ids(test_client):
|
||||
"""
|
||||
Restrict every caller in this module to ``READABLE_DAG_IDS``.
|
||||
|
||||
The client fixtures below share one auth manager instance, so the patch lives here: a test
|
||||
asking for two of them would otherwise autospec-patch an attribute the first fixture already
|
||||
replaced with a mock, which ``create_autospec`` rejects.
|
||||
"""
|
||||
auth_manager = test_client.app.state.auth_manager
|
||||
with mock.patch.object(
|
||||
auth_manager,
|
||||
"get_authorized_dag_ids",
|
||||
autospec=True,
|
||||
return_value=READABLE_DAG_IDS,
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_client(test_client, readable_dag_ids):
|
||||
"""
|
||||
An admin restricted to the same readable Dags.
|
||||
|
||||
``test_client`` authenticates as an admin, which satisfies ``IMPORT_ERRORS_ALL`` under
|
||||
SimpleAuthManager, so this is the caller that may also see a bundle holding no Dag at all.
|
||||
"""
|
||||
return test_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dag_scoped_client(test_client, readable_dag_ids):
|
||||
"""
|
||||
A caller who may read every Dag except the one in ``OTHER_TEAM_BUNDLE``.
|
||||
|
||||
The list filter reads the Dag ids from the app's auth manager (``AuthManagerDep``), so the
|
||||
real instance is patched rather than the ``get_auth_manager`` module lookup.
|
||||
|
||||
``test_client`` authenticates as an admin, so ``IMPORT_ERRORS_ALL`` is denied here to keep Dag
|
||||
scoping the only thing deciding which bundles are listed. A bundle holding no Dag rides on that
|
||||
admin view instead, and has its own tests.
|
||||
"""
|
||||
def _patch_dag_bundles_view(test_client, is_authorized):
|
||||
"""Decide ``AccessView.DAG_BUNDLES`` with ``is_authorized(team_name)``; other views are left as they are."""
|
||||
auth_manager = test_client.app.state.auth_manager
|
||||
real_authorize_view = auth_manager.authorize_view
|
||||
|
||||
def deny_dagless_view(*, access_view, user, team_name=None):
|
||||
if access_view is AccessView.IMPORT_ERRORS_ALL:
|
||||
return False
|
||||
def authorize_view(*, access_view, user, team_name=None):
|
||||
if access_view is AccessView.DAG_BUNDLES:
|
||||
return is_authorized(team_name)
|
||||
return real_authorize_view(access_view=access_view, user=user, team_name=team_name)
|
||||
|
||||
with mock.patch.object(auth_manager, "authorize_view", autospec=True, side_effect=deny_dagless_view):
|
||||
return mock.patch.object(auth_manager, "authorize_view", autospec=True, side_effect=authorize_view)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_dag_read_client(test_client):
|
||||
"""A caller who holds the bundle view but may read no Dag at all."""
|
||||
with mock.patch.object(
|
||||
test_client.app.state.auth_manager, "get_authorized_dag_ids", autospec=True, return_value=set()
|
||||
):
|
||||
yield test_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def viewer_headers(test_client, readable_dag_ids):
|
||||
"""
|
||||
A viewer with the same readable Dags: may read import errors, but not the admin-gated view.
|
||||
def no_bundle_view_client(test_client):
|
||||
"""A caller who may read every Dag but is denied ``AccessView.DAG_BUNDLES``."""
|
||||
with _patch_dag_bundles_view(test_client, lambda team_name: False):
|
||||
yield test_client
|
||||
|
||||
``test_client`` authenticates as an admin, which satisfies ``IMPORT_ERRORS_ALL`` under
|
||||
SimpleAuthManager and so cannot distinguish the two import-error kinds on its own.
|
||||
"""
|
||||
|
||||
def _headers_for(test_client, role: str) -> dict[str, str]:
|
||||
auth_manager = test_client.app.state.auth_manager
|
||||
token = auth_manager._get_token_signer().generate(
|
||||
auth_manager.serialize_user(SimpleAuthManagerUser(username="viewer", role="viewer"))
|
||||
auth_manager.serialize_user(SimpleAuthManagerUser(username=role, role=role))
|
||||
)
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def op_headers(test_client):
|
||||
"""
|
||||
A SimpleAuthManager op, the lowest role holding the bundle view by default.
|
||||
|
||||
``test_client`` authenticates as an admin, so this is the caller that shows the admin-only
|
||||
import-error views do not narrow the bundle pages either.
|
||||
"""
|
||||
return _headers_for(test_client, "op")
|
||||
|
||||
|
||||
class TestGetDagBundles:
|
||||
def test_should_raise_401_unauthenticated(self, unauthenticated_test_client):
|
||||
assert unauthenticated_test_client.get("/dagBundles").status_code == 401
|
||||
@@ -295,64 +266,47 @@ class TestGetDagBundles:
|
||||
def test_should_raise_403_unauthorized(self, unauthorized_test_client):
|
||||
assert unauthorized_test_client.get("/dagBundles").status_code == 403
|
||||
|
||||
def test_lists_bundles_holding_a_readable_dag(self, dag_scoped_client):
|
||||
response = dag_scoped_client.get("/dagBundles")
|
||||
def test_should_raise_403_without_the_bundle_view(self, no_bundle_view_client):
|
||||
"""Reading every Dag does not grant the bundle view."""
|
||||
assert no_bundle_view_client.get("/dagBundles").status_code == 403
|
||||
|
||||
@pytest.mark.parametrize("role", ["viewer", "user"])
|
||||
def test_should_raise_403_below_op(self, test_client, role):
|
||||
"""Bundles list every deployed file, so SimpleAuthManager keeps them from viewers and users."""
|
||||
response = test_client.get("/dagBundles", headers=_headers_for(test_client, role))
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
def test_lists_every_bundle(self, test_client):
|
||||
response = test_client.get("/dagBundles")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert [bundle["name"] for bundle in body["dag_bundles"]] == READABLE_BUNDLES
|
||||
assert body["total_entries"] == 3
|
||||
assert [bundle["name"] for bundle in body["dag_bundles"]] == ALL_BUNDLES
|
||||
assert body["total_entries"] == len(ALL_BUNDLES)
|
||||
|
||||
def test_excludes_a_bundle_whose_dags_are_not_readable(self, dag_scoped_client):
|
||||
"""The point of the Dag-derived filter: another team's bundle must not appear."""
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
def test_lists_every_bundle_to_a_caller_who_may_read_no_dag(self, no_dag_read_client):
|
||||
"""Bundles are authorized on their own view, not derived from the Dags inside them."""
|
||||
body = no_dag_read_client.get("/dagBundles").json()
|
||||
|
||||
assert OTHER_TEAM_BUNDLE not in [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
# Absent from the count too, so its existence does not leak through pagination.
|
||||
assert body["total_entries"] == 3
|
||||
assert [bundle["name"] for bundle in body["dag_bundles"]] == ALL_BUNDLES
|
||||
|
||||
def test_excludes_a_bundle_with_no_dags_without_the_admin_view(self, dag_scoped_client):
|
||||
"""A bundle nothing has parsed from has no Dag to authorize against, so Dag scoping alone
|
||||
cannot show it."""
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
|
||||
assert DAGLESS_BUNDLE not in [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
assert body["total_entries"] == 3
|
||||
|
||||
def test_hides_a_bundle_with_no_dags_from_a_viewer(self, viewer_headers, test_client):
|
||||
def test_import_error_count_covers_the_whole_bundle_for_an_op(self, op_headers, test_client):
|
||||
"""
|
||||
The bundle name and its version are the disclosure, so a viewer must not get them.
|
||||
Every error recorded against the bundle is counted, whoever the caller is.
|
||||
|
||||
Same reasoning as the admin gate on import errors for a file that never registered a Dag:
|
||||
with no Dag to authorize against, there is nothing weaker than the admin view to fall back
|
||||
on.
|
||||
``GIT_BUNDLE`` holds an error in a file with a Dag, one in a file that never registered a
|
||||
Dag, and one in a file holding another Dag. A count narrowed to what the caller may read
|
||||
reads as "nothing is wrong" when something is.
|
||||
"""
|
||||
body = test_client.get("/dagBundles", headers=viewer_headers).json()
|
||||
body = test_client.get("/dagBundles", headers=op_headers).json()
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == GIT_BUNDLE)
|
||||
|
||||
assert DAGLESS_BUNDLE not in [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
# Absent from the count too, so its existence does not leak through pagination.
|
||||
assert body["total_entries"] == 3
|
||||
assert bundle["import_error_count"] == 3
|
||||
|
||||
def test_lists_a_bundle_with_no_dags_for_an_admin(self, admin_client):
|
||||
"""
|
||||
The case the page exists for: a first deploy whose only file fails to import.
|
||||
|
||||
Deriving visibility from readable Dags alone would hide the bundle precisely when its
|
||||
import-error count is the thing worth reading, so ``IMPORT_ERRORS_ALL`` opens it up.
|
||||
"""
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
names = [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
|
||||
assert DAGLESS_BUNDLE in names
|
||||
# The Dag-scoped rows are unchanged, and the other team's bundle is still excluded: the
|
||||
# admin view adds the Dag-less bundles, it does not bypass Dag scoping.
|
||||
assert names == sorted([*READABLE_BUNDLES, DAGLESS_BUNDLE])
|
||||
assert OTHER_TEAM_BUNDLE not in names
|
||||
assert body["total_entries"] == 4
|
||||
|
||||
def test_dagless_bundle_reports_its_fields(self, admin_client):
|
||||
def test_dagless_bundle_reports_its_fields(self, test_client):
|
||||
"""A Dag-less bundle is a real row, not a name-only placeholder."""
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
body = test_client.get("/dagBundles").json()
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == DAGLESS_BUNDLE)
|
||||
|
||||
assert bundle["version"] == "0badcafe"
|
||||
@@ -360,41 +314,12 @@ class TestGetDagBundles:
|
||||
assert bundle["last_refreshed"].startswith("2026-09-10T12:00:00")
|
||||
assert bundle["import_error_count"] == 0
|
||||
|
||||
def test_lists_a_bundle_whose_dags_are_all_stale(self, dag_scoped_client, session):
|
||||
"""
|
||||
The other side of deriving visibility from Dags: deleting them must not hide the bundle.
|
||||
|
||||
A bundle whose Dags have all been removed is exactly when an operator wants to watch it
|
||||
still being refreshed, and ``get_authorized_dag_ids`` does not filter on ``is_stale``
|
||||
either, so the stale row keeps the bundle visible.
|
||||
"""
|
||||
session.execute(update(DagModel).where(DagModel.bundle_name == GIT_BUNDLE).values(is_stale=True))
|
||||
session.commit()
|
||||
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
|
||||
assert GIT_BUNDLE in [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
|
||||
def test_returns_nothing_when_no_dag_is_readable(self, viewer_headers, test_client):
|
||||
# The fixture already patched this attribute, so retarget its mock rather than nesting a
|
||||
# second autospec patch over it.
|
||||
test_client.app.state.auth_manager.get_authorized_dag_ids.return_value = set()
|
||||
|
||||
body = test_client.get("/dagBundles", headers=viewer_headers).json()
|
||||
|
||||
assert body == {"dag_bundles": [], "total_entries": 0}
|
||||
|
||||
def test_returns_only_dagless_bundles_when_an_admin_can_read_no_dag(self, test_client):
|
||||
"""The admin view is additive: it never brings back a bundle whose Dags are unreadable."""
|
||||
auth_manager = test_client.app.state.auth_manager
|
||||
with mock.patch.object(auth_manager, "get_authorized_dag_ids", autospec=True, return_value=set()):
|
||||
body = test_client.get("/dagBundles").json()
|
||||
|
||||
assert [bundle["name"] for bundle in body["dag_bundles"]] == [DAGLESS_BUNDLE]
|
||||
assert body["total_entries"] == 1
|
||||
|
||||
def test_versioned_bundle_fields(self, admin_client):
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
def test_versioned_bundle_fields(self, no_dag_read_client):
|
||||
"""The bundle url is part of the bundle, so it needs no Dag or Dag version read."""
|
||||
with mock.patch.object(
|
||||
no_dag_read_client.app.state.auth_manager, "is_authorized_dag", autospec=True, return_value=False
|
||||
):
|
||||
body = no_dag_read_client.get("/dagBundles").json()
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == GIT_BUNDLE)
|
||||
|
||||
assert bundle["version"] == GIT_VERSION
|
||||
@@ -402,39 +327,10 @@ class TestGetDagBundles:
|
||||
assert bundle["last_refreshed"].startswith("2026-09-10T12:00:00")
|
||||
# Rendered from the signed template with the stored template params filled in.
|
||||
assert bundle["bundle_url"] == f"https://github.com/example/repo/tree/{GIT_VERSION}/dags"
|
||||
# The admin client may read import errors for files with no Dag, so it sees both.
|
||||
assert bundle["import_error_count"] == 2
|
||||
|
||||
def test_withholds_the_url_without_dag_version_access(self, dag_scoped_client):
|
||||
"""
|
||||
A rendered bundle url needs Dag *version* read, which is a separate grant.
|
||||
|
||||
The same url is otherwise only reachable through ``GET /dags/{dag_id}/dagVersions``, so a
|
||||
caller who may read the Dag but not its versions must still see the bundle -- the row is
|
||||
the point of the page -- with the link withheld rather than the row dropped.
|
||||
"""
|
||||
auth_manager = dag_scoped_client.app.state.auth_manager
|
||||
real_is_authorized_dag = auth_manager.is_authorized_dag
|
||||
|
||||
def deny_version(*, method, access_entity=None, details=None, user):
|
||||
if access_entity is DagAccessEntity.VERSION:
|
||||
return False
|
||||
return real_is_authorized_dag(
|
||||
method=method, access_entity=access_entity, details=details, user=user
|
||||
)
|
||||
|
||||
with mock.patch.object(auth_manager, "is_authorized_dag", autospec=True, side_effect=deny_version):
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == GIT_BUNDLE)
|
||||
assert bundle["bundle_url"] is None
|
||||
# Still listed, and the rest of the row is intact.
|
||||
assert bundle["version"] == GIT_VERSION
|
||||
assert [b["name"] for b in body["dag_bundles"]] == READABLE_BUNDLES
|
||||
|
||||
def test_non_versioned_bundle_has_no_version_but_keeps_a_timestamp(self, dag_scoped_client):
|
||||
def test_non_versioned_bundle_has_no_version_but_keeps_a_timestamp(self, test_client):
|
||||
"""A bundle that does not support versioning still reports when it was last refreshed."""
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
body = test_client.get("/dagBundles").json()
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == LOCAL_BUNDLE)
|
||||
|
||||
assert bundle["version"] is None
|
||||
@@ -442,7 +338,7 @@ class TestGetDagBundles:
|
||||
assert bundle["last_refreshed"].startswith("2026-09-10T12:00:00")
|
||||
assert bundle["import_error_count"] == 0
|
||||
|
||||
def test_reports_a_refresh_that_did_not_change_the_version(self, dag_scoped_client, session):
|
||||
def test_reports_a_refresh_that_did_not_change_the_version(self, test_client, session):
|
||||
"""
|
||||
The behaviour the whole feature rests on, guarded on the read side.
|
||||
|
||||
@@ -453,7 +349,7 @@ class TestGetDagBundles:
|
||||
``test_manager.py::TestDagFileProcessorManager::test_refresh_dag_bundles_versioned_version_unchanged_calls_update_bundle_state``;
|
||||
this pins that the endpoint actually surfaces it.
|
||||
"""
|
||||
before = dag_scoped_client.get("/dagBundles").json()
|
||||
before = test_client.get("/dagBundles").json()
|
||||
was = next(b for b in before["dag_bundles"] if b["name"] == GIT_BUNDLE)
|
||||
|
||||
later = REFRESHED_AT + timedelta(minutes=5)
|
||||
@@ -462,51 +358,21 @@ class TestGetDagBundles:
|
||||
)
|
||||
session.commit()
|
||||
|
||||
after = dag_scoped_client.get("/dagBundles").json()
|
||||
after = test_client.get("/dagBundles").json()
|
||||
now = next(b for b in after["dag_bundles"] if b["name"] == GIT_BUNDLE)
|
||||
|
||||
assert now["last_refreshed"] != was["last_refreshed"]
|
||||
assert now["last_refreshed"].startswith("2026-09-10T12:05:00")
|
||||
assert now["version"] == was["version"] == GIT_VERSION
|
||||
|
||||
def test_deactivated_bundle_is_still_listed(self, dag_scoped_client):
|
||||
def test_deactivated_bundle_is_still_listed(self, test_client):
|
||||
"""A bundle dropped from the config stays visible and flagged, rather than vanishing."""
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
body = test_client.get("/dagBundles").json()
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == GONE_BUNDLE)
|
||||
|
||||
assert bundle["active"] is False
|
||||
assert bundle["version"] == "deadbeef"
|
||||
|
||||
def test_import_error_count_authorizes_on_the_same_terms_as_import_errors(
|
||||
self, viewer_headers, test_client
|
||||
):
|
||||
"""
|
||||
Count on the same terms as ``GET /importErrors``, not "every row for this bundle".
|
||||
|
||||
``GIT_BUNDLE`` carries three errors, and a viewer may be told about exactly one:
|
||||
|
||||
* ``REGISTERED_FILE`` holds a Dag this viewer can read, so it counts.
|
||||
* ``UNREGISTERED_FILE`` has no ``DagModel`` row, so it needs the admin-by-default
|
||||
``IMPORT_ERRORS_ALL``. Counting it would hand a viewer the existence of a file that
|
||||
failed before defining anything, on a page that polls.
|
||||
* ``UNREADABLE_FILE`` *is* registered, in this same bundle, but to a Dag outside
|
||||
``READABLE_DAG_IDS``, so only the ``dag_id IN readable_dag_ids`` restriction excludes it.
|
||||
|
||||
Dropping either restriction takes the count to 2, so one assertion pins both halves.
|
||||
"""
|
||||
body = test_client.get("/dagBundles", headers=viewer_headers).json()
|
||||
bundle = next(b for b in body["dag_bundles"] if b["name"] == GIT_BUNDLE)
|
||||
|
||||
assert bundle["import_error_count"] == 1
|
||||
|
||||
def test_import_error_count_is_withheld_without_permission(self, admin_client):
|
||||
auth_manager = admin_client.app.state.auth_manager
|
||||
with mock.patch.object(auth_manager, "authorize_view", autospec=True, return_value=False):
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
|
||||
# ``None``, not 0: "you may not see this" must not read as "nothing is wrong".
|
||||
assert {bundle["import_error_count"] for bundle in body["dag_bundles"]} == {None}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"template",
|
||||
[
|
||||
@@ -514,192 +380,140 @@ class TestGetDagBundles:
|
||||
pytest.param("https://example.com/{version.foo}", id="attribute-placeholder"),
|
||||
],
|
||||
)
|
||||
def test_a_malformed_url_template_does_not_fail_the_collection(
|
||||
self, dag_scoped_client, template, session
|
||||
):
|
||||
def test_a_malformed_url_template_does_not_fail_the_collection(self, test_client, template, session):
|
||||
"""``render_url`` guards only KeyError/ValueError, so a bad template must degrade to null."""
|
||||
bundle = session.get(DagBundleModel, GIT_BUNDLE)
|
||||
bundle.signed_url_template = _sign(template, GIT_BUNDLE)
|
||||
session.commit()
|
||||
|
||||
response = dag_scoped_client.get("/dagBundles")
|
||||
response = test_client.get("/dagBundles")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert next(b for b in body["dag_bundles"] if b["name"] == GIT_BUNDLE)["bundle_url"] is None
|
||||
# Every other row is unaffected.
|
||||
assert body["total_entries"] == 3
|
||||
assert body["total_entries"] == len(ALL_BUNDLES)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("params", "expected"),
|
||||
[
|
||||
pytest.param({"limit": 2}, [GIT_BUNDLE, GONE_BUNDLE], id="limit"),
|
||||
pytest.param({"limit": 2, "offset": 2}, [LOCAL_BUNDLE], id="offset"),
|
||||
pytest.param({"order_by": "-name"}, [LOCAL_BUNDLE, GONE_BUNDLE, GIT_BUNDLE], id="name-desc"),
|
||||
pytest.param(
|
||||
{"order_by": "last_refreshed"},
|
||||
[GONE_BUNDLE, GIT_BUNDLE, LOCAL_BUNDLE],
|
||||
id="last-refreshed-asc",
|
||||
),
|
||||
pytest.param({"limit": 2}, [DAGLESS_BUNDLE, GIT_BUNDLE], id="limit"),
|
||||
pytest.param({"limit": 2, "offset": 2}, [GONE_BUNDLE, LOCAL_BUNDLE], id="offset"),
|
||||
pytest.param({"order_by": "-name"}, list(reversed(ALL_BUNDLES)), id="name-desc"),
|
||||
# GONE_BUNDLE alone was refreshed earlier; the rest tie on REFRESHED_AT.
|
||||
pytest.param({"order_by": "last_refreshed", "limit": 1}, [GONE_BUNDLE], id="last-refreshed-asc"),
|
||||
],
|
||||
)
|
||||
def test_pagination_and_sorting(self, dag_scoped_client, params, expected):
|
||||
response = dag_scoped_client.get("/dagBundles", params=params)
|
||||
def test_pagination_and_sorting(self, test_client, params, expected):
|
||||
response = test_client.get("/dagBundles", params=params)
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert [bundle["name"] for bundle in body["dag_bundles"]] == expected
|
||||
# total_entries counts the whole permitted set, not the page.
|
||||
assert body["total_entries"] == 3
|
||||
assert body["total_entries"] == len(ALL_BUNDLES)
|
||||
|
||||
def test_rejects_an_unsupported_sort_column(self, dag_scoped_client):
|
||||
response = dag_scoped_client.get("/dagBundles", params={"order_by": "signed_url_template"})
|
||||
def test_rejects_an_unsupported_sort_column(self, test_client):
|
||||
response = test_client.get("/dagBundles", params={"order_by": "signed_url_template"})
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_never_exposes_the_signed_url_template(self, dag_scoped_client):
|
||||
def test_never_exposes_the_signed_url_template(self, test_client):
|
||||
"""The template is signed with the fernet key; only the rendered url may leave."""
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
body = test_client.get("/dagBundles").json()
|
||||
|
||||
for bundle in body["dag_bundles"]:
|
||||
assert "signed_url_template" not in bundle
|
||||
assert "template_params" not in bundle
|
||||
|
||||
@conf_vars({("core", "multi_team"): "False"})
|
||||
def test_team_name_is_null_when_multi_team_is_off(self, dag_scoped_client):
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
def test_team_name_is_null_when_multi_team_is_off(self, test_client):
|
||||
body = test_client.get("/dagBundles").json()
|
||||
|
||||
assert {bundle["team_name"] for bundle in body["dag_bundles"]} == {None}
|
||||
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_reports_the_owning_team_in_multi_team_mode(self, dag_scoped_client, session):
|
||||
"""
|
||||
Exercise the branch that reads teams at all.
|
||||
|
||||
Without ``multi_team`` on -- which is the shipped default, and so what every other test
|
||||
here runs under -- the ``get_team_names`` call is skipped entirely, leaving the batched
|
||||
read that exists to avoid a per-row lazy load uncovered.
|
||||
"""
|
||||
session.add(Team(name=TEAM_NAME))
|
||||
session.commit()
|
||||
session.execute(
|
||||
insert(dag_bundle_team_association_table).values(dag_bundle_name=GIT_BUNDLE, team_name=TEAM_NAME)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
body = dag_scoped_client.get("/dagBundles").json()
|
||||
|
||||
by_name = {bundle["name"]: bundle for bundle in body["dag_bundles"]}
|
||||
assert by_name[GIT_BUNDLE]["team_name"] == TEAM_NAME
|
||||
# Bundles with no team mapping still report null rather than inheriting one.
|
||||
assert by_name[LOCAL_BUNDLE]["team_name"] is None
|
||||
|
||||
|
||||
class TestDaglessBundleTeamScoping:
|
||||
"""
|
||||
A Dag-less bundle is authorized per team, not by one unscoped check standing in for all.
|
||||
|
||||
Deciding it once without a ``team_name`` fails in both directions under a team-aware policy:
|
||||
a caller granted the unscoped view sees every team's Dag-less bundles, and a caller granted
|
||||
only their own team's sees none. Both are covered below.
|
||||
"""
|
||||
class TestDagBundleTeamScoping:
|
||||
"""In multi-team mode the bundle view is checked against each bundle's own team."""
|
||||
|
||||
OTHER_TEAM = "team_b"
|
||||
DAGLESS_IN_TEAM = "dagless_team_a_bundle"
|
||||
DAGLESS_IN_OTHER_TEAM = "dagless_team_b_bundle"
|
||||
|
||||
@pytest.fixture
|
||||
@pytest.fixture(autouse=True)
|
||||
def team_bundles(self, session) -> None:
|
||||
"""Two bundles with no Dag at all, owned by different teams."""
|
||||
session.add_all([Team(name=TEAM_NAME), Team(name=self.OTHER_TEAM)])
|
||||
session.add_all(
|
||||
[
|
||||
_make_bundle(self.DAGLESS_IN_TEAM, version="aaaaaaa", last_refreshed=REFRESHED_AT),
|
||||
_make_bundle(self.DAGLESS_IN_OTHER_TEAM, version="bbbbbbb", last_refreshed=REFRESHED_AT),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
session.execute(
|
||||
insert(dag_bundle_team_association_table).values(
|
||||
[
|
||||
{"dag_bundle_name": self.DAGLESS_IN_TEAM, "team_name": TEAM_NAME},
|
||||
{"dag_bundle_name": self.DAGLESS_IN_OTHER_TEAM, "team_name": self.OTHER_TEAM},
|
||||
{"dag_bundle_name": GIT_BUNDLE, "team_name": TEAM_NAME},
|
||||
{"dag_bundle_name": OTHER_TEAM_BUNDLE, "team_name": self.OTHER_TEAM},
|
||||
]
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
def _client_granting(test_client, permitted_teams: set[str | None]):
|
||||
"""
|
||||
A caller whose ``IMPORT_ERRORS_ALL`` is granted only for ``permitted_teams``.
|
||||
|
||||
``IMPORT_ERRORS`` stays granted throughout so the import-error counts are still computed;
|
||||
only the admin-gated view is scoped, which is what a team-aware manager varies.
|
||||
"""
|
||||
auth_manager = test_client.app.state.auth_manager
|
||||
|
||||
def team_aware_authorize_view(*, access_view, user, team_name=None):
|
||||
if access_view is AccessView.IMPORT_ERRORS_ALL:
|
||||
return team_name in permitted_teams
|
||||
return True
|
||||
|
||||
return mock.patch.object(
|
||||
auth_manager, "authorize_view", autospec=True, side_effect=team_aware_authorize_view
|
||||
)
|
||||
|
||||
def test_does_not_expose_another_teams_dagless_bundle(self, team_bundles, admin_client):
|
||||
"""
|
||||
Granting the unscoped view must not hand over a team the policy denies.
|
||||
|
||||
The caller holds ``IMPORT_ERRORS_ALL`` unscoped and for their own team, but not for
|
||||
``team_b``. Deciding the disjunct from the unscoped answer alone would admit every
|
||||
Dag-less bundle, ``team_b``'s included.
|
||||
"""
|
||||
with self._client_granting(admin_client, {None, TEAM_NAME}):
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_lists_only_the_bundles_of_permitted_teams(self, test_client):
|
||||
# ``None`` authorizes the team-less bundles.
|
||||
with _patch_dag_bundles_view(test_client, lambda team_name: team_name in (None, TEAM_NAME)):
|
||||
body = test_client.get("/dagBundles").json()
|
||||
|
||||
names = [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
assert self.DAGLESS_IN_TEAM in names
|
||||
assert self.DAGLESS_IN_OTHER_TEAM not in names
|
||||
# Filtered in the query, so the denied bundle is absent from the count as well.
|
||||
assert OTHER_TEAM_BUNDLE not in names
|
||||
assert names == [DAGLESS_BUNDLE, GIT_BUNDLE, GONE_BUNDLE, LOCAL_BUNDLE]
|
||||
assert next(b for b in body["dag_bundles"] if b["name"] == GIT_BUNDLE)["team_name"] == TEAM_NAME
|
||||
# Filtered in the query, so the other team's bundle is absent from the count as well.
|
||||
assert body["total_entries"] == len(names)
|
||||
|
||||
def test_does_not_hide_a_dagless_bundle_the_caller_holds(self, team_bundles, admin_client):
|
||||
"""
|
||||
Denying the unscoped view must not hide a team the policy grants.
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_authorizes_a_team_less_bundle_without_a_team(self, test_client):
|
||||
with _patch_dag_bundles_view(test_client, lambda team_name: team_name is None):
|
||||
body = test_client.get("/dagBundles").json()
|
||||
|
||||
The caller holds ``IMPORT_ERRORS_ALL`` for ``team_a`` only, so the unscoped check returns
|
||||
False. Deciding the disjunct from that answer would hide their own team's bundle.
|
||||
"""
|
||||
with self._client_granting(admin_client, {TEAM_NAME}):
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
assert [bundle["name"] for bundle in body["dag_bundles"]] == [
|
||||
DAGLESS_BUNDLE,
|
||||
GONE_BUNDLE,
|
||||
LOCAL_BUNDLE,
|
||||
]
|
||||
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_admits_a_caller_granted_the_view_for_only_one_team(self, test_client):
|
||||
with _patch_dag_bundles_view(test_client, lambda team_name: team_name == TEAM_NAME):
|
||||
response = test_client.get("/dagBundles")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [bundle["name"] for bundle in response.json()["dag_bundles"]] == [GIT_BUNDLE]
|
||||
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_should_raise_403_without_the_view_for_any_team(self, no_bundle_view_client):
|
||||
assert no_bundle_view_client.get("/dagBundles").status_code == 403
|
||||
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_hides_another_teams_bundles_when_the_view_ignores_the_team(self, test_client):
|
||||
"""An auth manager may grant the view whatever ``team_name`` is; team membership still applies."""
|
||||
with (
|
||||
_patch_dag_bundles_view(test_client, lambda team_name: True),
|
||||
mock.patch.object(
|
||||
test_client.app.state.auth_manager,
|
||||
"filter_authorized_teams",
|
||||
autospec=True,
|
||||
side_effect=lambda *, teams_names, user, method="GET": teams_names & {TEAM_NAME},
|
||||
),
|
||||
):
|
||||
body = test_client.get("/dagBundles").json()
|
||||
|
||||
names = [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
assert self.DAGLESS_IN_TEAM in names
|
||||
assert self.DAGLESS_IN_OTHER_TEAM not in names
|
||||
# The team-less Dag-less bundle is authorized with team_name=None, which this caller
|
||||
# does not hold.
|
||||
assert DAGLESS_BUNDLE not in names
|
||||
assert names == [DAGLESS_BUNDLE, GIT_BUNDLE, GONE_BUNDLE, LOCAL_BUNDLE]
|
||||
assert body["total_entries"] == len(names)
|
||||
|
||||
def test_grants_a_team_less_bundle_only_on_the_unscoped_view(self, team_bundles, admin_client):
|
||||
"""A bundle mapped to no team is authorized with ``team_name=None``."""
|
||||
with self._client_granting(admin_client, {None}):
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
@pytest.mark.parametrize("path", ["", "/files"])
|
||||
def test_404_for_a_bundle_of_another_team(self, test_client, path):
|
||||
"""A 404 rather than a 403, so the response does not confirm the bundle exists."""
|
||||
with _patch_dag_bundles_view(test_client, lambda team_name: team_name in (None, TEAM_NAME)):
|
||||
response = test_client.get(f"/dagBundles/{OTHER_TEAM_BUNDLE}{path}")
|
||||
|
||||
names = [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
assert DAGLESS_BUNDLE in names
|
||||
assert self.DAGLESS_IN_TEAM not in names
|
||||
assert self.DAGLESS_IN_OTHER_TEAM not in names
|
||||
|
||||
def test_hides_every_dagless_bundle_without_the_view(self, team_bundles, admin_client):
|
||||
with self._client_granting(admin_client, set()):
|
||||
body = admin_client.get("/dagBundles").json()
|
||||
|
||||
names = [bundle["name"] for bundle in body["dag_bundles"]]
|
||||
assert names == READABLE_BUNDLES
|
||||
assert body["total_entries"] == len(READABLE_BUNDLES)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestGetDagBundle:
|
||||
@@ -709,8 +523,11 @@ class TestGetDagBundle:
|
||||
def test_should_raise_403_unauthorized(self, unauthorized_test_client):
|
||||
assert unauthorized_test_client.get(f"/dagBundles/{GIT_BUNDLE}").status_code == 403
|
||||
|
||||
def test_returns_the_bundle(self, dag_scoped_client):
|
||||
response = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}")
|
||||
def test_should_raise_403_without_the_bundle_view(self, no_bundle_view_client):
|
||||
assert no_bundle_view_client.get(f"/dagBundles/{GIT_BUNDLE}").status_code == 403
|
||||
|
||||
def test_returns_the_bundle(self, test_client):
|
||||
response = test_client.get(f"/dagBundles/{GIT_BUNDLE}")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
@@ -719,97 +536,30 @@ class TestGetDagBundle:
|
||||
assert body["last_refreshed"] == "2026-09-10T12:00:00Z"
|
||||
assert body["active"] is True
|
||||
assert body["bundle_url"] == f"https://github.com/example/repo/tree/{GIT_VERSION}/dags"
|
||||
assert body["import_error_count"] == 3
|
||||
|
||||
def test_dag_count_excludes_dags_the_caller_may_not_read(self, dag_scoped_client):
|
||||
"""``GIT_BUNDLE`` holds two Dags, one of them absent from ``READABLE_DAG_IDS``."""
|
||||
body = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}").json()
|
||||
def test_counts_every_live_dag_whatever_the_caller_may_read(self, no_dag_read_client):
|
||||
"""``GIT_BUNDLE`` holds two live Dags and one stale one."""
|
||||
body = no_dag_read_client.get(f"/dagBundles/{GIT_BUNDLE}").json()
|
||||
|
||||
assert body["dag_count"] == 1
|
||||
assert body["dag_count"] == 2
|
||||
|
||||
def test_404_for_a_bundle_whose_dags_are_not_readable(self, dag_scoped_client):
|
||||
"""A 404 rather than a 403, so the response does not confirm the bundle exists."""
|
||||
assert dag_scoped_client.get(f"/dagBundles/{OTHER_TEAM_BUNDLE}").status_code == 404
|
||||
|
||||
def test_404_for_a_bundle_with_no_dags_without_the_admin_view(self, dag_scoped_client):
|
||||
"""``dag_scoped_client`` is denied ``IMPORT_ERRORS_ALL``, so Dag scoping alone decides."""
|
||||
assert dag_scoped_client.get(f"/dagBundles/{DAGLESS_BUNDLE}").status_code == 404
|
||||
assert dag_scoped_client.get(f"/dagBundles/{DAGLESS_BUNDLE}/files").status_code == 404
|
||||
|
||||
def test_shows_a_bundle_with_no_dags_to_a_caller_holding_the_admin_view(self, admin_client):
|
||||
"""
|
||||
The first-deploy-is-broken case: a bundle whose only file fails before defining a Dag.
|
||||
|
||||
It has no Dag to authorize against, so it rides on ``IMPORT_ERRORS_ALL`` the way the
|
||||
collection route does; the detail route must not re-derive a stricter rule and hide it.
|
||||
"""
|
||||
response = admin_client.get(f"/dagBundles/{DAGLESS_BUNDLE}")
|
||||
def test_shows_a_bundle_with_no_dags(self, op_headers, test_client):
|
||||
"""The first-deploy-is-broken case: a bundle whose only file fails before defining a Dag."""
|
||||
response = test_client.get(f"/dagBundles/{DAGLESS_BUNDLE}", headers=op_headers)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["dag_count"] == 0
|
||||
assert admin_client.get(f"/dagBundles/{DAGLESS_BUNDLE}/files").status_code == 200
|
||||
|
||||
def test_404_for_an_unknown_bundle(self, dag_scoped_client):
|
||||
assert dag_scoped_client.get("/dagBundles/no_such_bundle").status_code == 404
|
||||
def test_404_for_an_unknown_bundle(self, test_client):
|
||||
assert test_client.get("/dagBundles/no_such_bundle").status_code == 404
|
||||
|
||||
def test_import_error_count_is_gated_like_the_collection(self, admin_client, viewer_headers):
|
||||
"""
|
||||
The admin sees the unregistered-file error as well; the viewer sees only the registered one.
|
||||
|
||||
Same two-part authorization as ``GET /importErrors``, asserted here so the detail route
|
||||
cannot drift from the collection route it shares a helper with.
|
||||
"""
|
||||
assert admin_client.get(f"/dagBundles/{GIT_BUNDLE}").json()["import_error_count"] == 2
|
||||
assert (
|
||||
admin_client.get(f"/dagBundles/{GIT_BUNDLE}", headers=viewer_headers).json()["import_error_count"]
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_import_error_count_is_withheld_without_permission(self, admin_client):
|
||||
auth_manager = admin_client.app.state.auth_manager
|
||||
with mock.patch.object(auth_manager, "authorize_view", autospec=True, return_value=False):
|
||||
body = admin_client.get(f"/dagBundles/{GIT_BUNDLE}").json()
|
||||
|
||||
assert body["import_error_count"] is None
|
||||
|
||||
@conf_vars({("core", "multi_team"): "True"})
|
||||
def test_reports_the_owning_team_in_multi_team_mode(self, dag_scoped_client, session):
|
||||
"""Without ``multi_team`` on -- the shipped default -- the team lookup is skipped entirely."""
|
||||
session.add(Team(name=TEAM_NAME))
|
||||
session.commit()
|
||||
session.execute(
|
||||
insert(dag_bundle_team_association_table).values(dag_bundle_name=GIT_BUNDLE, team_name=TEAM_NAME)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
assert dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}").json()["team_name"] == TEAM_NAME
|
||||
|
||||
def test_bundle_url_is_withheld_without_dag_version_read(self, dag_scoped_client):
|
||||
"""
|
||||
A rendered bundle url is otherwise only reachable through ``GET /dags/{dag_id}/dagVersions``.
|
||||
|
||||
That route additionally requires Dag *version* read, so a role that grants Dag read and
|
||||
withholds version read must not get the repository address here instead.
|
||||
"""
|
||||
auth_manager = dag_scoped_client.app.state.auth_manager
|
||||
real = auth_manager.is_authorized_dag
|
||||
|
||||
def _deny_versions(*args, **kwargs):
|
||||
if kwargs.get("access_entity") is DagAccessEntity.VERSION:
|
||||
return False
|
||||
return real(*args, **kwargs)
|
||||
|
||||
with mock.patch.object(auth_manager, "is_authorized_dag", autospec=True, side_effect=_deny_versions):
|
||||
body = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}").json()
|
||||
|
||||
assert body["version"] == GIT_VERSION
|
||||
assert body["bundle_url"] is None
|
||||
|
||||
def test_dag_count_excludes_stale_dags(self, dag_scoped_client, session):
|
||||
def test_dag_count_excludes_stale_dags(self, test_client, session):
|
||||
"""``GIT_BUNDLE``'s Dags all go stale, so nothing live is left to count."""
|
||||
session.execute(update(DagModel).where(DagModel.bundle_name == GIT_BUNDLE).values(is_stale=True))
|
||||
session.commit()
|
||||
|
||||
body = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}").json()
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}").json()
|
||||
|
||||
assert body["dag_count"] == 0
|
||||
|
||||
@@ -821,33 +571,29 @@ class TestGetDagBundleFiles:
|
||||
def test_should_raise_403_unauthorized(self, unauthorized_test_client):
|
||||
assert unauthorized_test_client.get(f"/dagBundles/{GIT_BUNDLE}/files").status_code == 403
|
||||
|
||||
def test_lists_files_ordered_by_path(self, admin_client):
|
||||
response = admin_client.get(f"/dagBundles/{GIT_BUNDLE}/files")
|
||||
def test_should_raise_403_without_the_bundle_view(self, no_bundle_view_client):
|
||||
assert no_bundle_view_client.get(f"/dagBundles/{GIT_BUNDLE}/files").status_code == 403
|
||||
|
||||
def test_lists_files_ordered_by_path(self, test_client):
|
||||
response = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert [file["relative_fileloc"] for file in body["dag_bundle_files"]] == [
|
||||
UNREGISTERED_FILE,
|
||||
REGISTERED_FILE,
|
||||
]
|
||||
assert body["total_entries"] == 2
|
||||
assert [file["relative_fileloc"] for file in body["dag_bundle_files"]] == GIT_BUNDLE_FILES
|
||||
assert body["total_entries"] == len(GIT_BUNDLE_FILES)
|
||||
|
||||
def test_reports_parse_time_and_duration(self, dag_scoped_client):
|
||||
body = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
def test_lists_every_file_to_a_caller_who_may_read_no_dag(self, no_dag_read_client):
|
||||
"""The list is the bundle's, so it is not narrowed to the files of readable Dags."""
|
||||
body = no_dag_read_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
|
||||
by_path = {file["relative_fileloc"]: file for file in body["dag_bundle_files"]}
|
||||
assert by_path[REGISTERED_FILE]["dag_count"] == 1
|
||||
assert by_path[REGISTERED_FILE]["last_parsed_time"] == "2026-09-10T12:01:00Z"
|
||||
assert by_path[REGISTERED_FILE]["last_parse_duration"] == PARSE_DURATION
|
||||
assert list(by_path) == GIT_BUNDLE_FILES
|
||||
assert by_path[SECOND_REGISTERED_FILE]["dag_count"] == 1
|
||||
assert by_path[SECOND_REGISTERED_FILE]["import_error_count"] == 1
|
||||
|
||||
def test_a_file_that_registered_no_dag_is_listed_with_its_error(self, admin_client):
|
||||
"""
|
||||
The case the page most needs to show: a file that failed before defining a Dag.
|
||||
|
||||
It has no Dag to authorize on, so it is admin-gated, and without it a brand new broken
|
||||
file would be invisible on the very page someone opens to find out why.
|
||||
"""
|
||||
body = admin_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
def test_an_op_sees_a_file_that_registered_no_dag(self, op_headers, test_client):
|
||||
"""The case the page most needs to show: a file that failed before defining a Dag."""
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files", headers=op_headers).json()
|
||||
|
||||
by_path = {file["relative_fileloc"]: file for file in body["dag_bundle_files"]}
|
||||
assert by_path[UNREGISTERED_FILE]["dag_count"] == 0
|
||||
@@ -855,30 +601,15 @@ class TestGetDagBundleFiles:
|
||||
assert by_path[UNREGISTERED_FILE]["last_parsed_time"] is None
|
||||
assert by_path[UNREGISTERED_FILE]["last_parse_duration"] is None
|
||||
|
||||
def test_excludes_a_file_whose_dag_is_not_readable(self, admin_client, viewer_headers):
|
||||
"""``UNREADABLE_FILE`` is registered, so only the readable-Dag filter keeps it out."""
|
||||
for headers in ({}, viewer_headers):
|
||||
body = admin_client.get(f"/dagBundles/{GIT_BUNDLE}/files", headers=headers).json()
|
||||
assert UNREADABLE_FILE not in {file["relative_fileloc"] for file in body["dag_bundle_files"]}
|
||||
def test_reports_parse_time_and_duration(self, test_client):
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
|
||||
def test_viewer_does_not_see_the_unregistered_file(self, viewer_headers, test_client):
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files", headers=viewer_headers).json()
|
||||
by_path = {file["relative_fileloc"]: file for file in body["dag_bundle_files"]}
|
||||
assert by_path[REGISTERED_FILE]["dag_count"] == 1
|
||||
assert by_path[REGISTERED_FILE]["last_parsed_time"] == "2026-09-10T12:01:00Z"
|
||||
assert by_path[REGISTERED_FILE]["last_parse_duration"] == PARSE_DURATION
|
||||
|
||||
assert [file["relative_fileloc"] for file in body["dag_bundle_files"]] == [REGISTERED_FILE]
|
||||
assert body["total_entries"] == 1
|
||||
|
||||
def test_import_error_count_is_withheld_without_permission(self, admin_client):
|
||||
"""``None``, not 0: "you may not see this" must not read as "nothing is wrong"."""
|
||||
auth_manager = admin_client.app.state.auth_manager
|
||||
with mock.patch.object(auth_manager, "authorize_view", autospec=True, return_value=False):
|
||||
body = admin_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
|
||||
assert [file["relative_fileloc"] for file in body["dag_bundle_files"]] == [REGISTERED_FILE]
|
||||
assert body["dag_bundle_files"][0]["import_error_count"] is None
|
||||
|
||||
def test_a_file_whose_dags_went_stale_stays_listed_while_it_has_an_error(
|
||||
self, dag_scoped_client, session
|
||||
):
|
||||
def test_a_file_whose_dags_went_stale_stays_listed_while_it_has_an_error(self, test_client, session):
|
||||
"""Marking a file's Dags stale must not hide the row that explains the breakage."""
|
||||
session.execute(
|
||||
update(DagModel)
|
||||
@@ -887,37 +618,37 @@ class TestGetDagBundleFiles:
|
||||
)
|
||||
session.commit()
|
||||
|
||||
body = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
|
||||
by_path = {file["relative_fileloc"]: file for file in body["dag_bundle_files"]}
|
||||
assert by_path[REGISTERED_FILE]["dag_count"] == 0
|
||||
assert by_path[REGISTERED_FILE]["import_error_count"] == 1
|
||||
|
||||
def test_parse_duration_ignores_a_stale_dag_from_an_older_parse(self, dag_scoped_client):
|
||||
def test_parse_duration_ignores_a_stale_dag_from_an_older_parse(self, test_client):
|
||||
"""
|
||||
A Dag removed from a file keeps its row, frozen at the parse it was last seen in.
|
||||
|
||||
Aggregating the two columns independently would pair that older, slower duration with the
|
||||
newer timestamp and report a parse time that never happened.
|
||||
"""
|
||||
body = dag_scoped_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files").json()
|
||||
|
||||
by_path = {file["relative_fileloc"]: file for file in body["dag_bundle_files"]}
|
||||
assert by_path[REGISTERED_FILE]["last_parsed_time"] == "2026-09-10T12:01:00Z"
|
||||
assert by_path[REGISTERED_FILE]["last_parse_duration"] == PARSE_DURATION
|
||||
|
||||
def test_a_file_with_no_live_dag_and_no_error_is_dropped(self, dag_scoped_client, session):
|
||||
def test_a_file_with_no_live_dag_and_no_error_is_dropped(self, test_client, session):
|
||||
"""A file whose Dags are all stale and which has no error is one the bundle no longer has."""
|
||||
session.execute(update(DagModel).where(DagModel.bundle_name == LOCAL_BUNDLE).values(is_stale=True))
|
||||
session.commit()
|
||||
|
||||
body = dag_scoped_client.get(f"/dagBundles/{LOCAL_BUNDLE}/files").json()
|
||||
body = test_client.get(f"/dagBundles/{LOCAL_BUNDLE}/files").json()
|
||||
|
||||
assert body["dag_bundle_files"] == []
|
||||
assert body["total_entries"] == 0
|
||||
|
||||
def test_a_file_with_no_error_reports_zero(self, dag_scoped_client):
|
||||
body = dag_scoped_client.get(f"/dagBundles/{LOCAL_BUNDLE}/files").json()
|
||||
def test_a_file_with_no_error_reports_zero(self, test_client):
|
||||
body = test_client.get(f"/dagBundles/{LOCAL_BUNDLE}/files").json()
|
||||
|
||||
assert [file["import_error_count"] for file in body["dag_bundle_files"]] == [0]
|
||||
|
||||
@@ -925,22 +656,19 @@ class TestGetDagBundleFiles:
|
||||
("params", "expected"),
|
||||
[
|
||||
pytest.param({"limit": 1}, [UNREGISTERED_FILE], id="limit"),
|
||||
pytest.param({"offset": 1}, [REGISTERED_FILE], id="offset"),
|
||||
pytest.param({"offset": 1}, [REGISTERED_FILE, SECOND_REGISTERED_FILE], id="offset"),
|
||||
pytest.param({"limit": 1, "offset": 1}, [REGISTERED_FILE], id="limit-and-offset"),
|
||||
# ``limit`` is a non-negative int, and ``Select.limit(0)`` returns nothing, so zero
|
||||
# has to mean nothing here too rather than being read as "unlimited".
|
||||
pytest.param({"limit": 0}, [], id="limit-zero"),
|
||||
],
|
||||
)
|
||||
def test_pagination(self, admin_client, params, expected):
|
||||
body = admin_client.get(f"/dagBundles/{GIT_BUNDLE}/files", params=params).json()
|
||||
def test_pagination(self, test_client, params, expected):
|
||||
body = test_client.get(f"/dagBundles/{GIT_BUNDLE}/files", params=params).json()
|
||||
|
||||
assert [file["relative_fileloc"] for file in body["dag_bundle_files"]] == expected
|
||||
# The total is the whole file list, not the page.
|
||||
assert body["total_entries"] == 2
|
||||
assert body["total_entries"] == len(GIT_BUNDLE_FILES)
|
||||
|
||||
def test_404_for_a_bundle_whose_dags_are_not_readable(self, dag_scoped_client):
|
||||
assert dag_scoped_client.get(f"/dagBundles/{OTHER_TEAM_BUNDLE}/files").status_code == 404
|
||||
|
||||
def test_404_for_an_unknown_bundle(self, dag_scoped_client):
|
||||
assert dag_scoped_client.get("/dagBundles/no_such_bundle/files").status_code == 404
|
||||
def test_404_for_an_unknown_bundle(self, test_client):
|
||||
assert test_client.get("/dagBundles/no_such_bundle/files").status_code == 404
|
||||
@@ -0,0 +1,103 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from airflow.api_fastapi.auth.managers.simple.user import SimpleAuthManagerUser
|
||||
from airflow.models import DagModel
|
||||
from airflow.models.dagbundle import DagBundleModel
|
||||
from airflow.models.errors import ParseImportError
|
||||
from airflow.utils.session import create_session
|
||||
|
||||
from tests_common.test_utils.db import clear_db_dag_bundles, clear_db_dags, clear_db_import_errors
|
||||
|
||||
pytestmark = pytest.mark.db_test
|
||||
|
||||
|
||||
def _clear() -> None:
|
||||
clear_db_import_errors()
|
||||
clear_db_dags()
|
||||
clear_db_dag_bundles()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def bundles():
|
||||
"""Two bundles with two live Dags each, and a third holding only a stale Dag, as when its every file fails."""
|
||||
_clear()
|
||||
with create_session() as session:
|
||||
session.add_all([DagBundleModel(name=name) for name in ("bundle_a", "bundle_b", "bundle_stale")])
|
||||
session.commit()
|
||||
for bundle_name, dag_ids, is_stale in [
|
||||
("bundle_a", ["a1", "a2"], False),
|
||||
("bundle_b", ["b1", "b2"], False),
|
||||
("bundle_stale", ["stale"], True),
|
||||
]:
|
||||
session.add_all(
|
||||
DagModel(dag_id=dag_id, bundle_name=bundle_name, fileloc=f"{dag_id}.py", is_stale=is_stale)
|
||||
for dag_id in dag_ids
|
||||
)
|
||||
session.commit()
|
||||
yield
|
||||
_clear()
|
||||
|
||||
|
||||
def test_counts_bundles_holding_only_stale_dags(test_client):
|
||||
response = test_client.get("/dagBundles/count")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"total_entries": 3}
|
||||
|
||||
|
||||
def _get_token(test_client, role: str) -> str:
|
||||
auth_manager = test_client.app.state.auth_manager
|
||||
return auth_manager._get_token_signer().generate(
|
||||
auth_manager.serialize_user(SimpleAuthManagerUser(username=role, role=role))
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("role", "expected_count"), [("admin", 4), ("viewer", 3)])
|
||||
def test_counts_a_bundle_whose_files_never_registered_a_dag_for_import_error_readers(
|
||||
test_client, role, expected_count
|
||||
):
|
||||
with create_session() as session:
|
||||
session.add(DagBundleModel(name="bundle_broken"))
|
||||
session.add(ParseImportError(filename="broken.py", bundle_name="bundle_broken", stacktrace="boom"))
|
||||
|
||||
response = test_client.get(
|
||||
"/dagBundles/count", headers={"Authorization": f"Bearer {_get_token(test_client, role)}"}
|
||||
)
|
||||
|
||||
assert response.json() == {"total_entries": expected_count}
|
||||
|
||||
|
||||
def test_counts_only_bundles_holding_a_readable_dag(test_client):
|
||||
with mock.patch.object(
|
||||
test_client.app.state.auth_manager,
|
||||
"get_authorized_dag_ids",
|
||||
autospec=True,
|
||||
return_value={"a1", "a2"},
|
||||
):
|
||||
response = test_client.get("/dagBundles/count")
|
||||
|
||||
assert response.json() == {"total_entries": 1}
|
||||
|
||||
|
||||
def test_should_raise_403_without_dag_read(unauthorized_test_client):
|
||||
assert unauthorized_test_client.get("/dagBundles/count").status_code == 403
|
||||
@@ -582,7 +582,7 @@ class DagBundleDetailResponse(BaseModel):
|
||||
bundle_url: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="A link to view the bundle at ``version``, when one is configured and the caller may read Dag versions.",
|
||||
description="A link to view the bundle at ``version``. Null when the bundle has no URL template, when the template needs a version and the bundle has not been refreshed yet, or when the template cannot be rendered.",
|
||||
title="Bundle Url",
|
||||
),
|
||||
]
|
||||
@@ -591,16 +591,16 @@ class DagBundleDetailResponse(BaseModel):
|
||||
Field(description="The team owning the bundle, in a multi-team deployment.", title="Team Name"),
|
||||
]
|
||||
import_error_count: Annotated[
|
||||
int | None,
|
||||
int,
|
||||
Field(
|
||||
description="Number of Dag import errors recorded against this bundle that the caller is permitted to see, counted on the same terms as ``GET /importErrors``. Null when the caller may not read import errors.",
|
||||
description="Number of Dag import errors recorded against the bundle. Reading the errors themselves is authorized separately, through ``GET /importErrors``.",
|
||||
title="Import Error Count",
|
||||
),
|
||||
]
|
||||
dag_count: Annotated[
|
||||
int,
|
||||
Field(
|
||||
description="Number of live Dags recorded against the bundle that the caller is permitted to see, counted on the same terms as ``GET /dags``.",
|
||||
description="Number of live Dags recorded against the bundle, including Dags the caller may not read.",
|
||||
title="Dag Count",
|
||||
),
|
||||
]
|
||||
@@ -615,7 +615,8 @@ class DagBundleFileResponse(BaseModel):
|
||||
dag_count: Annotated[
|
||||
int,
|
||||
Field(
|
||||
description="Number of live Dags the file defines that the caller may read.", title="Dag Count"
|
||||
description="Number of live Dags the file defines, including Dags the caller may not read.",
|
||||
title="Dag Count",
|
||||
),
|
||||
]
|
||||
last_parsed_time: Annotated[
|
||||
@@ -633,9 +634,9 @@ class DagBundleFileResponse(BaseModel):
|
||||
),
|
||||
]
|
||||
import_error_count: Annotated[
|
||||
int | None,
|
||||
int,
|
||||
Field(
|
||||
description="Number of import errors recorded against the file, which is at most one. Null when the caller may not read import errors -- deliberately not zero, which would read as a file with nothing wrong.",
|
||||
description="Number of import errors recorded against the file, which is at most one. Reading the error itself is authorized separately, through ``GET /importErrors``.",
|
||||
title="Import Error Count",
|
||||
),
|
||||
]
|
||||
@@ -671,7 +672,7 @@ class DagBundleResponse(BaseModel):
|
||||
bundle_url: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="A link to view the bundle at ``version``, when one is configured and the caller may read Dag versions.",
|
||||
description="A link to view the bundle at ``version``. Null when the bundle has no URL template, when the template needs a version and the bundle has not been refreshed yet, or when the template cannot be rendered.",
|
||||
title="Bundle Url",
|
||||
),
|
||||
]
|
||||
@@ -680,9 +681,9 @@ class DagBundleResponse(BaseModel):
|
||||
Field(description="The team owning the bundle, in a multi-team deployment.", title="Team Name"),
|
||||
]
|
||||
import_error_count: Annotated[
|
||||
int | None,
|
||||
int,
|
||||
Field(
|
||||
description="Number of Dag import errors recorded against this bundle that the caller is permitted to see, counted on the same terms as ``GET /importErrors``. Null when the caller may not read import errors.",
|
||||
description="Number of Dag import errors recorded against the bundle. Reading the errors themselves is authorized separately, through ``GET /importErrors``.",
|
||||
title="Import Error Count",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -378,16 +378,16 @@ Stable API Permissions
|
||||
- Op
|
||||
* - ``/api/v2/dagBundles``
|
||||
- GET
|
||||
- DAGs.can_read
|
||||
- Viewer
|
||||
- Dag Bundles.can_read
|
||||
- Op
|
||||
* - ``/api/v2/dagBundles/{bundle_name}``
|
||||
- GET
|
||||
- DAGs.can_read
|
||||
- Viewer
|
||||
- Dag Bundles.can_read
|
||||
- Op
|
||||
* - ``/api/v2/dagBundles/{bundle_name}/files``
|
||||
- GET
|
||||
- DAGs.can_read
|
||||
- Viewer
|
||||
- Dag Bundles.can_read
|
||||
- Op
|
||||
* - ``/api/v2/dagSources/{dag_id}``
|
||||
- GET
|
||||
- DAGs.can_read, DAG Code.can_read
|
||||
|
||||
@@ -20,6 +20,12 @@
|
||||
Changelog
|
||||
---------
|
||||
|
||||
.. note::
|
||||
Airflow 3.4.0 gives Dag bundles their own ``Dag Bundles`` resource, which ``Op`` and ``Admin``
|
||||
get by default. Upgrade this provider together with Airflow 3.4.0: older releases deny the Dag
|
||||
bundle pages and API to every role, including ``Admin``, while still showing the Browse menu
|
||||
entry. Custom roles need ``can read`` and ``menu access`` on ``Dag Bundles`` added by hand.
|
||||
|
||||
3.10.2
|
||||
......
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ from airflow.providers.fab.www.security.permissions import (
|
||||
RESOURCE_CONFIG,
|
||||
RESOURCE_CONNECTION,
|
||||
RESOURCE_DAG,
|
||||
RESOURCE_DAG_BUNDLE,
|
||||
RESOURCE_DAG_CODE,
|
||||
RESOURCE_DAG_DEPENDENCIES,
|
||||
RESOURCE_DAG_PREFIX,
|
||||
@@ -192,13 +193,14 @@ if AIRFLOW_V_3_1_PLUS:
|
||||
if hasattr(MenuItem, "DEADLINES"):
|
||||
_MAP_MENU_ITEM_TO_FAB_RESOURCE_TYPE[MenuItem.DEADLINES] = RESOURCE_DAG_RUN
|
||||
|
||||
# Dag bundle visibility rides on Dag access -- a bundle is shown to a user who can read a Dag that
|
||||
# came from it -- so the menu entry is gated the same way the Dags entry is. Without a mapping the
|
||||
# lookup below falls back to the raw enum value as a resource name, and unlike "Jobs" or "Providers"
|
||||
# (whose enum values happen to equal an existing FAB resource) no "Dag Bundles" resource exists, so
|
||||
# the entry would silently vanish for every FAB deployment.
|
||||
# The Dag bundles view and menu item first ship in Airflow 3.4.0, newer than this provider's minimum
|
||||
# Airflow. Unmapped, the view is denied to every role, and the menu lookup falls back to the raw
|
||||
# enum value as a resource name, so the entry would vanish for every FAB deployment.
|
||||
DAG_BUNDLES_ACCESS_VIEW: AccessView | None = getattr(AccessView, "DAG_BUNDLES", None)
|
||||
if DAG_BUNDLES_ACCESS_VIEW is not None:
|
||||
_MAP_ACCESS_VIEW_TO_FAB_RESOURCE_TYPE[DAG_BUNDLES_ACCESS_VIEW] = RESOURCE_DAG_BUNDLE
|
||||
if hasattr(MenuItem, "DAG_BUNDLES"):
|
||||
_MAP_MENU_ITEM_TO_FAB_RESOURCE_TYPE[MenuItem.DAG_BUNDLES] = RESOURCE_DAG
|
||||
_MAP_MENU_ITEM_TO_FAB_RESOURCE_TYPE[MenuItem.DAG_BUNDLES] = RESOURCE_DAG_BUNDLE
|
||||
|
||||
|
||||
class FabAuthManager(BaseAuthManager[User]):
|
||||
|
||||
@@ -319,6 +319,8 @@ class FabAirflowSecurityManagerOverride(AirflowSecurityManagerV2):
|
||||
# [START security_op_perms]
|
||||
OP_PERMISSIONS = [
|
||||
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_ADMIN_MENU),
|
||||
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_DAG_BUNDLE),
|
||||
(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_BUNDLE),
|
||||
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_CONFIG),
|
||||
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_CONNECTION),
|
||||
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_POOL),
|
||||
|
||||
@@ -28,6 +28,7 @@ RESOURCE_BROWSE_MENU = "Browse"
|
||||
RESOURCE_CONFIG = "Configurations"
|
||||
RESOURCE_CONNECTION = "Connections"
|
||||
RESOURCE_DAG = "DAGs"
|
||||
RESOURCE_DAG_BUNDLE = "Dag Bundles"
|
||||
RESOURCE_DAG_CODE = "DAG Code"
|
||||
RESOURCE_DAG_DEPENDENCIES = "DAG Dependencies"
|
||||
RESOURCE_DAG_PREFIX = "DAG:"
|
||||
|
||||
@@ -55,7 +55,7 @@ from airflow.providers.common.compat.security.permissions import (
|
||||
RESOURCE_ASSET_ALIAS,
|
||||
RESOURCE_BACKFILL,
|
||||
)
|
||||
from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager
|
||||
from airflow.providers.fab.auth_manager.fab_auth_manager import DAG_BUNDLES_ACCESS_VIEW, FabAuthManager
|
||||
from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
|
||||
from airflow.providers.fab.www.security.permissions import (
|
||||
ACTION_CAN_ACCESS_MENU,
|
||||
@@ -67,6 +67,7 @@ from airflow.providers.fab.www.security.permissions import (
|
||||
RESOURCE_CONFIG,
|
||||
RESOURCE_CONNECTION,
|
||||
RESOURCE_DAG,
|
||||
RESOURCE_DAG_BUNDLE,
|
||||
RESOURCE_DAG_RUN,
|
||||
RESOURCE_DOCS,
|
||||
RESOURCE_JOB,
|
||||
@@ -825,6 +826,15 @@ class TestFabAuthManager:
|
||||
[(ACTION_CAN_READ, RESOURCE_DOCS)],
|
||||
False,
|
||||
),
|
||||
# Dag bundles are their own resource, not a side effect of reading every Dag
|
||||
*(
|
||||
[
|
||||
(DAG_BUNDLES_ACCESS_VIEW, [(ACTION_CAN_READ, RESOURCE_DAG_BUNDLE)], True),
|
||||
(DAG_BUNDLES_ACCESS_VIEW, [(ACTION_CAN_READ, RESOURCE_DAG)], False),
|
||||
]
|
||||
if DAG_BUNDLES_ACCESS_VIEW is not None
|
||||
else []
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_is_authorized_view(self, access_view, user_permissions, expected_result, auth_manager):
|
||||
@@ -904,20 +914,17 @@ class TestFabAuthManager:
|
||||
if hasattr(MenuItem, "DEADLINES")
|
||||
else []
|
||||
),
|
||||
# Dag bundle visibility rides on Dag access, so the menu entry is mapped onto
|
||||
# RESOURCE_DAG. The mapping is load-bearing rather than cosmetic: unmapped items fall
|
||||
# back to the enum value as a resource name, and "Dag Bundles" is not a FAB resource,
|
||||
# so dropping the mapping would silently hide the entry for every FAB deployment.
|
||||
# The menu entry follows the bundle resource, not Dag access.
|
||||
*(
|
||||
[
|
||||
(
|
||||
[MenuItem.DAG_BUNDLES],
|
||||
[(ACTION_CAN_ACCESS_MENU, RESOURCE_DAG)],
|
||||
[(ACTION_CAN_ACCESS_MENU, RESOURCE_DAG_BUNDLE)],
|
||||
[MenuItem.DAG_BUNDLES],
|
||||
),
|
||||
(
|
||||
[MenuItem.DAG_BUNDLES],
|
||||
[(ACTION_CAN_ACCESS_MENU, "Dag Bundles")],
|
||||
[(ACTION_CAN_ACCESS_MENU, RESOURCE_DAG)],
|
||||
[],
|
||||
),
|
||||
]
|
||||
|
||||
@@ -119,8 +119,8 @@ Note: the CLI creates groups, resources, permissions, and policies, but **does n
|
||||
You must assign the appropriate Keycloak roles (Admin/Op/User/Viewer or SuperAdmin) to each user separately.
|
||||
In multi-team mode, the ``Admin`` role is **team-scoped** (group + role). Only ``SuperAdmin`` grants global
|
||||
admin access across all teams.
|
||||
Views over records that are not tied to a Dag or a team -- audit log entries not tied to a Dag,
|
||||
import errors for files with no registered Dag and reparsing such files -- are checked against the
|
||||
Views over records beyond the Dags a caller can read -- audit log entries not tied to a Dag,
|
||||
import errors for files with no registered Dag, reparsing such files, and Dag bundles -- are checked against the
|
||||
``AdminView`` resource, which the CLI grants to ``SuperAdmin`` only. The other views are checked against
|
||||
the ``View`` resource, which every team role can read.
|
||||
|
||||
|
||||
@@ -25,6 +25,12 @@
|
||||
Changelog
|
||||
---------
|
||||
|
||||
.. note::
|
||||
Airflow 3.4.0 adds a Dag bundles view, which in multi-team mode is checked against the
|
||||
``AdminView`` resource, so only ``SuperAdmin`` holds it. Multi-team deployments must upgrade this
|
||||
provider together with Airflow 3.4.0: older releases check the view against ``View``, which every
|
||||
team role can read.
|
||||
|
||||
0.11.2
|
||||
......
|
||||
|
||||
|
||||
@@ -64,12 +64,6 @@ TEAM_MENU_ITEMS = {
|
||||
MenuItem.ASSETS,
|
||||
MenuItem.DOCS,
|
||||
}
|
||||
# A Dag bundle is visible to whoever can read a Dag inside it, so this entry belongs with ``DAGS``
|
||||
# in the base team set rather than the admin one: every team role that can already reach the
|
||||
# bundles through the API should be able to find the page. Guarded because this provider also runs
|
||||
# against Airflow versions predating the menu item.
|
||||
if hasattr(MenuItem, "DAG_BUNDLES"):
|
||||
TEAM_MENU_ITEMS.add(MenuItem.DAG_BUNDLES)
|
||||
TEAM_ADMIN_MENU_ITEMS = TEAM_MENU_ITEMS | {
|
||||
MenuItem.CONNECTIONS,
|
||||
MenuItem.POOLS,
|
||||
|
||||
+4
-3
@@ -91,9 +91,10 @@ log = logging.getLogger(__name__)
|
||||
|
||||
RESOURCE_ID_ATTRIBUTE_NAME = "resource_id"
|
||||
|
||||
# ``AccessView`` values covering records that carry no per-Dag or per-team key to authorize on. They
|
||||
# are admin-by-default in core. Compared by value because older Airflow versions lack some members.
|
||||
ADMIN_ACCESS_VIEW_NAMES = frozenset({"AUDIT_LOGS_ALL", "IMPORT_ERRORS_ALL", "REPARSE_ALL"})
|
||||
# ``AccessView`` values showing records beyond the Dags a caller can read. In multi-team mode they
|
||||
# are checked against ``AdminView``, which only ``SuperAdmin`` holds. Compared by value because
|
||||
# older Airflow versions lack some members.
|
||||
ADMIN_ACCESS_VIEW_NAMES = frozenset({"AUDIT_LOGS_ALL", "DAG_BUNDLES", "IMPORT_ERRORS_ALL", "REPARSE_ALL"})
|
||||
|
||||
|
||||
TEAM_SCOPED_RESOURCES = frozenset(
|
||||
|
||||
@@ -43,15 +43,9 @@ from airflow.providers.keycloak.auth_manager.resources import KeycloakResource
|
||||
from tests_common.test_utils.config import conf_vars
|
||||
from tests_common.test_utils.version_compat import AIRFLOW_V_3_2_PLUS
|
||||
|
||||
# "Dag Bundles" is granted to every team role rather than only the admin one: the page is scoped
|
||||
# by the Dags a caller can read, so a Viewer who can already reach the bundles through the API has
|
||||
# to be able to find them under Browse. The provider still supports Airflow versions predating the
|
||||
# menu item, where ``TEAM_MENU_ITEMS`` drops it, so the expected grants follow the same guard the
|
||||
# source does rather than hard-coding a name that does not exist there.
|
||||
DAG_BUNDLES_MENU = ["Dag Bundles"] if hasattr(MenuItem, "DAG_BUNDLES") else []
|
||||
EXPECTED_TEAM_MENU_RESOURCES = sorted(["Assets", "Dags", "Docs", *DAG_BUNDLES_MENU])
|
||||
EXPECTED_TEAM_MENU_RESOURCES = sorted(["Assets", "Dags", "Docs"])
|
||||
EXPECTED_TEAM_ADMIN_MENU_RESOURCES = sorted(
|
||||
["Assets", "Connections", "Dags", "Docs", "Pools", "Variables", "XComs", *DAG_BUNDLES_MENU]
|
||||
["Assets", "Connections", "Dags", "Docs", "Pools", "Variables", "XComs"]
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1013,6 +1013,8 @@ class TestKeycloakAuthManager:
|
||||
("AUDIT_LOGS_ALL", "False", "View#GET"),
|
||||
("IMPORT_ERRORS", "True", "View#GET"),
|
||||
("PLUGINS", "True", "View#GET"),
|
||||
("DAG_BUNDLES", "True", "AdminView#GET"),
|
||||
("DAG_BUNDLES", "False", "View#GET"),
|
||||
],
|
||||
)
|
||||
def test_is_authorized_view_resource(
|
||||
|
||||
@@ -280,6 +280,7 @@ _FN_TO_RESOURCE_INFO: dict[str, tuple[str, str | None]] = {
|
||||
"requires_access_asset": ("Asset", None),
|
||||
"requires_access_asset_alias": ("AssetAlias", None),
|
||||
"requires_access_view": ("View", None),
|
||||
"requires_access_dag_bundles": ("View", "DAG_BUNDLES"),
|
||||
}
|
||||
|
||||
|
||||
@@ -387,7 +388,8 @@ def extract_from_file(path: pathlib.Path) -> list[PermissionEntry]:
|
||||
method = _extract_method_arg(inner)
|
||||
entity = _extract_entity_arg(inner)
|
||||
resource = _build_resource_label(fn_name, entity)
|
||||
permission = entity if fn_name == "requires_access_view" else method
|
||||
view_base, forced_view = _FN_TO_RESOURCE_INFO.get(fn_name, (None, None))
|
||||
permission = (forced_view or entity) if view_base == "View" else method
|
||||
if not isinstance(permission, str):
|
||||
raise ValueError(
|
||||
f"Could not resolve required permission for {fn_name} in {path.name}"
|
||||
|
||||
@@ -158,6 +158,23 @@ def _entity_maps() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
|
||||
if member and names:
|
||||
view_map[member] = names[0]
|
||||
|
||||
# Views newer than FAB's minimum Airflow are mapped by subscript assignment, keyed on a
|
||||
# ``<MEMBER>_ACCESS_VIEW`` name that is ``None`` on an older core.
|
||||
for stmt in ast.walk(tree):
|
||||
if not isinstance(stmt, ast.Assign) or len(stmt.targets) != 1:
|
||||
continue
|
||||
target = stmt.targets[0]
|
||||
if not (
|
||||
isinstance(target, ast.Subscript)
|
||||
and _name_of(target.value) == "_MAP_ACCESS_VIEW_TO_FAB_RESOURCE_TYPE"
|
||||
and isinstance(target.slice, ast.Name)
|
||||
and target.slice.id.endswith("_ACCESS_VIEW")
|
||||
):
|
||||
continue
|
||||
names = _const_names(stmt.value)
|
||||
if names:
|
||||
view_map[target.slice.id.removesuffix("_ACCESS_VIEW")] = names[0]
|
||||
|
||||
return dag_map, view_map
|
||||
|
||||
|
||||
|
||||
@@ -514,6 +514,23 @@ class TestExtractFromFile:
|
||||
assert e.resource == "View.PLUGINS"
|
||||
assert e.required_permission == "PLUGINS"
|
||||
|
||||
def test_dag_bundles_access_is_a_view(self, tmp_path):
|
||||
f = _make_route_file(
|
||||
tmp_path,
|
||||
"""
|
||||
from fastapi import Depends
|
||||
dag_bundles_router = AirflowRouter(tags=["Dag Bundle"], prefix="/dagBundles")
|
||||
|
||||
@dag_bundles_router.get("", dependencies=[Depends(requires_access_dag_bundles())])
|
||||
def get_dag_bundles(): ...
|
||||
""",
|
||||
)
|
||||
entries = extract_from_file(f)
|
||||
assert len(entries) == 1
|
||||
e = entries[0]
|
||||
assert e.resource == "View.DAG_BUNDLES"
|
||||
assert e.required_permission == "DAG_BUNDLES"
|
||||
|
||||
def test_no_dependencies_kwarg_extracted_as_public(self, tmp_path):
|
||||
f = _make_route_file(
|
||||
tmp_path,
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
from unittest import mock
|
||||
|
||||
import fab_permissions_doc
|
||||
|
||||
|
||||
def test_entity_maps_reads_views_mapped_by_assignment(tmp_path):
|
||||
source = tmp_path / "fab_auth_manager.py"
|
||||
source.write_text(
|
||||
textwrap.dedent(
|
||||
"""
|
||||
_MAP_ACCESS_VIEW_TO_FAB_RESOURCE_TYPE = {AccessView.JOBS: RESOURCE_JOB}
|
||||
if DAG_BUNDLES_ACCESS_VIEW is not None:
|
||||
_MAP_ACCESS_VIEW_TO_FAB_RESOURCE_TYPE[DAG_BUNDLES_ACCESS_VIEW] = RESOURCE_DAG_BUNDLE
|
||||
"""
|
||||
)
|
||||
)
|
||||
with mock.patch.object(fab_permissions_doc, "AUTH_MANAGER_PY", source):
|
||||
_, view_map = fab_permissions_doc._entity_maps()
|
||||
|
||||
assert view_map == {"JOBS": "RESOURCE_JOB", "DAG_BUNDLES": "RESOURCE_DAG_BUNDLE"}
|
||||
Reference in new issue
Block a user