Add schema-validated registry API contracts with API Explorer (#63025)

Pydantic contract models validate every JSON payload the registry
produces (providers catalog, modules, parameters, connections,
version metadata, provider versions). Validation runs at generation
time — no separate jsonschema layer needed.

An OpenAPI 3.1 spec is generated from these contracts at build time
and powers an interactive API Explorer at /api-explorer/, using
swagger-ui vendored from node_modules (Apache Infra blocks external
CDN via CSP). The explorer shares the site layout (header, nav,
footer) and uses swagger-ui's native dark-mode class for theme sync.

Also adds a /api/providers/{id}/versions.json endpoint used by CI
to publish new provider versions without a full site rebuild.
This commit is contained in:
Kaxil Naik authored and GitHub committed 2026-03-13 04:09:05 +00:00
1 parent 8db394977f
commit 5ac251bd53
26 files changed
+930 -62

No files matched your search

@@ -145,6 +145,8 @@ def publish_versions(s3_bucket: str, providers_json_path: Path | None = None) ->
# If the declared latest isn't deployed yet, use the highest deployed version.
effective_latest = latest if latest in versions else versions[0]
if effective_latest not in versions:
raise ValueError(f"latest version {effective_latest!r} not in versions list for {pid}")
data = {"versions": versions, "latest": effective_latest}
key = (
f"{prefix}/api/providers/{pid}/versions.json" if prefix else f"api/providers/{pid}/versions.json"
@@ -1,3 +1,4 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,4 +15,30 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Generate the OpenAPI spec for the registry API from shared Pydantic contracts.
Usage:
python dev/registry/export_registry_schemas.py
"""
from __future__ import annotations
import json
from pathlib import Path
from registry_contract_models import build_openapi_document
AIRFLOW_ROOT = Path(__file__).resolve().parents[2]
OPENAPI_PATH = AIRFLOW_ROOT / "registry" / "schemas" / "openapi.json"
def main() -> int:
content = json.dumps(build_openapi_document(), indent=2, sort_keys=True) + "\n"
OPENAPI_PATH.parent.mkdir(parents=True, exist_ok=True)
OPENAPI_PATH.write_text(content)
print(f" wrote {OPENAPI_PATH}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+11 -7
View File
@@ -43,6 +43,8 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from registry_contract_models import validate_provider_connections
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
SCRIPT_DIR = Path(__file__).parent
@@ -264,13 +266,15 @@ def main():
version_dir = output_dir / "versions" / pid / version
version_dir.mkdir(parents=True, exist_ok=True)
provider_data = {
"provider_id": pid,
"provider_name": provider_names.get(pid, pid),
"version": version,
"generated_at": generated_at,
"connection_types": conns,
}
provider_data = validate_provider_connections(
{
"provider_id": pid,
"provider_name": provider_names.get(pid, pid),
"version": version,
"generated_at": generated_at,
"connection_types": conns,
}
)
with open(version_dir / "connections.json", "w") as f:
json.dump(provider_data, f, separators=(",", ":"))
written += 1
+2 -1
View File
@@ -43,6 +43,7 @@ from typing import Any
import tomllib
import yaml
from registry_contract_models import validate_providers_catalog
# External endpoints used by metadata extraction.
PYPISTATS_RECENT_URL = "https://pypistats.org/api/packages/{package_name}/recent"
@@ -626,7 +627,7 @@ def main():
break
new_providers.sort(key=lambda p: p["name"].lower())
providers_json = {"providers": new_providers}
providers_json = validate_providers_catalog({"providers": new_providers})
# Write output files to all output directories.
# Inside breeze, registry/ is not mounted so OUTPUT_DIR writes are lost.
+11 -8
View File
@@ -53,6 +53,7 @@ from pathlib import Path
import yaml
from extract_metadata import fetch_provider_inventory, read_inventory
from registry_contract_models import validate_modules_catalog, validate_provider_parameters
from registry_tools.types import BASE_CLASS_IMPORTS, CLASS_LEVEL_SECTIONS, MODULE_LEVEL_SECTIONS
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
@@ -768,13 +769,15 @@ def _write_parameter_files(
version_dir = output_dir / "versions" / pid / version
version_dir.mkdir(parents=True, exist_ok=True)
provider_data = {
"provider_id": pid,
"provider_name": provider_names.get(pid, pid),
"version": version,
"generated_at": generated_at,
"classes": classes,
}
provider_data = validate_provider_parameters(
{
"provider_id": pid,
"provider_name": provider_names.get(pid, pid),
"version": version,
"generated_at": generated_at,
"classes": classes,
}
)
with open(version_dir / "parameters.json", "w") as f:
json.dump(provider_data, f, separators=(",", ":"))
written += 1
@@ -921,7 +924,7 @@ def _main_discover(
# With --provider, the output would be incomplete and would clobber the
# full modules.json from a previous build.
if not only_provider:
modules_json = {"modules": all_discovered}
modules_json = validate_modules_catalog({"modules": all_discovered})
output_dirs = [SCRIPT_DIR, AIRFLOW_ROOT / "registry" / "src" / "_data"]
for out_dir in output_dirs:
if not out_dir.parent.exists():
+14 -11
View File
@@ -47,6 +47,7 @@ from pathlib import Path
from typing import Any
import tomllib
from registry_contract_models import validate_provider_version_metadata
try:
import yaml
@@ -389,17 +390,19 @@ def extract_version_data(
modules = extract_modules_from_yaml(provider_yaml, tag, layout, dir_path, provider_id, version)
module_counts = count_modules(modules)
return {
"provider_id": provider_id,
"version": version,
"generated_at": datetime.now(timezone.utc).isoformat(),
"requires_python": pyproject_data["requires_python"],
"dependencies": pyproject_data["dependencies"],
"optional_extras": pyproject_data["optional_extras"],
"connection_types": connection_types,
"module_counts": module_counts,
"modules": modules,
}
return validate_provider_version_metadata(
{
"provider_id": provider_id,
"version": version,
"generated_at": datetime.now(timezone.utc).isoformat(),
"requires_python": pyproject_data["requires_python"],
"dependencies": pyproject_data["dependencies"],
"optional_extras": pyproject_data["optional_extras"],
"connection_types": connection_types,
"module_counts": module_counts,
"modules": modules,
}
)
def extract_and_write_version_data(provider_id: str, version: str, dir_path: str) -> dict[str, Any] | None:
+7 -2
View File
@@ -37,6 +37,8 @@ import argparse
import json
from pathlib import Path
from registry_contract_models import validate_modules_catalog, validate_providers_catalog
def merge(
existing_providers_path: Path,
@@ -72,10 +74,13 @@ def merge(
date_by_provider = {p["id"]: p.get("last_updated", "") for p in merged_providers}
merged_modules.sort(key=lambda m: date_by_provider.get(m["provider_id"], ""), reverse=True)
providers_payload = validate_providers_catalog({"providers": merged_providers})
modules_payload = validate_modules_catalog({"modules": merged_modules})
# Write merged output
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "providers.json").write_text(json.dumps({"providers": merged_providers}, indent=2) + "\n")
(output_dir / "modules.json").write_text(json.dumps({"modules": merged_modules}, indent=2) + "\n")
(output_dir / "providers.json").write_text(json.dumps(providers_payload, indent=2) + "\n")
(output_dir / "modules.json").write_text(json.dumps(modules_payload, indent=2) + "\n")
print(f"Merged {len(new_ids)} updated provider(s) into {len(merged_providers)} total providers")
print(f"Total modules: {len(merged_modules)}")
+4 -1
View File
@@ -32,7 +32,10 @@ description = "Extraction and build tools for the Airflow Provider Registry"
version = "0.0.1"
requires-python = ">=3.10"
classifiers = ["Private :: Do Not Upload"]
dependencies = ["pyyaml>=6.0.3"]
dependencies = [
"pydantic>=2.12.0",
"pyyaml>=6.0.3",
]
[dependency-groups]
dev = ["pytest"]
+472
View File
@@ -0,0 +1,472 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Shared contracts for registry generator payloads and API schemas.
The generator scripts use these Pydantic models to validate payloads before
writing JSON files. The same models are also used to export JSON Schema
artifacts and to build the OpenAPI document for the registry API.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from pydantic import BaseModel, ConfigDict, Field, model_validator
class CategoryContract(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str
name: str
module_count: int = 0
class DownloadStatsContract(BaseModel):
model_config = ConfigDict(extra="forbid")
weekly: int = 0
monthly: int = 0
total: int = 0
class ConnectionTypeContract(BaseModel):
model_config = ConfigDict(extra="forbid")
conn_type: str
hook_class: str = ""
docs_url: str | None = None
class ProviderContract(BaseModel):
"""Top-level provider entry in providers.json."""
model_config = ConfigDict(extra="forbid")
id: str
name: str
package_name: str
description: str
lifecycle: str = "production"
logo: str | None = None
version: str
versions: list[str]
airflow_versions: list[str] = Field(default_factory=list)
pypi_downloads: DownloadStatsContract = Field(default_factory=DownloadStatsContract)
module_counts: dict[str, int] = Field(default_factory=dict)
categories: list[CategoryContract] = Field(default_factory=list)
connection_types: list[ConnectionTypeContract] = Field(default_factory=list)
requires_python: str = ""
dependencies: list[str] = Field(default_factory=list)
optional_extras: dict[str, list[str]] = Field(default_factory=dict)
dependents: list[str] = Field(default_factory=list)
related_providers: list[str] = Field(default_factory=list)
docs_url: str
source_url: str
pypi_url: str
first_released: str = ""
last_updated: str = ""
class ProvidersCatalogContract(BaseModel):
model_config = ConfigDict(extra="forbid")
providers: list[ProviderContract]
class ModuleContract(BaseModel):
"""A registry module entry.
``module_path`` is optional for older versioned metadata generated from git
tags where only import paths are available.
"""
model_config = ConfigDict(extra="forbid")
id: str | None = None
name: str
type: str
import_path: str
module_path: str | None = None
short_description: str
docs_url: str
source_url: str
category: str
provider_id: str | None = None
provider_name: str | None = None
class ModulesCatalogContract(BaseModel):
model_config = ConfigDict(extra="forbid")
modules: list[ModuleContract]
class ProviderModulesContract(BaseModel):
model_config = ConfigDict(extra="forbid")
provider_id: str
provider_name: str
version: str
modules: list[ModuleContract]
class ParameterContract(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str
type: str | None = None
default: Any = None
required: bool
origin: str
description: str | None = None
class ClassParametersEntryContract(BaseModel):
model_config = ConfigDict(extra="forbid", populate_by_name=True)
name: str
type: str
mro_chain: list[str] = Field(alias="mro", serialization_alias="mro")
parameters: list[ParameterContract]
class ProviderParametersContract(BaseModel):
model_config = ConfigDict(extra="forbid")
provider_id: str
provider_name: str
version: str
generated_at: str | None = None
classes: dict[str, ClassParametersEntryContract]
class StandardConnectionFieldContract(BaseModel):
model_config = ConfigDict(extra="forbid")
visible: bool
label: str
placeholder: str | None = None
class CustomConnectionFieldContract(BaseModel):
# Keep this extensible for provider-specific form metadata.
model_config = ConfigDict(extra="allow")
label: str
type: Any
default: Any = None
format: str | None = None
description: str | None = None
is_sensitive: bool = False
enum: list[Any] | None = None
minimum: int | float | None = None
maximum: int | float | None = None
class ProviderConnectionTypeContract(BaseModel):
# Keep this extensible for provider-specific hook metadata.
model_config = ConfigDict(extra="allow")
connection_type: str
hook_class: str | None = None
standard_fields: dict[str, StandardConnectionFieldContract]
custom_fields: dict[str, CustomConnectionFieldContract] = Field(default_factory=dict)
class ProviderConnectionsContract(BaseModel):
model_config = ConfigDict(extra="forbid")
provider_id: str
provider_name: str
version: str
generated_at: str | None = None
connection_types: list[ProviderConnectionTypeContract]
class ProviderVersionMetadataContract(BaseModel):
model_config = ConfigDict(extra="forbid")
provider_id: str
version: str
generated_at: str
requires_python: str
dependencies: list[str]
optional_extras: dict[str, list[str]]
connection_types: list[ConnectionTypeContract]
module_counts: dict[str, int]
modules: list[ModuleContract]
class ProviderVersionsContract(BaseModel):
model_config = ConfigDict(extra="forbid")
latest: str
versions: list[str]
@model_validator(mode="after")
def ensure_latest_is_listed(self) -> ProviderVersionsContract:
if self.latest not in self.versions:
raise ValueError("latest version must be included in versions list")
return self
def _validate(model_type: type[BaseModel], payload: dict[str, Any]) -> dict[str, Any]:
model_type.model_validate(payload)
return payload
def validate_providers_catalog(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ProvidersCatalogContract, payload)
def validate_modules_catalog(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ModulesCatalogContract, payload)
def validate_provider_modules(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ProviderModulesContract, payload)
def validate_provider_parameters(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ProviderParametersContract, payload)
def validate_provider_connections(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ProviderConnectionsContract, payload)
def validate_provider_version_metadata(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ProviderVersionMetadataContract, payload)
def validate_provider_versions(payload: dict[str, Any]) -> dict[str, Any]:
return _validate(ProviderVersionsContract, payload)
@dataclass(frozen=True)
class OpenApiEndpoint:
path: str
tag: str
operation_id: str
summary: str
response_description: str
response_component: str
parameters: tuple[str, ...] = ()
include_not_found: bool = False
OPENAPI_ENDPOINTS: tuple[OpenApiEndpoint, ...] = (
OpenApiEndpoint(
path="/api/providers.json",
tag="Catalog",
operation_id="listProviders",
summary="List providers",
response_description="Provider catalog.",
response_component="ProvidersCatalogPayload",
),
OpenApiEndpoint(
path="/api/modules.json",
tag="Catalog",
operation_id="listModules",
summary="List modules",
response_description="Module catalog.",
response_component="ModulesCatalogPayload",
),
OpenApiEndpoint(
path="/api/providers/{providerId}/modules.json",
tag="Providers",
operation_id="getProviderModulesLatest",
summary="Get provider modules (latest)",
response_description="Provider modules for latest version.",
response_component="ProviderModulesPayload",
parameters=("ProviderId",),
include_not_found=True,
),
OpenApiEndpoint(
path="/api/providers/{providerId}/parameters.json",
tag="Providers",
operation_id="getProviderParametersLatest",
summary="Get provider parameters (latest)",
response_description="Provider parameters for latest version.",
response_component="ProviderParametersPayload",
parameters=("ProviderId",),
include_not_found=True,
),
OpenApiEndpoint(
path="/api/providers/{providerId}/connections.json",
tag="Providers",
operation_id="getProviderConnectionsLatest",
summary="Get provider connections (latest)",
response_description="Provider connections for latest version.",
response_component="ProviderConnectionsPayload",
parameters=("ProviderId",),
include_not_found=True,
),
OpenApiEndpoint(
path="/api/providers/{providerId}/{version}/modules.json",
tag="Provider Versions",
operation_id="getProviderModulesByVersion",
summary="Get provider modules (versioned)",
response_description="Versioned provider modules.",
response_component="ProviderModulesPayload",
parameters=("ProviderId", "Version"),
include_not_found=True,
),
OpenApiEndpoint(
path="/api/providers/{providerId}/{version}/parameters.json",
tag="Provider Versions",
operation_id="getProviderParametersByVersion",
summary="Get provider parameters (versioned)",
response_description="Versioned provider parameters.",
response_component="ProviderParametersPayload",
parameters=("ProviderId", "Version"),
include_not_found=True,
),
OpenApiEndpoint(
path="/api/providers/{providerId}/{version}/connections.json",
tag="Provider Versions",
operation_id="getProviderConnectionsByVersion",
summary="Get provider connections (versioned)",
response_description="Versioned provider connections.",
response_component="ProviderConnectionsPayload",
parameters=("ProviderId", "Version"),
include_not_found=True,
),
OpenApiEndpoint(
path="/api/providers/{providerId}/versions.json",
tag="Provider Versions",
operation_id="getProviderVersions",
summary="Get provider versions",
response_description="Published provider versions.",
response_component="ProviderVersionsPayload",
parameters=("ProviderId",),
include_not_found=True,
),
)
def _strip_schema_meta(schema: dict[str, Any]) -> dict[str, Any]:
sanitized = dict(schema)
sanitized.pop("$schema", None)
sanitized.pop("$id", None)
return sanitized
_OPENAPI_COMPONENT_MODELS: dict[str, type[BaseModel]] = {
"ProvidersCatalogPayload": ProvidersCatalogContract,
"ModulesCatalogPayload": ModulesCatalogContract,
"ProviderModulesPayload": ProviderModulesContract,
"ProviderParametersPayload": ProviderParametersContract,
"ProviderConnectionsPayload": ProviderConnectionsContract,
"ProviderVersionMetadataPayload": ProviderVersionMetadataContract,
"ProviderVersionsPayload": ProviderVersionsContract,
}
def _collect_openapi_component_schemas(
root_models: dict[str, type[BaseModel]],
) -> dict[str, dict[str, Any]]:
components: dict[str, dict[str, Any]] = {}
for schema_name, model_type in root_models.items():
schema = model_type.model_json_schema(ref_template="#/components/schemas/{model}")
defs = schema.pop("$defs", {})
schema = _strip_schema_meta(schema)
if schema_name in components and components[schema_name] != schema:
raise ValueError(f"Conflicting OpenAPI schema definition for {schema_name}")
components[schema_name] = schema
for def_name, def_schema in defs.items():
cleaned = _strip_schema_meta(def_schema)
if def_name in components and components[def_name] != cleaned:
raise ValueError(f"Conflicting OpenAPI schema definition for {def_name}")
components.setdefault(def_name, cleaned)
return components
def _build_openapi_get_operation(endpoint: OpenApiEndpoint) -> dict[str, Any]:
operation: dict[str, Any] = {
"tags": [endpoint.tag],
"operationId": endpoint.operation_id,
"summary": endpoint.summary,
"responses": {
"200": {
"description": endpoint.response_description,
"content": {
"application/json": {
"schema": {"$ref": f"#/components/schemas/{endpoint.response_component}"}
}
},
}
},
}
if endpoint.parameters:
operation["parameters"] = [
{"$ref": f"#/components/parameters/{parameter_name}"} for parameter_name in endpoint.parameters
]
if endpoint.include_not_found:
operation["responses"]["404"] = {"$ref": "#/components/responses/NotFound"}
return operation
def _build_openapi_paths(endpoints: tuple[OpenApiEndpoint, ...]) -> dict[str, dict[str, Any]]:
return {endpoint.path: {"get": _build_openapi_get_operation(endpoint)} for endpoint in endpoints}
def build_openapi_document() -> dict[str, Any]:
"""Build OpenAPI 3.1 schema from shared registry contracts."""
component_schemas = _collect_openapi_component_schemas(_OPENAPI_COMPONENT_MODELS)
return {
"openapi": "3.1.0",
"jsonSchemaDialect": "https://spec.openapis.org/oas/3.1/dialect/base",
"info": {
"title": "Airflow Registry API",
"version": "1.0.0",
"description": "JSON endpoints for Apache Airflow provider and module discovery.",
},
"tags": [
{"name": "Catalog", "description": "Global registry datasets."},
{"name": "Providers", "description": "Provider-scoped latest metadata."},
{"name": "Provider Versions", "description": "Version-specific provider metadata."},
],
"paths": _build_openapi_paths(OPENAPI_ENDPOINTS),
"components": {
"parameters": {
"ProviderId": {
"name": "providerId",
"in": "path",
"required": True,
"schema": {"type": "string"},
"description": "Provider identifier (for example: amazon).",
},
"Version": {
"name": "version",
"in": "path",
"required": True,
"schema": {"type": "string"},
"description": "Provider version (for example: 9.22.0).",
},
},
"responses": {
"NotFound": {"description": "Static endpoint file not found."},
},
"schemas": component_schemas,
},
}
+66 -22
View File
@@ -37,14 +37,58 @@ def _write_json(path: Path, data: dict) -> Path:
return path
def _provider(provider_id: str, name: str, last_updated: str) -> dict:
return {
"id": provider_id,
"name": name,
"package_name": f"apache-airflow-providers-{provider_id}",
"description": f"{name} provider",
"lifecycle": "production",
"logo": None,
"version": "1.0.0",
"versions": ["1.0.0"],
"airflow_versions": ["3.0+"],
"pypi_downloads": {"weekly": 0, "monthly": 0, "total": 0},
"module_counts": {"operator": 1},
"categories": [],
"connection_types": [],
"requires_python": ">=3.10",
"dependencies": [],
"optional_extras": {},
"dependents": [],
"related_providers": [],
"docs_url": "https://example.invalid/docs",
"source_url": "https://example.invalid/source",
"pypi_url": "https://example.invalid/pypi",
"first_released": "",
"last_updated": last_updated,
}
def _module(module_id: str, provider_id: str) -> dict:
return {
"id": module_id,
"name": "ExampleOperator",
"type": "operator",
"import_path": f"airflow.providers.{provider_id}.operators.example.ExampleOperator",
"module_path": f"airflow.providers.{provider_id}.operators.example",
"short_description": "Example operator",
"docs_url": "https://example.invalid/docs",
"source_url": "https://example.invalid/source",
"category": "test",
"provider_id": provider_id,
"provider_name": provider_id.capitalize(),
}
class TestMerge:
def test_replaces_existing_provider(self, tmp_path, output_dir):
existing_providers = _write_json(
tmp_path / "existing_providers.json",
{
"providers": [
{"id": "amazon", "name": "Amazon", "last_updated": "2024-01-01"},
{"id": "google", "name": "Google", "last_updated": "2024-02-01"},
_provider("amazon", "Amazon", "2024-01-01"),
_provider("google", "Google", "2024-02-01"),
]
},
)
@@ -52,18 +96,18 @@ class TestMerge:
tmp_path / "existing_modules.json",
{
"modules": [
{"id": "amazon-s3-op", "provider_id": "amazon"},
{"id": "google-bq-op", "provider_id": "google"},
_module("amazon-s3-op", "amazon"),
_module("google-bq-op", "google"),
]
},
)
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": "2025-01-01"}]},
{"providers": [_provider("amazon", "Amazon", "2025-01-01")]},
)
new_modules = _write_json(
tmp_path / "new_modules.json",
{"modules": [{"id": "amazon-s3-op-v2", "provider_id": "amazon"}]},
{"modules": [_module("amazon-s3-op-v2", "amazon")]},
)
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
@@ -88,19 +132,19 @@ class TestMerge:
def test_adds_new_provider(self, tmp_path, output_dir):
existing_providers = _write_json(
tmp_path / "existing_providers.json",
{"providers": [{"id": "google", "name": "Google", "last_updated": "2024-02-01"}]},
{"providers": [_provider("google", "Google", "2024-02-01")]},
)
existing_modules = _write_json(
tmp_path / "existing_modules.json",
{"modules": [{"id": "google-bq-op", "provider_id": "google"}]},
{"modules": [_module("google-bq-op", "google")]},
)
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": "2025-01-01"}]},
{"providers": [_provider("amazon", "Amazon", "2025-01-01")]},
)
new_modules = _write_json(
tmp_path / "new_modules.json",
{"modules": [{"id": "amazon-s3-op", "provider_id": "amazon"}]},
{"modules": [_module("amazon-s3-op", "amazon")]},
)
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
@@ -112,12 +156,12 @@ class TestMerge:
def test_providers_sorted_by_name(self, tmp_path, output_dir):
existing_providers = _write_json(
tmp_path / "existing_providers.json",
{"providers": [{"id": "zzz", "name": "Zzz Provider", "last_updated": ""}]},
{"providers": [_provider("zzz", "Zzz Provider", "")]},
)
existing_modules = _write_json(tmp_path / "existing_modules.json", {"modules": []})
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "aaa", "name": "Aaa Provider", "last_updated": ""}]},
{"providers": [_provider("aaa", "Aaa Provider", "")]},
)
new_modules = _write_json(tmp_path / "new_modules.json", {"modules": []})
@@ -132,21 +176,21 @@ class TestMerge:
tmp_path / "existing_providers.json",
{
"providers": [
{"id": "old", "name": "Old Provider", "last_updated": "2020-01-01"},
_provider("old", "Old Provider", "2020-01-01"),
]
},
)
existing_modules = _write_json(
tmp_path / "existing_modules.json",
{"modules": [{"id": "old-mod", "provider_id": "old"}]},
{"modules": [_module("old-mod", "old")]},
)
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "new", "name": "New Provider", "last_updated": "2025-06-01"}]},
{"providers": [_provider("new", "New Provider", "2025-06-01")]},
)
new_modules = _write_json(
tmp_path / "new_modules.json",
{"modules": [{"id": "new-mod", "provider_id": "new"}]},
{"modules": [_module("new-mod", "new")]},
)
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
@@ -159,17 +203,17 @@ class TestMerge:
def test_missing_existing_modules_file(self, tmp_path, output_dir):
existing_providers = _write_json(
tmp_path / "existing_providers.json",
{"providers": [{"id": "google", "name": "Google", "last_updated": ""}]},
{"providers": [_provider("google", "Google", "")]},
)
# existing_modules file does not exist
existing_modules = tmp_path / "nonexistent_modules.json"
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": ""}]},
{"providers": [_provider("amazon", "Amazon", "")]},
)
new_modules = _write_json(
tmp_path / "new_modules.json",
{"modules": [{"id": "amazon-s3-op", "provider_id": "amazon"}]},
{"modules": [_module("amazon-s3-op", "amazon")]},
)
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
@@ -184,11 +228,11 @@ class TestMerge:
existing_modules = _write_json(tmp_path / "existing_modules.json", {"modules": []})
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": ""}]},
{"providers": [_provider("amazon", "Amazon", "")]},
)
new_modules = _write_json(
tmp_path / "new_modules.json",
{"modules": [{"id": "amazon-s3-op", "provider_id": "amazon"}]},
{"modules": [_module("amazon-s3-op", "amazon")]},
)
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
@@ -206,7 +250,7 @@ class TestMerge:
existing_modules = _write_json(tmp_path / "existing_modules.json", {"modules": []})
new_providers = _write_json(
tmp_path / "new_providers.json",
{"providers": [{"id": "test", "name": "Test", "last_updated": ""}]},
{"providers": [_provider("test", "Test", "")]},
)
new_modules = _write_json(tmp_path / "new_modules.json", {"modules": []})
@@ -0,0 +1,103 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Unit tests for dev/registry/registry_contract_models.py."""
from __future__ import annotations
import pytest
from pydantic import ValidationError
from registry_contract_models import (
build_openapi_document,
validate_provider_parameters,
validate_provider_version_metadata,
validate_provider_versions,
)
def test_build_openapi_document_produces_valid_spec():
doc = build_openapi_document()
assert doc["openapi"] == "3.1.0"
assert "/api/providers.json" in doc["paths"]
assert "/api/providers/{providerId}/versions.json" in doc["paths"]
# Every endpoint $ref should resolve to a component schema
for path_item in doc["paths"].values():
ref = path_item["get"]["responses"]["200"]["content"]["application/json"]["schema"]["$ref"]
schema_name = ref.split("/")[-1]
assert schema_name in doc["components"]["schemas"], f"{ref} not in component schemas"
def test_validate_provider_versions_requires_latest_in_versions():
with pytest.raises(ValidationError):
validate_provider_versions({"latest": "2.0.0", "versions": ["1.0.0"]})
def test_validate_provider_parameters_preserves_mro_alias():
payload = {
"provider_id": "test",
"provider_name": "Test",
"version": "1.0.0",
"generated_at": "2026-02-18T00:00:00+00:00",
"classes": {
"airflow.providers.test.mod.Example": {
"name": "Example",
"type": "operator",
"mro": ["airflow.providers.test.mod.Example"],
"parameters": [
{
"name": "value",
"type": "str",
"default": None,
"required": True,
"origin": "airflow.providers.test.mod.Example",
"description": None,
}
],
}
},
}
validated = validate_provider_parameters(payload)
class_entry = validated["classes"]["airflow.providers.test.mod.Example"]
assert "mro" in class_entry
assert "mro_chain" not in class_entry
def test_validate_version_metadata_accepts_legacy_version_modules_without_ids():
payload = {
"provider_id": "test",
"version": "0.9.0",
"generated_at": "2026-02-18T00:00:00+00:00",
"requires_python": ">=3.10",
"dependencies": [],
"optional_extras": {},
"connection_types": [{"conn_type": "test", "hook_class": "x.y.Hook"}],
"module_counts": {"operator": 1},
"modules": [
{
"name": "LegacyOperator",
"type": "operator",
"import_path": "airflow.providers.test.operators.legacy.LegacyOperator",
"short_description": "Legacy module shape from older backfills.",
"docs_url": "https://example.invalid/docs",
"source_url": "https://example.invalid/source",
"category": "test",
}
],
}
validated = validate_provider_version_metadata(payload)
assert "id" not in validated["modules"][0]
assert "provider_id" not in validated["modules"][0]
assert "provider_name" not in validated["modules"][0]
+8
View File
@@ -26,6 +26,14 @@ module.exports = function(eleventyConfig) {
// Copy public directory contents to root of output
eleventyConfig.addPassthroughCopy({ "public": "/" });
// Vendor swagger-ui from node_modules (external CDN blocked by Apache Infra CSP)
eleventyConfig.addPassthroughCopy({
"node_modules/swagger-ui-dist/swagger-ui.css": "vendor/swagger-ui/swagger-ui.css",
});
eleventyConfig.addPassthroughCopy({
"node_modules/swagger-ui-dist/swagger-ui-bundle.js": "vendor/swagger-ui/swagger-ui-bundle.js",
});
// Watch CSS and JS for changes
eleventyConfig.addWatchTarget("src/css/");
eleventyConfig.addWatchTarget("src/js/");
+3
View File
@@ -6,6 +6,9 @@ node_modules/
_site/
dist/
# Generated OpenAPI spec (produced by dev/registry/export_registry_schemas.py at build time)
schemas/
# Provider logos (copied from providers/*/docs/integration-logos/ at build time)
public/logos/
+5 -3
View File
@@ -10,9 +10,9 @@ This document contains rules, patterns, and guidelines for working on the Airflo
### 0. Minimal JavaScript
- Keep JavaScript to minimum necessary
- Theme toggle only
- Theme toggle + progressive enhancement (filters, search)
- No frameworks (no React, Vue, etc.)
- Progressive enhancement approach
- Exception: swagger-ui is vendored from `node_modules/swagger-ui-dist` for the API Explorer
## 1. Quality Standards
@@ -386,7 +386,9 @@ Key files in the project:
- `src/css/main.css` - Main styles
- `src/css/tokens.css` - Design tokens
- `src/` - All page templates (index, provider-detail, providers, explore, stats)
- `src/` - All page templates (index, provider-detail, providers, explore, stats, api-explorer)
- `../dev/registry/registry_contract_models.py` - Pydantic contracts for all JSON payloads
- `../dev/registry/export_registry_schemas.py` - Generates OpenAPI spec from contracts
## Development Server
+16
View File
@@ -125,6 +125,15 @@ extraction:
types, defaults, and docstrings. Writes
`versions/{provider_id}/{version}/parameters.json`.
**`registry_contract_models.py`** defines Pydantic models that validate the shape of
every JSON payload the registry produces. Each extraction script calls
`_validate(ModelType, payload)` before writing JSON — this catches schema drift at
generation time without a separate jsonschema layer. The same models generate the
OpenAPI 3.1 spec served at `/api/openapi.json`.
**`export_registry_schemas.py`** generates `registry/schemas/openapi.json` from the
contract models. It runs automatically via `pnpm prebuild` before Eleventy builds.
**`extract_connections.py`** (runs inside breeze) reads `connection-types` from
`provider.yaml`, falling back to runtime inspection of hook
`get_connection_form_widgets()` and `get_ui_field_behaviour()`. Writes
@@ -177,6 +186,8 @@ the same Sphinx build that generates the docs.
| `statsData.js` | Checked-in | Computed statistics (lifecycle counts, top providers, etc.) |
| `providerVersions.js` | Checked-in | Builds the provider × version page collection |
| `latestVersionData.js` | Checked-in | Latest version parameters/connections lookup |
| `openapiSpec.js` | Checked-in | Builds OpenAPI 3.1 spec from Pydantic contract models |
| `providerVersionPayloads.js` | Checked-in | Generates `/api/providers/{id}/versions.json` payloads |
### Pages
@@ -188,6 +199,7 @@ the same Sphinx build that generates the docs.
| Statistics | `src/stats.njk` | `/stats/` |
| Provider Detail | `src/provider-detail.njk` | `/providers/{id}/` (redirects to latest) |
| Provider Version | `src/provider-version.njk` | `/providers/{id}/{version}/` |
| API Explorer | `src/api-explorer.njk` | `/api-explorer/` |
### Client-Side JavaScript
@@ -272,6 +284,10 @@ providing programmatic access to provider and module data:
- `/api/providers/{id}/{version}/modules.json` — Version-specific modules
- `/api/providers/{id}/{version}/parameters.json` — Version-specific parameters
- `/api/providers/{id}/{version}/connections.json` — Version-specific connections
- `/api/openapi.json` — OpenAPI 3.1 spec (generated at build time from Pydantic contracts)
An interactive **API Explorer** at `/api-explorer/` renders the OpenAPI spec using
swagger-ui (vendored from `node_modules/swagger-ui-dist`).
## Incremental Builds
+4 -2
View File
@@ -4,11 +4,13 @@
"description": "Apache Airflow Provider Registry",
"scripts": {
"dev": "REGISTRY_PATH_PREFIX=/ pnpm build && REGISTRY_PATH_PREFIX=/ eleventy --serve --port=8080",
"build": "eleventy",
"prebuild": "uv run python ../dev/registry/export_registry_schemas.py",
"build": "rm -rf _site && eleventy",
"postbuild": "cleancss -o _site/css/main.css _site/css/main.css && node scripts/build-pagefind-index.mjs"
},
"dependencies": {
"@11ty/eleventy": "^3.1.2"
"@11ty/eleventy": "^3.1.2",
"swagger-ui-dist": "^5.32.0"
},
"devDependencies": {
"clean-css-cli": "^5.6.3",
+15
View File
@@ -17,6 +17,9 @@ importers:
'@11ty/eleventy':
specifier: ^3.1.2
version: 3.1.2
swagger-ui-dist:
specifier: ^5.32.0
version: 5.32.0
devDependencies:
clean-css-cli:
specifier: ^5.6.3
@@ -93,6 +96,9 @@ packages:
cpu: [x64]
os: [win32]
'@scarf/scarf@1.4.0':
resolution: {integrity: sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ==}
'@sindresorhus/slugify@2.2.1':
resolution: {integrity: sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==}
engines: {node: '>=12'}
@@ -593,6 +599,9 @@ packages:
resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==}
engines: {node: '>=0.10.0'}
swagger-ui-dist@5.32.0:
resolution: {integrity: sha512-nKZB0OuDvacB0s/lC2gbge+RigYvGRGpLLMWMFxaTUwfM+CfndVk9Th2IaTinqXiz6Mn26GK2zriCpv6/+5m3Q==}
tinyglobby@0.2.15:
resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==}
engines: {node: '>=12.0.0'}
@@ -747,6 +756,8 @@ snapshots:
'@pagefind/windows-x64@1.4.0':
optional: true
'@scarf/scarf@1.4.0': {}
'@sindresorhus/slugify@2.2.1':
dependencies:
'@sindresorhus/transliterate': 1.6.0
@@ -1191,6 +1202,10 @@ snapshots:
strip-bom-string@1.0.0: {}
swagger-ui-dist@5.32.0:
dependencies:
'@scarf/scarf': 1.4.0
tinyglobby@0.2.15:
dependencies:
fdir: 6.5.0(picomatch@4.0.3)
+33
View File
@@ -0,0 +1,33 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
const fs = require("fs");
const path = require("path");
const OPENAPI_SCHEMA_PATH = path.join(__dirname, "..", "..", "schemas", "openapi.json");
module.exports = function() {
if (!fs.existsSync(OPENAPI_SCHEMA_PATH)) {
throw new Error(
`Missing OpenAPI schema artifact at ${OPENAPI_SCHEMA_PATH}. ` +
"Run `pnpm build` (prebuild generates it) or `uv run python ../dev/registry/export_registry_schemas.py`.",
);
}
return JSON.parse(fs.readFileSync(OPENAPI_SCHEMA_PATH, "utf8"));
};
@@ -0,0 +1,48 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
const providersData = require("./providers.json");
const buildProviderVersions = require("./providerVersions");
module.exports = function () {
const providerVersions = buildProviderVersions();
const payloads = {};
for (const provider of providersData.providers) {
payloads[provider.id] = {
latest: provider.version,
versions: [provider.version],
};
}
for (const pv of providerVersions) {
if (!pv || !pv.provider || !pv.provider.id || !pv.isLatest) {
continue;
}
const versions = Array.isArray(pv.availableVersions) && pv.availableVersions.length > 0
? pv.availableVersions
: [pv.provider.version];
payloads[pv.provider.id] = {
latest: pv.provider.version,
versions,
};
}
return payloads;
};
+2
View File
@@ -63,6 +63,7 @@
<a href="{{ '/providers/' | url }}">Providers</a>
<a href="{{ '/stats/' | url }}">Stats</a>
<a href="{{ '/explore/' | url }}">Explore</a>
<a href="{{ '/api-explorer/' | url }}">API Explorer</a>
<a href="https://airflow.apache.org/docs/" target="_blank" rel="noopener">
Docs
<svg width="12" height="12" fill="none" stroke="currentColor" viewBox="0 0 24 24" aria-hidden="true" style="display:inline;vertical-align:middle;margin-left:2px">
@@ -98,6 +99,7 @@
<a href="{{ '/providers/' | url }}">Providers</a>
<a href="{{ '/stats/' | url }}">Stats</a>
<a href="{{ '/explore/' | url }}">Explore</a>
<a href="{{ '/api-explorer/' | url }}">API Explorer</a>
<a href="https://airflow.apache.org/docs/" target="_blank" rel="noopener">Docs</a>
</nav>
</div>
+46
View File
@@ -0,0 +1,46 @@
---
layout: base.njk
title: API Explorer
description: Interactive explorer for the Airflow Registry JSON API.
permalink: "/api-explorer/"
mainClass: "api-explorer-page"
---
<link rel="stylesheet" href="{{ '/vendor/swagger-ui/swagger-ui.css' | url }}">
<div id="swagger-ui"></div>
<script src="{{ '/vendor/swagger-ui/swagger-ui-bundle.js' | url }}"></script>
<script>
(function () {
const openApiUrl = new URL("../api/openapi.json", window.location.href).toString();
const serverPath = new URL("../", window.location.href).pathname.replace(/\/$/, "") || "/";
async function renderSwaggerUi() {
try {
const response = await fetch(openApiUrl, { credentials: "same-origin" });
if (!response.ok) {
throw new Error(`OpenAPI load failed: ${response.status}`);
}
const spec = await response.json();
spec.servers = [{ url: serverPath, description: "Registry base path" }];
SwaggerUIBundle({
spec,
dom_id: "#swagger-ui",
deepLinking: true,
displayRequestDuration: true,
docExpansion: "list",
defaultModelsExpandDepth: -1,
tryItOutEnabled: true,
supportedSubmitMethods: ["get"],
presets: [SwaggerUIBundle.presets.apis],
});
} catch (error) {
document.getElementById("swagger-ui").innerHTML =
"<p style=\"padding: 1rem; font-family: sans-serif; color: #444;\">" +
"Could not load interactive console. " +
"<a href=\"" + openApiUrl + "\" target=\"_blank\" rel=\"noopener\">Open raw OpenAPI document</a>." +
"</p>";
}
}
renderSwaggerUi();
})();
</script>
+5
View File
@@ -0,0 +1,5 @@
---
permalink: "/api/openapi.json"
eleventyExcludeFromCollections: true
---
{{ openapiSpec | dump | safe }}
+13
View File
@@ -0,0 +1,13 @@
---
pagination:
data: providers.providers
size: 1
alias: provider
permalink: "/api/providers/{{ provider.id }}/versions.json"
eleventyExcludeFromCollections: true
---
{%- if providerVersionPayloads[provider.id] -%}
{{ providerVersionPayloads[provider.id] | dump | safe }}
{%- else -%}
{{ { "latest": provider.version, "versions": [provider.version] } | dump | safe }}
{%- endif -%}
+6
View File
@@ -4222,3 +4222,9 @@ main {
.error-links a:hover {
opacity: 0.9;
}
/* API Explorer (swagger-ui inside site layout) */
.api-explorer-page .swagger-ui .topbar,
.api-explorer-page .swagger-ui .scheme-container { display: none; }
.api-explorer-page .swagger-ui .info { margin: var(--space-4) 0; }
.api-explorer-page { padding-bottom: var(--space-8); }
+6 -5
View File
@@ -29,15 +29,16 @@
return html.style.colorScheme || 'dark';
}
// Update icon visibility based on current theme
function updateIcons() {
// Update icon visibility and swagger-ui dark mode class
function updateTheme() {
const isLight = getCurrentTheme() === 'light';
if (sunIcon) sunIcon.style.display = isLight ? 'block' : 'none';
if (moonIcon) moonIcon.style.display = isLight ? 'none' : 'block';
html.classList.toggle('dark-mode', !isLight);
}
// Initialize icons on load
updateIcons();
// Initialize on load
updateTheme();
// Toggle theme
if (themeToggle) {
@@ -47,7 +48,7 @@
html.style.colorScheme = newTheme;
localStorage.setItem('theme', newTheme);
updateIcons();
updateTheme();
});
}
})();
+1
View File
@@ -8,6 +8,7 @@ eleventyExcludeFromCollections: true
<url><loc>https://airflow.apache.org/registry/providers/</loc></url>
<url><loc>https://airflow.apache.org/registry/explore/</loc></url>
<url><loc>https://airflow.apache.org/registry/stats/</loc></url>
<url><loc>https://airflow.apache.org/registry/api-explorer/</loc></url>
{%- for provider in providers.providers %}
<url><loc>https://airflow.apache.org/registry/providers/{{ provider.id }}/{{ provider.version }}/</loc></url>
{%- endfor %}