mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Add schema-validated registry API contracts with API Explorer (#63025)
Pydantic contract models validate every JSON payload the registry
produces (providers catalog, modules, parameters, connections,
version metadata, provider versions). Validation runs at generation
time — no separate jsonschema layer needed.
An OpenAPI 3.1 spec is generated from these contracts at build time
and powers an interactive API Explorer at /api-explorer/, using
swagger-ui vendored from node_modules (Apache Infra blocks external
CDN via CSP). The explorer shares the site layout (header, nav,
footer) and uses swagger-ui's native dark-mode class for theme sync.
Also adds a /api/providers/{id}/versions.json endpoint used by CI
to publish new provider versions without a full site rebuild.
This commit is contained in:
1 parent
8db394977f
commit
5ac251bd53
26 files changed
+930
-62
No files matched your search
@@ -145,6 +145,8 @@ def publish_versions(s3_bucket: str, providers_json_path: Path | None = None) ->
|
||||
|
||||
# If the declared latest isn't deployed yet, use the highest deployed version.
|
||||
effective_latest = latest if latest in versions else versions[0]
|
||||
if effective_latest not in versions:
|
||||
raise ValueError(f"latest version {effective_latest!r} not in versions list for {pid}")
|
||||
data = {"versions": versions, "latest": effective_latest}
|
||||
key = (
|
||||
f"{prefix}/api/providers/{pid}/versions.json" if prefix else f"api/providers/{pid}/versions.json"
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
@@ -14,4 +15,30 @@
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Generate the OpenAPI spec for the registry API from shared Pydantic contracts.
|
||||
|
||||
Usage:
|
||||
python dev/registry/export_registry_schemas.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from registry_contract_models import build_openapi_document
|
||||
|
||||
AIRFLOW_ROOT = Path(__file__).resolve().parents[2]
|
||||
OPENAPI_PATH = AIRFLOW_ROOT / "registry" / "schemas" / "openapi.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
content = json.dumps(build_openapi_document(), indent=2, sort_keys=True) + "\n"
|
||||
OPENAPI_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
OPENAPI_PATH.write_text(content)
|
||||
print(f" wrote {OPENAPI_PATH}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -43,6 +43,8 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from registry_contract_models import validate_provider_connections
|
||||
|
||||
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
|
||||
SCRIPT_DIR = Path(__file__).parent
|
||||
|
||||
@@ -264,13 +266,15 @@ def main():
|
||||
version_dir = output_dir / "versions" / pid / version
|
||||
version_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
provider_data = {
|
||||
"provider_id": pid,
|
||||
"provider_name": provider_names.get(pid, pid),
|
||||
"version": version,
|
||||
"generated_at": generated_at,
|
||||
"connection_types": conns,
|
||||
}
|
||||
provider_data = validate_provider_connections(
|
||||
{
|
||||
"provider_id": pid,
|
||||
"provider_name": provider_names.get(pid, pid),
|
||||
"version": version,
|
||||
"generated_at": generated_at,
|
||||
"connection_types": conns,
|
||||
}
|
||||
)
|
||||
with open(version_dir / "connections.json", "w") as f:
|
||||
json.dump(provider_data, f, separators=(",", ":"))
|
||||
written += 1
|
||||
|
||||
@@ -43,6 +43,7 @@ from typing import Any
|
||||
|
||||
import tomllib
|
||||
import yaml
|
||||
from registry_contract_models import validate_providers_catalog
|
||||
|
||||
# External endpoints used by metadata extraction.
|
||||
PYPISTATS_RECENT_URL = "https://pypistats.org/api/packages/{package_name}/recent"
|
||||
@@ -626,7 +627,7 @@ def main():
|
||||
break
|
||||
|
||||
new_providers.sort(key=lambda p: p["name"].lower())
|
||||
providers_json = {"providers": new_providers}
|
||||
providers_json = validate_providers_catalog({"providers": new_providers})
|
||||
|
||||
# Write output files to all output directories.
|
||||
# Inside breeze, registry/ is not mounted so OUTPUT_DIR writes are lost.
|
||||
|
||||
@@ -53,6 +53,7 @@ from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from extract_metadata import fetch_provider_inventory, read_inventory
|
||||
from registry_contract_models import validate_modules_catalog, validate_provider_parameters
|
||||
from registry_tools.types import BASE_CLASS_IMPORTS, CLASS_LEVEL_SECTIONS, MODULE_LEVEL_SECTIONS
|
||||
|
||||
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
|
||||
@@ -768,13 +769,15 @@ def _write_parameter_files(
|
||||
version_dir = output_dir / "versions" / pid / version
|
||||
version_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
provider_data = {
|
||||
"provider_id": pid,
|
||||
"provider_name": provider_names.get(pid, pid),
|
||||
"version": version,
|
||||
"generated_at": generated_at,
|
||||
"classes": classes,
|
||||
}
|
||||
provider_data = validate_provider_parameters(
|
||||
{
|
||||
"provider_id": pid,
|
||||
"provider_name": provider_names.get(pid, pid),
|
||||
"version": version,
|
||||
"generated_at": generated_at,
|
||||
"classes": classes,
|
||||
}
|
||||
)
|
||||
with open(version_dir / "parameters.json", "w") as f:
|
||||
json.dump(provider_data, f, separators=(",", ":"))
|
||||
written += 1
|
||||
@@ -921,7 +924,7 @@ def _main_discover(
|
||||
# With --provider, the output would be incomplete and would clobber the
|
||||
# full modules.json from a previous build.
|
||||
if not only_provider:
|
||||
modules_json = {"modules": all_discovered}
|
||||
modules_json = validate_modules_catalog({"modules": all_discovered})
|
||||
output_dirs = [SCRIPT_DIR, AIRFLOW_ROOT / "registry" / "src" / "_data"]
|
||||
for out_dir in output_dirs:
|
||||
if not out_dir.parent.exists():
|
||||
|
||||
@@ -47,6 +47,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import tomllib
|
||||
from registry_contract_models import validate_provider_version_metadata
|
||||
|
||||
try:
|
||||
import yaml
|
||||
@@ -389,17 +390,19 @@ def extract_version_data(
|
||||
modules = extract_modules_from_yaml(provider_yaml, tag, layout, dir_path, provider_id, version)
|
||||
module_counts = count_modules(modules)
|
||||
|
||||
return {
|
||||
"provider_id": provider_id,
|
||||
"version": version,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"requires_python": pyproject_data["requires_python"],
|
||||
"dependencies": pyproject_data["dependencies"],
|
||||
"optional_extras": pyproject_data["optional_extras"],
|
||||
"connection_types": connection_types,
|
||||
"module_counts": module_counts,
|
||||
"modules": modules,
|
||||
}
|
||||
return validate_provider_version_metadata(
|
||||
{
|
||||
"provider_id": provider_id,
|
||||
"version": version,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"requires_python": pyproject_data["requires_python"],
|
||||
"dependencies": pyproject_data["dependencies"],
|
||||
"optional_extras": pyproject_data["optional_extras"],
|
||||
"connection_types": connection_types,
|
||||
"module_counts": module_counts,
|
||||
"modules": modules,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def extract_and_write_version_data(provider_id: str, version: str, dir_path: str) -> dict[str, Any] | None:
|
||||
|
||||
@@ -37,6 +37,8 @@ import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from registry_contract_models import validate_modules_catalog, validate_providers_catalog
|
||||
|
||||
|
||||
def merge(
|
||||
existing_providers_path: Path,
|
||||
@@ -72,10 +74,13 @@ def merge(
|
||||
date_by_provider = {p["id"]: p.get("last_updated", "") for p in merged_providers}
|
||||
merged_modules.sort(key=lambda m: date_by_provider.get(m["provider_id"], ""), reverse=True)
|
||||
|
||||
providers_payload = validate_providers_catalog({"providers": merged_providers})
|
||||
modules_payload = validate_modules_catalog({"modules": merged_modules})
|
||||
|
||||
# Write merged output
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "providers.json").write_text(json.dumps({"providers": merged_providers}, indent=2) + "\n")
|
||||
(output_dir / "modules.json").write_text(json.dumps({"modules": merged_modules}, indent=2) + "\n")
|
||||
(output_dir / "providers.json").write_text(json.dumps(providers_payload, indent=2) + "\n")
|
||||
(output_dir / "modules.json").write_text(json.dumps(modules_payload, indent=2) + "\n")
|
||||
|
||||
print(f"Merged {len(new_ids)} updated provider(s) into {len(merged_providers)} total providers")
|
||||
print(f"Total modules: {len(merged_modules)}")
|
||||
|
||||
@@ -32,7 +32,10 @@ description = "Extraction and build tools for the Airflow Provider Registry"
|
||||
version = "0.0.1"
|
||||
requires-python = ">=3.10"
|
||||
classifiers = ["Private :: Do Not Upload"]
|
||||
dependencies = ["pyyaml>=6.0.3"]
|
||||
dependencies = [
|
||||
"pydantic>=2.12.0",
|
||||
"pyyaml>=6.0.3",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest"]
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
#!/usr/bin/env python3
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Shared contracts for registry generator payloads and API schemas.
|
||||
|
||||
The generator scripts use these Pydantic models to validate payloads before
|
||||
writing JSON files. The same models are also used to export JSON Schema
|
||||
artifacts and to build the OpenAPI document for the registry API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class CategoryContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
name: str
|
||||
module_count: int = 0
|
||||
|
||||
|
||||
class DownloadStatsContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
weekly: int = 0
|
||||
monthly: int = 0
|
||||
total: int = 0
|
||||
|
||||
|
||||
class ConnectionTypeContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
conn_type: str
|
||||
hook_class: str = ""
|
||||
docs_url: str | None = None
|
||||
|
||||
|
||||
class ProviderContract(BaseModel):
|
||||
"""Top-level provider entry in providers.json."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
name: str
|
||||
package_name: str
|
||||
description: str
|
||||
lifecycle: str = "production"
|
||||
logo: str | None = None
|
||||
version: str
|
||||
versions: list[str]
|
||||
airflow_versions: list[str] = Field(default_factory=list)
|
||||
pypi_downloads: DownloadStatsContract = Field(default_factory=DownloadStatsContract)
|
||||
module_counts: dict[str, int] = Field(default_factory=dict)
|
||||
categories: list[CategoryContract] = Field(default_factory=list)
|
||||
connection_types: list[ConnectionTypeContract] = Field(default_factory=list)
|
||||
requires_python: str = ""
|
||||
dependencies: list[str] = Field(default_factory=list)
|
||||
optional_extras: dict[str, list[str]] = Field(default_factory=dict)
|
||||
dependents: list[str] = Field(default_factory=list)
|
||||
related_providers: list[str] = Field(default_factory=list)
|
||||
docs_url: str
|
||||
source_url: str
|
||||
pypi_url: str
|
||||
first_released: str = ""
|
||||
last_updated: str = ""
|
||||
|
||||
|
||||
class ProvidersCatalogContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
providers: list[ProviderContract]
|
||||
|
||||
|
||||
class ModuleContract(BaseModel):
|
||||
"""A registry module entry.
|
||||
|
||||
``module_path`` is optional for older versioned metadata generated from git
|
||||
tags where only import paths are available.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str | None = None
|
||||
name: str
|
||||
type: str
|
||||
import_path: str
|
||||
module_path: str | None = None
|
||||
short_description: str
|
||||
docs_url: str
|
||||
source_url: str
|
||||
category: str
|
||||
provider_id: str | None = None
|
||||
provider_name: str | None = None
|
||||
|
||||
|
||||
class ModulesCatalogContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
modules: list[ModuleContract]
|
||||
|
||||
|
||||
class ProviderModulesContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
provider_id: str
|
||||
provider_name: str
|
||||
version: str
|
||||
modules: list[ModuleContract]
|
||||
|
||||
|
||||
class ParameterContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
type: str | None = None
|
||||
default: Any = None
|
||||
required: bool
|
||||
origin: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class ClassParametersEntryContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
name: str
|
||||
type: str
|
||||
mro_chain: list[str] = Field(alias="mro", serialization_alias="mro")
|
||||
parameters: list[ParameterContract]
|
||||
|
||||
|
||||
class ProviderParametersContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
provider_id: str
|
||||
provider_name: str
|
||||
version: str
|
||||
generated_at: str | None = None
|
||||
classes: dict[str, ClassParametersEntryContract]
|
||||
|
||||
|
||||
class StandardConnectionFieldContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
visible: bool
|
||||
label: str
|
||||
placeholder: str | None = None
|
||||
|
||||
|
||||
class CustomConnectionFieldContract(BaseModel):
|
||||
# Keep this extensible for provider-specific form metadata.
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
label: str
|
||||
type: Any
|
||||
default: Any = None
|
||||
format: str | None = None
|
||||
description: str | None = None
|
||||
is_sensitive: bool = False
|
||||
enum: list[Any] | None = None
|
||||
minimum: int | float | None = None
|
||||
maximum: int | float | None = None
|
||||
|
||||
|
||||
class ProviderConnectionTypeContract(BaseModel):
|
||||
# Keep this extensible for provider-specific hook metadata.
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
connection_type: str
|
||||
hook_class: str | None = None
|
||||
standard_fields: dict[str, StandardConnectionFieldContract]
|
||||
custom_fields: dict[str, CustomConnectionFieldContract] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ProviderConnectionsContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
provider_id: str
|
||||
provider_name: str
|
||||
version: str
|
||||
generated_at: str | None = None
|
||||
connection_types: list[ProviderConnectionTypeContract]
|
||||
|
||||
|
||||
class ProviderVersionMetadataContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
provider_id: str
|
||||
version: str
|
||||
generated_at: str
|
||||
requires_python: str
|
||||
dependencies: list[str]
|
||||
optional_extras: dict[str, list[str]]
|
||||
connection_types: list[ConnectionTypeContract]
|
||||
module_counts: dict[str, int]
|
||||
modules: list[ModuleContract]
|
||||
|
||||
|
||||
class ProviderVersionsContract(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
latest: str
|
||||
versions: list[str]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def ensure_latest_is_listed(self) -> ProviderVersionsContract:
|
||||
if self.latest not in self.versions:
|
||||
raise ValueError("latest version must be included in versions list")
|
||||
return self
|
||||
|
||||
|
||||
def _validate(model_type: type[BaseModel], payload: dict[str, Any]) -> dict[str, Any]:
|
||||
model_type.model_validate(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def validate_providers_catalog(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ProvidersCatalogContract, payload)
|
||||
|
||||
|
||||
def validate_modules_catalog(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ModulesCatalogContract, payload)
|
||||
|
||||
|
||||
def validate_provider_modules(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ProviderModulesContract, payload)
|
||||
|
||||
|
||||
def validate_provider_parameters(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ProviderParametersContract, payload)
|
||||
|
||||
|
||||
def validate_provider_connections(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ProviderConnectionsContract, payload)
|
||||
|
||||
|
||||
def validate_provider_version_metadata(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ProviderVersionMetadataContract, payload)
|
||||
|
||||
|
||||
def validate_provider_versions(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate(ProviderVersionsContract, payload)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenApiEndpoint:
|
||||
path: str
|
||||
tag: str
|
||||
operation_id: str
|
||||
summary: str
|
||||
response_description: str
|
||||
response_component: str
|
||||
parameters: tuple[str, ...] = ()
|
||||
include_not_found: bool = False
|
||||
|
||||
|
||||
OPENAPI_ENDPOINTS: tuple[OpenApiEndpoint, ...] = (
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers.json",
|
||||
tag="Catalog",
|
||||
operation_id="listProviders",
|
||||
summary="List providers",
|
||||
response_description="Provider catalog.",
|
||||
response_component="ProvidersCatalogPayload",
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/modules.json",
|
||||
tag="Catalog",
|
||||
operation_id="listModules",
|
||||
summary="List modules",
|
||||
response_description="Module catalog.",
|
||||
response_component="ModulesCatalogPayload",
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/modules.json",
|
||||
tag="Providers",
|
||||
operation_id="getProviderModulesLatest",
|
||||
summary="Get provider modules (latest)",
|
||||
response_description="Provider modules for latest version.",
|
||||
response_component="ProviderModulesPayload",
|
||||
parameters=("ProviderId",),
|
||||
include_not_found=True,
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/parameters.json",
|
||||
tag="Providers",
|
||||
operation_id="getProviderParametersLatest",
|
||||
summary="Get provider parameters (latest)",
|
||||
response_description="Provider parameters for latest version.",
|
||||
response_component="ProviderParametersPayload",
|
||||
parameters=("ProviderId",),
|
||||
include_not_found=True,
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/connections.json",
|
||||
tag="Providers",
|
||||
operation_id="getProviderConnectionsLatest",
|
||||
summary="Get provider connections (latest)",
|
||||
response_description="Provider connections for latest version.",
|
||||
response_component="ProviderConnectionsPayload",
|
||||
parameters=("ProviderId",),
|
||||
include_not_found=True,
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/{version}/modules.json",
|
||||
tag="Provider Versions",
|
||||
operation_id="getProviderModulesByVersion",
|
||||
summary="Get provider modules (versioned)",
|
||||
response_description="Versioned provider modules.",
|
||||
response_component="ProviderModulesPayload",
|
||||
parameters=("ProviderId", "Version"),
|
||||
include_not_found=True,
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/{version}/parameters.json",
|
||||
tag="Provider Versions",
|
||||
operation_id="getProviderParametersByVersion",
|
||||
summary="Get provider parameters (versioned)",
|
||||
response_description="Versioned provider parameters.",
|
||||
response_component="ProviderParametersPayload",
|
||||
parameters=("ProviderId", "Version"),
|
||||
include_not_found=True,
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/{version}/connections.json",
|
||||
tag="Provider Versions",
|
||||
operation_id="getProviderConnectionsByVersion",
|
||||
summary="Get provider connections (versioned)",
|
||||
response_description="Versioned provider connections.",
|
||||
response_component="ProviderConnectionsPayload",
|
||||
parameters=("ProviderId", "Version"),
|
||||
include_not_found=True,
|
||||
),
|
||||
OpenApiEndpoint(
|
||||
path="/api/providers/{providerId}/versions.json",
|
||||
tag="Provider Versions",
|
||||
operation_id="getProviderVersions",
|
||||
summary="Get provider versions",
|
||||
response_description="Published provider versions.",
|
||||
response_component="ProviderVersionsPayload",
|
||||
parameters=("ProviderId",),
|
||||
include_not_found=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _strip_schema_meta(schema: dict[str, Any]) -> dict[str, Any]:
|
||||
sanitized = dict(schema)
|
||||
sanitized.pop("$schema", None)
|
||||
sanitized.pop("$id", None)
|
||||
return sanitized
|
||||
|
||||
|
||||
_OPENAPI_COMPONENT_MODELS: dict[str, type[BaseModel]] = {
|
||||
"ProvidersCatalogPayload": ProvidersCatalogContract,
|
||||
"ModulesCatalogPayload": ModulesCatalogContract,
|
||||
"ProviderModulesPayload": ProviderModulesContract,
|
||||
"ProviderParametersPayload": ProviderParametersContract,
|
||||
"ProviderConnectionsPayload": ProviderConnectionsContract,
|
||||
"ProviderVersionMetadataPayload": ProviderVersionMetadataContract,
|
||||
"ProviderVersionsPayload": ProviderVersionsContract,
|
||||
}
|
||||
|
||||
|
||||
def _collect_openapi_component_schemas(
|
||||
root_models: dict[str, type[BaseModel]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
components: dict[str, dict[str, Any]] = {}
|
||||
for schema_name, model_type in root_models.items():
|
||||
schema = model_type.model_json_schema(ref_template="#/components/schemas/{model}")
|
||||
defs = schema.pop("$defs", {})
|
||||
schema = _strip_schema_meta(schema)
|
||||
if schema_name in components and components[schema_name] != schema:
|
||||
raise ValueError(f"Conflicting OpenAPI schema definition for {schema_name}")
|
||||
components[schema_name] = schema
|
||||
for def_name, def_schema in defs.items():
|
||||
cleaned = _strip_schema_meta(def_schema)
|
||||
if def_name in components and components[def_name] != cleaned:
|
||||
raise ValueError(f"Conflicting OpenAPI schema definition for {def_name}")
|
||||
components.setdefault(def_name, cleaned)
|
||||
return components
|
||||
|
||||
|
||||
def _build_openapi_get_operation(endpoint: OpenApiEndpoint) -> dict[str, Any]:
|
||||
operation: dict[str, Any] = {
|
||||
"tags": [endpoint.tag],
|
||||
"operationId": endpoint.operation_id,
|
||||
"summary": endpoint.summary,
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": endpoint.response_description,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {"$ref": f"#/components/schemas/{endpoint.response_component}"}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
if endpoint.parameters:
|
||||
operation["parameters"] = [
|
||||
{"$ref": f"#/components/parameters/{parameter_name}"} for parameter_name in endpoint.parameters
|
||||
]
|
||||
if endpoint.include_not_found:
|
||||
operation["responses"]["404"] = {"$ref": "#/components/responses/NotFound"}
|
||||
return operation
|
||||
|
||||
|
||||
def _build_openapi_paths(endpoints: tuple[OpenApiEndpoint, ...]) -> dict[str, dict[str, Any]]:
|
||||
return {endpoint.path: {"get": _build_openapi_get_operation(endpoint)} for endpoint in endpoints}
|
||||
|
||||
|
||||
def build_openapi_document() -> dict[str, Any]:
|
||||
"""Build OpenAPI 3.1 schema from shared registry contracts."""
|
||||
|
||||
component_schemas = _collect_openapi_component_schemas(_OPENAPI_COMPONENT_MODELS)
|
||||
|
||||
return {
|
||||
"openapi": "3.1.0",
|
||||
"jsonSchemaDialect": "https://spec.openapis.org/oas/3.1/dialect/base",
|
||||
"info": {
|
||||
"title": "Airflow Registry API",
|
||||
"version": "1.0.0",
|
||||
"description": "JSON endpoints for Apache Airflow provider and module discovery.",
|
||||
},
|
||||
"tags": [
|
||||
{"name": "Catalog", "description": "Global registry datasets."},
|
||||
{"name": "Providers", "description": "Provider-scoped latest metadata."},
|
||||
{"name": "Provider Versions", "description": "Version-specific provider metadata."},
|
||||
],
|
||||
"paths": _build_openapi_paths(OPENAPI_ENDPOINTS),
|
||||
"components": {
|
||||
"parameters": {
|
||||
"ProviderId": {
|
||||
"name": "providerId",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "Provider identifier (for example: amazon).",
|
||||
},
|
||||
"Version": {
|
||||
"name": "version",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "Provider version (for example: 9.22.0).",
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"NotFound": {"description": "Static endpoint file not found."},
|
||||
},
|
||||
"schemas": component_schemas,
|
||||
},
|
||||
}
|
||||
@@ -37,14 +37,58 @@ def _write_json(path: Path, data: dict) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def _provider(provider_id: str, name: str, last_updated: str) -> dict:
|
||||
return {
|
||||
"id": provider_id,
|
||||
"name": name,
|
||||
"package_name": f"apache-airflow-providers-{provider_id}",
|
||||
"description": f"{name} provider",
|
||||
"lifecycle": "production",
|
||||
"logo": None,
|
||||
"version": "1.0.0",
|
||||
"versions": ["1.0.0"],
|
||||
"airflow_versions": ["3.0+"],
|
||||
"pypi_downloads": {"weekly": 0, "monthly": 0, "total": 0},
|
||||
"module_counts": {"operator": 1},
|
||||
"categories": [],
|
||||
"connection_types": [],
|
||||
"requires_python": ">=3.10",
|
||||
"dependencies": [],
|
||||
"optional_extras": {},
|
||||
"dependents": [],
|
||||
"related_providers": [],
|
||||
"docs_url": "https://example.invalid/docs",
|
||||
"source_url": "https://example.invalid/source",
|
||||
"pypi_url": "https://example.invalid/pypi",
|
||||
"first_released": "",
|
||||
"last_updated": last_updated,
|
||||
}
|
||||
|
||||
|
||||
def _module(module_id: str, provider_id: str) -> dict:
|
||||
return {
|
||||
"id": module_id,
|
||||
"name": "ExampleOperator",
|
||||
"type": "operator",
|
||||
"import_path": f"airflow.providers.{provider_id}.operators.example.ExampleOperator",
|
||||
"module_path": f"airflow.providers.{provider_id}.operators.example",
|
||||
"short_description": "Example operator",
|
||||
"docs_url": "https://example.invalid/docs",
|
||||
"source_url": "https://example.invalid/source",
|
||||
"category": "test",
|
||||
"provider_id": provider_id,
|
||||
"provider_name": provider_id.capitalize(),
|
||||
}
|
||||
|
||||
|
||||
class TestMerge:
|
||||
def test_replaces_existing_provider(self, tmp_path, output_dir):
|
||||
existing_providers = _write_json(
|
||||
tmp_path / "existing_providers.json",
|
||||
{
|
||||
"providers": [
|
||||
{"id": "amazon", "name": "Amazon", "last_updated": "2024-01-01"},
|
||||
{"id": "google", "name": "Google", "last_updated": "2024-02-01"},
|
||||
_provider("amazon", "Amazon", "2024-01-01"),
|
||||
_provider("google", "Google", "2024-02-01"),
|
||||
]
|
||||
},
|
||||
)
|
||||
@@ -52,18 +96,18 @@ class TestMerge:
|
||||
tmp_path / "existing_modules.json",
|
||||
{
|
||||
"modules": [
|
||||
{"id": "amazon-s3-op", "provider_id": "amazon"},
|
||||
{"id": "google-bq-op", "provider_id": "google"},
|
||||
_module("amazon-s3-op", "amazon"),
|
||||
_module("google-bq-op", "google"),
|
||||
]
|
||||
},
|
||||
)
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": "2025-01-01"}]},
|
||||
{"providers": [_provider("amazon", "Amazon", "2025-01-01")]},
|
||||
)
|
||||
new_modules = _write_json(
|
||||
tmp_path / "new_modules.json",
|
||||
{"modules": [{"id": "amazon-s3-op-v2", "provider_id": "amazon"}]},
|
||||
{"modules": [_module("amazon-s3-op-v2", "amazon")]},
|
||||
)
|
||||
|
||||
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
|
||||
@@ -88,19 +132,19 @@ class TestMerge:
|
||||
def test_adds_new_provider(self, tmp_path, output_dir):
|
||||
existing_providers = _write_json(
|
||||
tmp_path / "existing_providers.json",
|
||||
{"providers": [{"id": "google", "name": "Google", "last_updated": "2024-02-01"}]},
|
||||
{"providers": [_provider("google", "Google", "2024-02-01")]},
|
||||
)
|
||||
existing_modules = _write_json(
|
||||
tmp_path / "existing_modules.json",
|
||||
{"modules": [{"id": "google-bq-op", "provider_id": "google"}]},
|
||||
{"modules": [_module("google-bq-op", "google")]},
|
||||
)
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": "2025-01-01"}]},
|
||||
{"providers": [_provider("amazon", "Amazon", "2025-01-01")]},
|
||||
)
|
||||
new_modules = _write_json(
|
||||
tmp_path / "new_modules.json",
|
||||
{"modules": [{"id": "amazon-s3-op", "provider_id": "amazon"}]},
|
||||
{"modules": [_module("amazon-s3-op", "amazon")]},
|
||||
)
|
||||
|
||||
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
|
||||
@@ -112,12 +156,12 @@ class TestMerge:
|
||||
def test_providers_sorted_by_name(self, tmp_path, output_dir):
|
||||
existing_providers = _write_json(
|
||||
tmp_path / "existing_providers.json",
|
||||
{"providers": [{"id": "zzz", "name": "Zzz Provider", "last_updated": ""}]},
|
||||
{"providers": [_provider("zzz", "Zzz Provider", "")]},
|
||||
)
|
||||
existing_modules = _write_json(tmp_path / "existing_modules.json", {"modules": []})
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "aaa", "name": "Aaa Provider", "last_updated": ""}]},
|
||||
{"providers": [_provider("aaa", "Aaa Provider", "")]},
|
||||
)
|
||||
new_modules = _write_json(tmp_path / "new_modules.json", {"modules": []})
|
||||
|
||||
@@ -132,21 +176,21 @@ class TestMerge:
|
||||
tmp_path / "existing_providers.json",
|
||||
{
|
||||
"providers": [
|
||||
{"id": "old", "name": "Old Provider", "last_updated": "2020-01-01"},
|
||||
_provider("old", "Old Provider", "2020-01-01"),
|
||||
]
|
||||
},
|
||||
)
|
||||
existing_modules = _write_json(
|
||||
tmp_path / "existing_modules.json",
|
||||
{"modules": [{"id": "old-mod", "provider_id": "old"}]},
|
||||
{"modules": [_module("old-mod", "old")]},
|
||||
)
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "new", "name": "New Provider", "last_updated": "2025-06-01"}]},
|
||||
{"providers": [_provider("new", "New Provider", "2025-06-01")]},
|
||||
)
|
||||
new_modules = _write_json(
|
||||
tmp_path / "new_modules.json",
|
||||
{"modules": [{"id": "new-mod", "provider_id": "new"}]},
|
||||
{"modules": [_module("new-mod", "new")]},
|
||||
)
|
||||
|
||||
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
|
||||
@@ -159,17 +203,17 @@ class TestMerge:
|
||||
def test_missing_existing_modules_file(self, tmp_path, output_dir):
|
||||
existing_providers = _write_json(
|
||||
tmp_path / "existing_providers.json",
|
||||
{"providers": [{"id": "google", "name": "Google", "last_updated": ""}]},
|
||||
{"providers": [_provider("google", "Google", "")]},
|
||||
)
|
||||
# existing_modules file does not exist
|
||||
existing_modules = tmp_path / "nonexistent_modules.json"
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": ""}]},
|
||||
{"providers": [_provider("amazon", "Amazon", "")]},
|
||||
)
|
||||
new_modules = _write_json(
|
||||
tmp_path / "new_modules.json",
|
||||
{"modules": [{"id": "amazon-s3-op", "provider_id": "amazon"}]},
|
||||
{"modules": [_module("amazon-s3-op", "amazon")]},
|
||||
)
|
||||
|
||||
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
|
||||
@@ -184,11 +228,11 @@ class TestMerge:
|
||||
existing_modules = _write_json(tmp_path / "existing_modules.json", {"modules": []})
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "amazon", "name": "Amazon", "last_updated": ""}]},
|
||||
{"providers": [_provider("amazon", "Amazon", "")]},
|
||||
)
|
||||
new_modules = _write_json(
|
||||
tmp_path / "new_modules.json",
|
||||
{"modules": [{"id": "amazon-s3-op", "provider_id": "amazon"}]},
|
||||
{"modules": [_module("amazon-s3-op", "amazon")]},
|
||||
)
|
||||
|
||||
merge(existing_providers, existing_modules, new_providers, new_modules, output_dir)
|
||||
@@ -206,7 +250,7 @@ class TestMerge:
|
||||
existing_modules = _write_json(tmp_path / "existing_modules.json", {"modules": []})
|
||||
new_providers = _write_json(
|
||||
tmp_path / "new_providers.json",
|
||||
{"providers": [{"id": "test", "name": "Test", "last_updated": ""}]},
|
||||
{"providers": [_provider("test", "Test", "")]},
|
||||
)
|
||||
new_modules = _write_json(tmp_path / "new_modules.json", {"modules": []})
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Unit tests for dev/registry/registry_contract_models.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from registry_contract_models import (
|
||||
build_openapi_document,
|
||||
validate_provider_parameters,
|
||||
validate_provider_version_metadata,
|
||||
validate_provider_versions,
|
||||
)
|
||||
|
||||
|
||||
def test_build_openapi_document_produces_valid_spec():
|
||||
doc = build_openapi_document()
|
||||
assert doc["openapi"] == "3.1.0"
|
||||
assert "/api/providers.json" in doc["paths"]
|
||||
assert "/api/providers/{providerId}/versions.json" in doc["paths"]
|
||||
# Every endpoint $ref should resolve to a component schema
|
||||
for path_item in doc["paths"].values():
|
||||
ref = path_item["get"]["responses"]["200"]["content"]["application/json"]["schema"]["$ref"]
|
||||
schema_name = ref.split("/")[-1]
|
||||
assert schema_name in doc["components"]["schemas"], f"{ref} not in component schemas"
|
||||
|
||||
|
||||
def test_validate_provider_versions_requires_latest_in_versions():
|
||||
with pytest.raises(ValidationError):
|
||||
validate_provider_versions({"latest": "2.0.0", "versions": ["1.0.0"]})
|
||||
|
||||
|
||||
def test_validate_provider_parameters_preserves_mro_alias():
|
||||
payload = {
|
||||
"provider_id": "test",
|
||||
"provider_name": "Test",
|
||||
"version": "1.0.0",
|
||||
"generated_at": "2026-02-18T00:00:00+00:00",
|
||||
"classes": {
|
||||
"airflow.providers.test.mod.Example": {
|
||||
"name": "Example",
|
||||
"type": "operator",
|
||||
"mro": ["airflow.providers.test.mod.Example"],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "str",
|
||||
"default": None,
|
||||
"required": True,
|
||||
"origin": "airflow.providers.test.mod.Example",
|
||||
"description": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
}
|
||||
validated = validate_provider_parameters(payload)
|
||||
class_entry = validated["classes"]["airflow.providers.test.mod.Example"]
|
||||
assert "mro" in class_entry
|
||||
assert "mro_chain" not in class_entry
|
||||
|
||||
|
||||
def test_validate_version_metadata_accepts_legacy_version_modules_without_ids():
|
||||
payload = {
|
||||
"provider_id": "test",
|
||||
"version": "0.9.0",
|
||||
"generated_at": "2026-02-18T00:00:00+00:00",
|
||||
"requires_python": ">=3.10",
|
||||
"dependencies": [],
|
||||
"optional_extras": {},
|
||||
"connection_types": [{"conn_type": "test", "hook_class": "x.y.Hook"}],
|
||||
"module_counts": {"operator": 1},
|
||||
"modules": [
|
||||
{
|
||||
"name": "LegacyOperator",
|
||||
"type": "operator",
|
||||
"import_path": "airflow.providers.test.operators.legacy.LegacyOperator",
|
||||
"short_description": "Legacy module shape from older backfills.",
|
||||
"docs_url": "https://example.invalid/docs",
|
||||
"source_url": "https://example.invalid/source",
|
||||
"category": "test",
|
||||
}
|
||||
],
|
||||
}
|
||||
validated = validate_provider_version_metadata(payload)
|
||||
assert "id" not in validated["modules"][0]
|
||||
assert "provider_id" not in validated["modules"][0]
|
||||
assert "provider_name" not in validated["modules"][0]
|
||||
@@ -26,6 +26,14 @@ module.exports = function(eleventyConfig) {
|
||||
// Copy public directory contents to root of output
|
||||
eleventyConfig.addPassthroughCopy({ "public": "/" });
|
||||
|
||||
// Vendor swagger-ui from node_modules (external CDN blocked by Apache Infra CSP)
|
||||
eleventyConfig.addPassthroughCopy({
|
||||
"node_modules/swagger-ui-dist/swagger-ui.css": "vendor/swagger-ui/swagger-ui.css",
|
||||
});
|
||||
eleventyConfig.addPassthroughCopy({
|
||||
"node_modules/swagger-ui-dist/swagger-ui-bundle.js": "vendor/swagger-ui/swagger-ui-bundle.js",
|
||||
});
|
||||
|
||||
// Watch CSS and JS for changes
|
||||
eleventyConfig.addWatchTarget("src/css/");
|
||||
eleventyConfig.addWatchTarget("src/js/");
|
||||
|
||||
@@ -6,6 +6,9 @@ node_modules/
|
||||
_site/
|
||||
dist/
|
||||
|
||||
# Generated OpenAPI spec (produced by dev/registry/export_registry_schemas.py at build time)
|
||||
schemas/
|
||||
|
||||
# Provider logos (copied from providers/*/docs/integration-logos/ at build time)
|
||||
public/logos/
|
||||
|
||||
|
||||
+5
-3
@@ -10,9 +10,9 @@ This document contains rules, patterns, and guidelines for working on the Airflo
|
||||
### 0. Minimal JavaScript
|
||||
|
||||
- Keep JavaScript to minimum necessary
|
||||
- Theme toggle only
|
||||
- Theme toggle + progressive enhancement (filters, search)
|
||||
- No frameworks (no React, Vue, etc.)
|
||||
- Progressive enhancement approach
|
||||
- Exception: swagger-ui is vendored from `node_modules/swagger-ui-dist` for the API Explorer
|
||||
|
||||
## 1. Quality Standards
|
||||
|
||||
@@ -386,7 +386,9 @@ Key files in the project:
|
||||
|
||||
- `src/css/main.css` - Main styles
|
||||
- `src/css/tokens.css` - Design tokens
|
||||
- `src/` - All page templates (index, provider-detail, providers, explore, stats)
|
||||
- `src/` - All page templates (index, provider-detail, providers, explore, stats, api-explorer)
|
||||
- `../dev/registry/registry_contract_models.py` - Pydantic contracts for all JSON payloads
|
||||
- `../dev/registry/export_registry_schemas.py` - Generates OpenAPI spec from contracts
|
||||
|
||||
## Development Server
|
||||
|
||||
|
||||
@@ -125,6 +125,15 @@ extraction:
|
||||
types, defaults, and docstrings. Writes
|
||||
`versions/{provider_id}/{version}/parameters.json`.
|
||||
|
||||
**`registry_contract_models.py`** defines Pydantic models that validate the shape of
|
||||
every JSON payload the registry produces. Each extraction script calls
|
||||
`_validate(ModelType, payload)` before writing JSON — this catches schema drift at
|
||||
generation time without a separate jsonschema layer. The same models generate the
|
||||
OpenAPI 3.1 spec served at `/api/openapi.json`.
|
||||
|
||||
**`export_registry_schemas.py`** generates `registry/schemas/openapi.json` from the
|
||||
contract models. It runs automatically via `pnpm prebuild` before Eleventy builds.
|
||||
|
||||
**`extract_connections.py`** (runs inside breeze) reads `connection-types` from
|
||||
`provider.yaml`, falling back to runtime inspection of hook
|
||||
`get_connection_form_widgets()` and `get_ui_field_behaviour()`. Writes
|
||||
@@ -177,6 +186,8 @@ the same Sphinx build that generates the docs.
|
||||
| `statsData.js` | Checked-in | Computed statistics (lifecycle counts, top providers, etc.) |
|
||||
| `providerVersions.js` | Checked-in | Builds the provider × version page collection |
|
||||
| `latestVersionData.js` | Checked-in | Latest version parameters/connections lookup |
|
||||
| `openapiSpec.js` | Checked-in | Builds OpenAPI 3.1 spec from Pydantic contract models |
|
||||
| `providerVersionPayloads.js` | Checked-in | Generates `/api/providers/{id}/versions.json` payloads |
|
||||
|
||||
### Pages
|
||||
|
||||
@@ -188,6 +199,7 @@ the same Sphinx build that generates the docs.
|
||||
| Statistics | `src/stats.njk` | `/stats/` |
|
||||
| Provider Detail | `src/provider-detail.njk` | `/providers/{id}/` (redirects to latest) |
|
||||
| Provider Version | `src/provider-version.njk` | `/providers/{id}/{version}/` |
|
||||
| API Explorer | `src/api-explorer.njk` | `/api-explorer/` |
|
||||
|
||||
### Client-Side JavaScript
|
||||
|
||||
@@ -272,6 +284,10 @@ providing programmatic access to provider and module data:
|
||||
- `/api/providers/{id}/{version}/modules.json` — Version-specific modules
|
||||
- `/api/providers/{id}/{version}/parameters.json` — Version-specific parameters
|
||||
- `/api/providers/{id}/{version}/connections.json` — Version-specific connections
|
||||
- `/api/openapi.json` — OpenAPI 3.1 spec (generated at build time from Pydantic contracts)
|
||||
|
||||
An interactive **API Explorer** at `/api-explorer/` renders the OpenAPI spec using
|
||||
swagger-ui (vendored from `node_modules/swagger-ui-dist`).
|
||||
|
||||
## Incremental Builds
|
||||
|
||||
|
||||
@@ -4,11 +4,13 @@
|
||||
"description": "Apache Airflow Provider Registry",
|
||||
"scripts": {
|
||||
"dev": "REGISTRY_PATH_PREFIX=/ pnpm build && REGISTRY_PATH_PREFIX=/ eleventy --serve --port=8080",
|
||||
"build": "eleventy",
|
||||
"prebuild": "uv run python ../dev/registry/export_registry_schemas.py",
|
||||
"build": "rm -rf _site && eleventy",
|
||||
"postbuild": "cleancss -o _site/css/main.css _site/css/main.css && node scripts/build-pagefind-index.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@11ty/eleventy": "^3.1.2"
|
||||
"@11ty/eleventy": "^3.1.2",
|
||||
"swagger-ui-dist": "^5.32.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"clean-css-cli": "^5.6.3",
|
||||
|
||||
Generated
+15
@@ -17,6 +17,9 @@ importers:
|
||||
'@11ty/eleventy':
|
||||
specifier: ^3.1.2
|
||||
version: 3.1.2
|
||||
swagger-ui-dist:
|
||||
specifier: ^5.32.0
|
||||
version: 5.32.0
|
||||
devDependencies:
|
||||
clean-css-cli:
|
||||
specifier: ^5.6.3
|
||||
@@ -93,6 +96,9 @@ packages:
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
'@scarf/scarf@1.4.0':
|
||||
resolution: {integrity: sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ==}
|
||||
|
||||
'@sindresorhus/slugify@2.2.1':
|
||||
resolution: {integrity: sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==}
|
||||
engines: {node: '>=12'}
|
||||
@@ -593,6 +599,9 @@ packages:
|
||||
resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
swagger-ui-dist@5.32.0:
|
||||
resolution: {integrity: sha512-nKZB0OuDvacB0s/lC2gbge+RigYvGRGpLLMWMFxaTUwfM+CfndVk9Th2IaTinqXiz6Mn26GK2zriCpv6/+5m3Q==}
|
||||
|
||||
tinyglobby@0.2.15:
|
||||
resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
@@ -747,6 +756,8 @@ snapshots:
|
||||
'@pagefind/windows-x64@1.4.0':
|
||||
optional: true
|
||||
|
||||
'@scarf/scarf@1.4.0': {}
|
||||
|
||||
'@sindresorhus/slugify@2.2.1':
|
||||
dependencies:
|
||||
'@sindresorhus/transliterate': 1.6.0
|
||||
@@ -1191,6 +1202,10 @@ snapshots:
|
||||
|
||||
strip-bom-string@1.0.0: {}
|
||||
|
||||
swagger-ui-dist@5.32.0:
|
||||
dependencies:
|
||||
'@scarf/scarf': 1.4.0
|
||||
|
||||
tinyglobby@0.2.15:
|
||||
dependencies:
|
||||
fdir: 6.5.0(picomatch@4.0.3)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/*!
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const OPENAPI_SCHEMA_PATH = path.join(__dirname, "..", "..", "schemas", "openapi.json");
|
||||
|
||||
module.exports = function() {
|
||||
if (!fs.existsSync(OPENAPI_SCHEMA_PATH)) {
|
||||
throw new Error(
|
||||
`Missing OpenAPI schema artifact at ${OPENAPI_SCHEMA_PATH}. ` +
|
||||
"Run `pnpm build` (prebuild generates it) or `uv run python ../dev/registry/export_registry_schemas.py`.",
|
||||
);
|
||||
}
|
||||
return JSON.parse(fs.readFileSync(OPENAPI_SCHEMA_PATH, "utf8"));
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
/*!
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
const providersData = require("./providers.json");
|
||||
const buildProviderVersions = require("./providerVersions");
|
||||
|
||||
module.exports = function () {
|
||||
const providerVersions = buildProviderVersions();
|
||||
const payloads = {};
|
||||
|
||||
for (const provider of providersData.providers) {
|
||||
payloads[provider.id] = {
|
||||
latest: provider.version,
|
||||
versions: [provider.version],
|
||||
};
|
||||
}
|
||||
|
||||
for (const pv of providerVersions) {
|
||||
if (!pv || !pv.provider || !pv.provider.id || !pv.isLatest) {
|
||||
continue;
|
||||
}
|
||||
const versions = Array.isArray(pv.availableVersions) && pv.availableVersions.length > 0
|
||||
? pv.availableVersions
|
||||
: [pv.provider.version];
|
||||
payloads[pv.provider.id] = {
|
||||
latest: pv.provider.version,
|
||||
versions,
|
||||
};
|
||||
}
|
||||
|
||||
return payloads;
|
||||
};
|
||||
@@ -63,6 +63,7 @@
|
||||
<a href="{{ '/providers/' | url }}">Providers</a>
|
||||
<a href="{{ '/stats/' | url }}">Stats</a>
|
||||
<a href="{{ '/explore/' | url }}">Explore</a>
|
||||
<a href="{{ '/api-explorer/' | url }}">API Explorer</a>
|
||||
<a href="https://airflow.apache.org/docs/" target="_blank" rel="noopener">
|
||||
Docs
|
||||
<svg width="12" height="12" fill="none" stroke="currentColor" viewBox="0 0 24 24" aria-hidden="true" style="display:inline;vertical-align:middle;margin-left:2px">
|
||||
@@ -98,6 +99,7 @@
|
||||
<a href="{{ '/providers/' | url }}">Providers</a>
|
||||
<a href="{{ '/stats/' | url }}">Stats</a>
|
||||
<a href="{{ '/explore/' | url }}">Explore</a>
|
||||
<a href="{{ '/api-explorer/' | url }}">API Explorer</a>
|
||||
<a href="https://airflow.apache.org/docs/" target="_blank" rel="noopener">Docs</a>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
layout: base.njk
|
||||
title: API Explorer
|
||||
description: Interactive explorer for the Airflow Registry JSON API.
|
||||
permalink: "/api-explorer/"
|
||||
mainClass: "api-explorer-page"
|
||||
---
|
||||
<link rel="stylesheet" href="{{ '/vendor/swagger-ui/swagger-ui.css' | url }}">
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="{{ '/vendor/swagger-ui/swagger-ui-bundle.js' | url }}"></script>
|
||||
<script>
|
||||
(function () {
|
||||
const openApiUrl = new URL("../api/openapi.json", window.location.href).toString();
|
||||
const serverPath = new URL("../", window.location.href).pathname.replace(/\/$/, "") || "/";
|
||||
|
||||
async function renderSwaggerUi() {
|
||||
try {
|
||||
const response = await fetch(openApiUrl, { credentials: "same-origin" });
|
||||
if (!response.ok) {
|
||||
throw new Error(`OpenAPI load failed: ${response.status}`);
|
||||
}
|
||||
const spec = await response.json();
|
||||
spec.servers = [{ url: serverPath, description: "Registry base path" }];
|
||||
SwaggerUIBundle({
|
||||
spec,
|
||||
dom_id: "#swagger-ui",
|
||||
deepLinking: true,
|
||||
displayRequestDuration: true,
|
||||
docExpansion: "list",
|
||||
defaultModelsExpandDepth: -1,
|
||||
tryItOutEnabled: true,
|
||||
supportedSubmitMethods: ["get"],
|
||||
presets: [SwaggerUIBundle.presets.apis],
|
||||
});
|
||||
} catch (error) {
|
||||
document.getElementById("swagger-ui").innerHTML =
|
||||
"<p style=\"padding: 1rem; font-family: sans-serif; color: #444;\">" +
|
||||
"Could not load interactive console. " +
|
||||
"<a href=\"" + openApiUrl + "\" target=\"_blank\" rel=\"noopener\">Open raw OpenAPI document</a>." +
|
||||
"</p>";
|
||||
}
|
||||
}
|
||||
|
||||
renderSwaggerUi();
|
||||
})();
|
||||
</script>
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
permalink: "/api/openapi.json"
|
||||
eleventyExcludeFromCollections: true
|
||||
---
|
||||
{{ openapiSpec | dump | safe }}
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
pagination:
|
||||
data: providers.providers
|
||||
size: 1
|
||||
alias: provider
|
||||
permalink: "/api/providers/{{ provider.id }}/versions.json"
|
||||
eleventyExcludeFromCollections: true
|
||||
---
|
||||
{%- if providerVersionPayloads[provider.id] -%}
|
||||
{{ providerVersionPayloads[provider.id] | dump | safe }}
|
||||
{%- else -%}
|
||||
{{ { "latest": provider.version, "versions": [provider.version] } | dump | safe }}
|
||||
{%- endif -%}
|
||||
@@ -4222,3 +4222,9 @@ main {
|
||||
.error-links a:hover {
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
/* API Explorer (swagger-ui inside site layout) */
|
||||
.api-explorer-page .swagger-ui .topbar,
|
||||
.api-explorer-page .swagger-ui .scheme-container { display: none; }
|
||||
.api-explorer-page .swagger-ui .info { margin: var(--space-4) 0; }
|
||||
.api-explorer-page { padding-bottom: var(--space-8); }
|
||||
@@ -29,15 +29,16 @@
|
||||
return html.style.colorScheme || 'dark';
|
||||
}
|
||||
|
||||
// Update icon visibility based on current theme
|
||||
function updateIcons() {
|
||||
// Update icon visibility and swagger-ui dark mode class
|
||||
function updateTheme() {
|
||||
const isLight = getCurrentTheme() === 'light';
|
||||
if (sunIcon) sunIcon.style.display = isLight ? 'block' : 'none';
|
||||
if (moonIcon) moonIcon.style.display = isLight ? 'none' : 'block';
|
||||
html.classList.toggle('dark-mode', !isLight);
|
||||
}
|
||||
|
||||
// Initialize icons on load
|
||||
updateIcons();
|
||||
// Initialize on load
|
||||
updateTheme();
|
||||
|
||||
// Toggle theme
|
||||
if (themeToggle) {
|
||||
@@ -47,7 +48,7 @@
|
||||
|
||||
html.style.colorScheme = newTheme;
|
||||
localStorage.setItem('theme', newTheme);
|
||||
updateIcons();
|
||||
updateTheme();
|
||||
});
|
||||
}
|
||||
})();
|
||||
@@ -8,6 +8,7 @@ eleventyExcludeFromCollections: true
|
||||
<url><loc>https://airflow.apache.org/registry/providers/</loc></url>
|
||||
<url><loc>https://airflow.apache.org/registry/explore/</loc></url>
|
||||
<url><loc>https://airflow.apache.org/registry/stats/</loc></url>
|
||||
<url><loc>https://airflow.apache.org/registry/api-explorer/</loc></url>
|
||||
{%- for provider in providers.providers %}
|
||||
<url><loc>https://airflow.apache.org/registry/providers/{{ provider.id }}/{{ provider.version }}/</loc></url>
|
||||
{%- endfor %}
|
||||
|
||||
Reference in new issue
Block a user