diff --git a/docs/README.md b/docs/README.md index fa6e1e00df8..9230edbe58a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,7 @@ - [Typical workflows](#typical-workflows) - [Publishing the documentation by the release manager](#publishing-the-documentation-by-the-release-manager) - [Publishing changes to the website (including theme)](#publishing-changes-to-the-website-including-theme) +- [Publishing changes manually](#publishing-changes-manually) - [Fixing historical documentation](#fixing-historical-documentation) - [Manually publishing documentation directly to S3](#manually-publishing-documentation-directly-to-s3) - [Manually publishing documentation via `apache-airflow-site-archive` repo](#manually-publishing-documentation-via-apache-airflow-site-archive-repo) @@ -78,7 +79,19 @@ There are a few repositories under `apache` organization that are used to build We have two S3 buckets where we can publish the documentation generated from the `apache-airflow` repository: * `s3://live-docs-airflow-apache-org/docs/` - live, [official documentation](https://airflow.apache.org/docs/) -* `s3://staging-docs-airflow-apache-org/docs/` - staging documentation [official documentation](https://staging-airflow.apache.org/docs/) TODO: make it work +* `s3://staging-docs-airflow-apache-org/docs/` - staging documentation [official documentation](https://staging-airflow.apache.org/docs/) + +Note that those S3 buckets are not served directly to Apache Server, but they are served via Cloudfront +in order to provide caching and automated resolution of folders into index.html files. + +The cloudfront distributions of ours are: + +* Live cloudfront url: https://d7fnmbhf26p21.cloudfront.net +* Staging cloudfront url: https://d3a2du7x0n8ydr.cloudfront.net + +Those cloudfront caches are automatically invalidated when we publish new documentation to S3 using +GitHub Actions workflows, but you can also manually invalidate them using the AWS Console if needed. + # Diagrams of the documentation architecture @@ -196,6 +209,21 @@ The version of sphinx theme is fixed in both repositories: In case of bigger changes to the theme, we can first iterate on the website and merge a new theme version, and only after that can we switch to the new version of the theme. +# Publishing changes manually + +Sometimes you do not want to use Publishing workflows to publish individual files and caches might get +into the way as both Cloudfront and Fastly caches might take some time to invalidate. In such a case, when +you manually upload the files to S3 bucket, you can immediately invalidate the caches: + +1) Manually run invalidation request in Cloudfront for the documentation S3 bucket you uploaded the files + to via AWS Console or AWS CLI (You can use `/*` to invalidate all files). + +![Cloudfront invalidation](images/cloudfront_invalidation.png) + +2) Run the `Build docs` workflow in `airflow-site` repository to invalidate Fastly cache for the website. + Use `main` branch to rebuild site for `live` site and `staging` to rebuild the `staging` site: + +![Build docs](images/build-docs.png) # Fixing historical documentation @@ -223,8 +251,8 @@ bad links or when we change some of the structure in the documentation. This can 7. In case you specify `full_sync`, you can also synchronize `all` docs or only selected documentation packages (for example `apache-airflow` or `docker-stack` or `amazon` or `helm-chart`) - you can specify more than one package separated by spaces. -8. The workflow will invalidate Cloudflare cache for "live" or "staging" bucket respectively. -9. Run the "Build docs" workflow in `airflow-site` repository to make sure that Fastly cache of the +8. The workflow will invalidate Cloudfront cache for "live" or "staging" bucket respectively. +9. Run the `Build docs` workflow in `airflow-site` repository to make sure that Fastly cache of the https://airflow.apache.org or https://airflow.staged.apache.org/ invalidated. Use `main` to rebuild site for `live` site and `staging` to rebuild the `staging` site: ![Build docs](images/build-docs.png) diff --git a/docs/images/cloudfront_invalidation.png b/docs/images/cloudfront_invalidation.png new file mode 100644 index 00000000000..55eec6fca45 Binary files /dev/null and b/docs/images/cloudfront_invalidation.png differ diff --git a/docs/images/documentation_architecture.md5sum b/docs/images/documentation_architecture.md5sum index de72fbd745d..a954bd1289e 100644 --- a/docs/images/documentation_architecture.md5sum +++ b/docs/images/documentation_architecture.md5sum @@ -1 +1 @@ -8948941e4684025792d1947d3e4aa971 +0a558da8b9adba9e2b8a596b8bd76d4c diff --git a/docs/images/documentation_architecture.png b/docs/images/documentation_architecture.png index 62e7db2d9f2..bde65505c9b 100644 Binary files a/docs/images/documentation_architecture.png and b/docs/images/documentation_architecture.png differ diff --git a/docs/images/documentation_architecture.py b/docs/images/documentation_architecture.py index 5e7df5ab16c..8ccf02f9943 100644 --- a/docs/images/documentation_architecture.py +++ b/docs/images/documentation_architecture.py @@ -19,6 +19,7 @@ from __future__ import annotations from pathlib import Path from diagrams import Cluster, Diagram, Edge +from diagrams.aws.network import CloudFront from diagrams.aws.storage import S3 from diagrams.custom import Custom from diagrams.onprem.client import User, Users @@ -31,6 +32,7 @@ MY_FILENAME = Path(__file__).with_suffix("").name console = Console(width=400, color_system="standard") GITHUB_LOGO = MY_DIR / "logos" / "github.png" +FASTLY_LOGO = MY_DIR / "logos" / "fastly.png" ASF_LOGO = MY_DIR / "logos" / "asf_logo_wide.png" graph_attr = { @@ -86,6 +88,7 @@ def generate_documentation_architecture_diagram(): live_bucket = S3("live-docs-airflow-apache-org") apache_airflow_site_archive_repo = Custom("apache-airflow-site-archive", GITHUB_LOGO.as_posix()) apache_live_webserver = Custom("https://airflow.apache.org", ASF_LOGO.as_posix()) + cloudfront_live_cache = CloudFront("CloudFront Live: https://d7fnmbhf26p21.cloudfront.net") release_manager >> Edge(color="black", style="solid", label="Publish package docs") >> live_bucket ( @@ -97,6 +100,11 @@ def generate_documentation_architecture_diagram(): ( live_bucket >> Edge(color="black", style="solid", label="Proxy docs from S3", minlen="2") + >> cloudfront_live_cache + ) + ( + cloudfront_live_cache + >> Edge(color="black", style="solid", label="Cloudfront cache", minlen="2") >> apache_live_webserver ) ( @@ -111,8 +119,10 @@ def generate_documentation_architecture_diagram(): >> live_bucket ) + fastly = Custom("Fastly CDN", FASTLY_LOGO.as_posix()) users = Users("Users") - apache_live_webserver >> Edge(color="black", style="solid", label="Served via Fastly") >> users + apache_live_webserver >> Edge(color="black", style="solid", label="Exposed to Fastly") >> fastly + fastly >> Edge(color="black", style="solid", label="Served to users") >> users console.print(f"[green]Generated architecture image {image_file}") diff --git a/docs/images/logos/fastly.png b/docs/images/logos/fastly.png new file mode 100644 index 00000000000..ebf3a75fddc Binary files /dev/null and b/docs/images/logos/fastly.png differ