mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Load the serialized Dag in ti_run only for stub tasks (#75795)
* Load the serialized Dag in ti_run only for stub tasks Starting any task deserialized its whole Dag so the route could find out whether the task was a stub and read its arg_bindings. Denormalize the serialized is_stub flag onto task_instance, which ti_run already fetches and locks, and load the Dag only when it is set. execution_timeout is resolved from the same definition, so it is now sent for a lang-SDK task only; #75387 has a Python task send its own to the supervisor. * Hold SQLite foreign keys off while the downgrade rebuilds Dropping a column rebuilds the table on SQLite, unlike adding one, so the downgrade copies task_instance rows while other tables still point at it. * Pin the bulk-insert path and reword the is_stub comments create_dagrun refreshes every row from its task right after the bulk write, so the existing test passed even when insert_mapping dropped the key. Assert the mapping itself instead, and record what a row keeps when it moves to a new Dag version without being refreshed. * Mark a Go or Java TriggerDagRun task a stub, as TypeScript does * Inline the table name in the is_stub migration ---------
This commit is contained in:
1 parent
b4320026a3
commit
c24242eea5
18 files changed
+214
-23
No files matched your search
@@ -145,15 +145,16 @@ private fun serializeTask(
|
||||
// value never matches the tuple default it is compared against), so it
|
||||
// is unconditional here too. Java tasks have no template fields.
|
||||
data["template_fields"] = emptyList<Any?>()
|
||||
// What marks a task whose arguments Airflow resolves per instance for a
|
||||
// runtime outside Python, as `@task.stub` does on the Python side.
|
||||
// `get_arg_bindings` reads nothing without it.
|
||||
data["is_stub"] = true
|
||||
argBindings(taskId, def)?.let { data["_arg_bindings"] = it }
|
||||
}
|
||||
// The Java runtime runs every task of a Java Dag, a trigger task included, so
|
||||
// every one of them is a stub. is_stub marks a task whose arguments Airflow
|
||||
// resolves per instance for a runtime outside Python, as `@task.stub` does on
|
||||
// the Python side. `get_arg_bindings` reads nothing without it.
|
||||
data["is_stub"] = true
|
||||
// Lets NotPreviouslySkippedDep re-skip a cleared downstream, as Python's SkipMixin does.
|
||||
if (def.decider != null) data["_can_skip_downstream"] = true
|
||||
// A trigger task takes the Dag's queue too, because the Java runtime runs it.
|
||||
// The worker picks a runtime by queue, so a trigger task takes the Dag's queue too.
|
||||
val config =
|
||||
if (dagQueue == null || "queue" in def.configValues) def.configValues else def.configValues + ("queue" to dagQueue)
|
||||
// Emit only config entries that differ from their schema default, mirroring
|
||||
|
||||
@@ -168,9 +168,10 @@ internal class TriggerDagRunTest {
|
||||
assertEquals("NOTSET", data["logical_date"])
|
||||
assertEquals(false, data["wait_for_completion"])
|
||||
assertEquals(false, data["skip_when_already_exists"])
|
||||
// A Java task's markers belong to a task the Java runtime runs a body for.
|
||||
// The Java runtime runs a trigger task too, so it carries the stub marker.
|
||||
assertEquals(true, data["is_stub"])
|
||||
// `language` marks a task the Java runtime runs a body for, which this is not.
|
||||
assertFalse("language" in data)
|
||||
assertFalse("is_stub" in data)
|
||||
assertFalse("reset_dag_run" in data)
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user