Visitar URL original
Change `ds`, `ts`, etc. back to use logical date (#19088) · apache/airflow@25a50bb · GitHub
Skip to content

Commit 25a50bb

Browse files
authored
Change ds, ts, etc. back to use logical date (#19088)
1 parent 8d703ae commit 25a50bb

3 files changed

Lines changed: 91 additions & 26 deletions

File tree

‎airflow/models/taskinstance.py‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
from datetime import datetime, timedelta
2828
from functools import partial
2929
from tempfile import NamedTemporaryFile
30-
from typing import IO, TYPE_CHECKING, Any, Dict, Iterable, List, NamedTuple, Optional, Tuple, Union
30+
from typing import IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple, Union
3131
from urllib.parse import quote
3232

3333
import dill
@@ -1786,17 +1786,16 @@ def get_template_context(self, session: Session = None, ignore_param_exceptions:
17861786
if conf.getboolean('core', 'dag_run_conf_overrides_params'):
17871787
self.overwrite_params_with_dag_run_conf(params=params, dag_run=dag_run)
17881788

1789-
interval_start = dag.get_run_data_interval(dag_run).start
1790-
ds = interval_start.strftime('%Y-%m-%d')
1789+
logical_date = timezone.coerce_datetime(self.execution_date)
1790+
ds = logical_date.strftime('%Y-%m-%d')
1791+
ds_nodash = ds.replace('-', '')
1792+
ts = logical_date.isoformat()
1793+
ts_nodash = logical_date.strftime('%Y%m%dT%H%M%S')
1794+
ts_nodash_with_tz = ts.replace('-', '').replace(':', '')
17911795

17921796
# Now validates Params and convert them into a simple dict
17931797
task.params = params.validate()
17941798

1795-
ds_nodash = ds.replace('-', '')
1796-
ts = interval_start.isoformat()
1797-
ts_nodash = interval_start.strftime('%Y%m%dT%H%M%S')
1798-
ts_nodash_with_tz = ts.replace('-', '').replace(':', '')
1799-
18001799
@cache # Prevent multiple database access.
18011800
def _get_previous_dagrun_success() -> Optional["DagRun"]:
18021801
return self.get_previous_dagrun(state=State.SUCCESS, session=session)
@@ -1908,14 +1907,23 @@ def get(
19081907

19091908
# Create lazy proxies for deprecated stuff.
19101909

1911-
def deprecated_proxy(func, *, key, replacement=None) -> lazy_object_proxy.Proxy:
1910+
def deprecated_proxy(
1911+
func: Callable[[], Any],
1912+
*,
1913+
key: str,
1914+
replacements: Optional[List[str]] = None,
1915+
) -> lazy_object_proxy.Proxy:
19121916
def deprecated_func():
19131917
message = (
19141918
f"Accessing {key!r} from the template is deprecated and "
19151919
f"will be removed in a future version."
19161920
)
1917-
if replacement:
1918-
message += f" Please use {replacement!r} instead."
1921+
if replacements:
1922+
display_except_last = ", ".join(repr(r) for r in replacements[:-1])
1923+
if display_except_last:
1924+
message += f" Please use {display_except_last} or {replacements[-1]!r} instead."
1925+
else:
1926+
message += f" Please use {replacements[-1]!r} instead."
19191927
warnings.warn(message, DeprecationWarning)
19201928
return func()
19211929

@@ -1993,22 +2001,23 @@ def get_prev_ds_nodash() -> Optional[str]:
19932001
'ds': ds,
19942002
'ds_nodash': ds_nodash,
19952003
'execution_date': deprecated_proxy(
1996-
lambda: timezone.coerce_datetime(self.execution_date),
2004+
lambda: logical_date,
19972005
key='execution_date',
1998-
replacement='data_interval_start',
2006+
replacements=['logical_date', 'data_interval_start'],
19992007
),
20002008
'inlets': task.inlets,
2009+
'logical_date': logical_date,
20012010
'macros': macros,
2002-
'next_ds': deprecated_proxy(get_next_ds, key="next_ds", replacement="data_interval_end | ds"),
2011+
'next_ds': deprecated_proxy(get_next_ds, key="next_ds", replacements=["data_interval_end | ds"]),
20032012
'next_ds_nodash': deprecated_proxy(
20042013
get_next_ds_nodash,
20052014
key="next_ds_nodash",
2006-
replacement="data_interval_end | ds_nodash",
2015+
replacements=["data_interval_end | ds_nodash"],
20072016
),
20082017
'next_execution_date': deprecated_proxy(
20092018
get_next_execution_date,
20102019
key='next_execution_date',
2011-
replacement='data_interval_end',
2020+
replacements=['data_interval_end'],
20122021
),
20132022
'outlets': task.outlets,
20142023
'params': task.params,
@@ -2020,7 +2029,7 @@ def get_prev_ds_nodash() -> Optional[str]:
20202029
'prev_execution_date_success': deprecated_proxy(
20212030
lambda: self.get_previous_execution_date(state=State.SUCCESS, session=session),
20222031
key='prev_execution_date_success',
2023-
replacement='prev_data_interval_start_success',
2032+
replacements=['prev_data_interval_start_success'],
20242033
),
20252034
'prev_start_date_success': lazy_object_proxy.Proxy(get_prev_start_date_success),
20262035
'run_id': self.run_id,

‎docs/apache-airflow/templates-ref.rst‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,14 @@ Variable Description
3838
========================================== ====================================
3939
``{{ data_interval_start }}`` Start of the data interval (`pendulum.DateTime`_).
4040
``{{ data_interval_end }}`` End of the data interval (`pendulum.DateTime`_).
41-
``{{ ds }}`` Start of the data interval as ``YYYY-MM-DD``.
42-
Same as ``{{ data_interval_start | ds }}``.
43-
``{{ ds_nodash }}`` Start of the data interval as ``YYYYMMDD``.
44-
Same as ``{{ data_interval_start | ds_nodash }}``.
45-
``{{ ts }}`` Same as ``{{ data_interval_start | ts }}``.
41+
``{{ ds }}`` The DAG run's logical date as ``YYYY-MM-DD``.
42+
Same as ``{{ dag_run.logical_date | ds }}``.
43+
``{{ ds_nodash }}`` Same as ``{{ dag_run.logical_date | ds_nodash }}``.
44+
``{{ ts }}`` Same as ``{{ dag_run.logical_date | ts }}``.
4645
Example: ``2018-01-01T00:00:00+00:00``.
47-
``{{ ts_nodash_with_tz }}`` Same as ``{{ data_interval_start | ts_nodash_with_tz }}``.
46+
``{{ ts_nodash_with_tz }}`` Same as ``{{ dag_run.logical_date | ts_nodash_with_tz }}``.
4847
Example: ``20180101T000000+0000``.
49-
``{{ ts_nodash }}`` Same as ``{{ data_interval_start | ts_nodash }}``.
48+
``{{ ts_nodash }}`` Same as ``{{ dag_run.logical_date | ts_nodash }}``.
5049
Example: ``20180101T000000``.
5150
``{{ prev_data_interval_start_success }}`` Start of the data interval from prior successful DAG run
5251
(`pendulum.DateTime`_ or ``None``).
@@ -78,13 +77,18 @@ Variable Description
7877
subcommand.
7978
========================================== ====================================
8079

80+
.. note::
81+
82+
The DAG run's logical date, and values derived from it, such as ``ds`` and
83+
``ts``, **should not** be considered unique in a DAG. Use ``run_id`` instead.
84+
8185
The following variables are deprecated. They are kept for backward compatibility, but you should convert
8286
existing code to use other variables instead.
8387

8488
===================================== ====================================
8589
Deprecated Variable Description
8690
===================================== ====================================
87-
``{{ execution_date }}`` the execution date (logical date), same as ``logical_date``
91+
``{{ execution_date }}`` the execution date (logical date), same as ``dag_run.logical_date``
8892
``{{ next_execution_date }}`` the next execution date (if available) (`pendulum.DateTime`_)
8993
if ``{{ execution_date }}`` is ``2018-01-01 00:00:00`` and
9094
``schedule_interval`` is ``@weekly``, ``{{ next_execution_date }}``

‎tests/utils/log/test_log_reader.py‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,21 @@
1616
# under the License.
1717

1818
import copy
19+
import datetime
1920
import logging
2021
import os
2122
import sys
2223
import tempfile
2324
from unittest import mock
2425

26+
import pendulum
2527
import pytest
2628

2729
from airflow import settings
2830
from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
31+
from airflow.models import DagRun
32+
from airflow.operators.python import PythonOperator
33+
from airflow.timetables.base import DataInterval
2934
from airflow.utils import timezone
3035
from airflow.utils.log.log_reader import TaskLogReader
3136
from airflow.utils.log.logging_mixin import ExternalLoggingMixin
@@ -87,7 +92,7 @@ def prepare_log_files(self, log_dir):
8792
f.flush()
8893

8994
@pytest.fixture(autouse=True)
90-
def prepare_db(self, session, create_task_instance):
95+
def prepare_db(self, create_task_instance):
9196
ti = create_task_instance(
9297
dag_id=self.DAG_ID,
9398
task_id=self.TASK_ID,
@@ -238,3 +243,50 @@ def test_supports_external_link(self):
238243

239244
mock_prop.return_value = True
240245
assert task_log_reader.supports_external_link
246+
247+
def test_task_log_filename_unique(self, dag_maker):
248+
"""Ensure the default log_filename_template produces a unique filename.
249+
250+
See discussion in apache/airflow#19058 [1]_ for how uniqueness may
251+
change in a future Airflow release. For now, the logical date is used
252+
to distinguish DAG runs. This test should be modified when the logical
253+
date is no longer used to ensure uniqueness.
254+
255+
[1]: https://github.com/apache/airflow/issues/19058
256+
"""
257+
dag_id = "test_task_log_filename_ts_corresponds_to_logical_date"
258+
task_id = "echo_run_type"
259+
260+
def echo_run_type(dag_run: DagRun, **kwargs):
261+
print(dag_run.run_type)
262+
263+
with dag_maker(dag_id, start_date=self.DEFAULT_DATE, schedule_interval="@daily") as dag:
264+
PythonOperator(task_id=task_id, python_callable=echo_run_type)
265+
266+
start = pendulum.datetime(2021, 1, 1)
267+
end = start + datetime.timedelta(days=1)
268+
trigger_time = end + datetime.timedelta(hours=4, minutes=29) # Arbitrary.
269+
270+
# Create two DAG runs that have the same data interval, but not the same
271+
# execution date, to check if they correctly use different log files.
272+
scheduled_dagrun: DagRun = dag_maker.create_dagrun(
273+
run_type=DagRunType.SCHEDULED,
274+
execution_date=start,
275+
data_interval=DataInterval(start, end),
276+
)
277+
manual_dagrun: DagRun = dag_maker.create_dagrun(
278+
run_type=DagRunType.MANUAL,
279+
execution_date=trigger_time,
280+
data_interval=DataInterval(start, end),
281+
)
282+
283+
scheduled_ti = scheduled_dagrun.get_task_instance(task_id)
284+
manual_ti = manual_dagrun.get_task_instance(task_id)
285+
assert scheduled_ti is not None
286+
assert manual_ti is not None
287+
288+
scheduled_ti.refresh_from_task(dag.get_task(task_id))
289+
manual_ti.refresh_from_task(dag.get_task(task_id))
290+
291+
reader = TaskLogReader()
292+
assert reader.render_log_filename(scheduled_ti, 1) != reader.render_log_filename(manual_ti, 1)

0 commit comments

Comments
 (0)