Skip to content

Commit 6c97ab1

Browse files
leahecoleleahecole
authored
refactor: package composer testing module (GoogleCloudPlatform#5309)
* add testing utilities * make setup.py consistent with requriements-test * try moving package to top level * remove egg info from commit * add py_modules field to setup.py * remove __init__.py from dagtestutils * rename unit_testing * modify setup.py * bring back unit_testing name * move under composer directory * update subdirectory in requirements and setup * reneame to dag_test_utils * add readme, fix lint * rename module (tests should fail) * complete rename to internal_unit_testing * fix lint oops * remove branch reference
1 parent f25ff5f commit 6c97ab1

19 files changed

+92
-35
lines changed

‎composer/dag_test_utils/README.md‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Cloud Composer DAG Testing Utility
2+
3+
This package is used internally to unit test the validity of all Cloud Composer sample DAGs. It is not supported for external production use.
4+
5+
## Instructions
6+
7+
Add the following to your `requirements-test.txt` file:
8+
9+
`git+https://github.com/GoogleCloudPlatform/python-docs-samples.git#egg=dag_test_utils&subdirectory=composer/dag_test_utils`
10+
11+
Import the internal unit testing module
12+
13+
```python
14+
import internal_unit_testing
15+
```
16+
17+
Test your DAG
18+
19+
```python
20+
def test_dag_import():
21+
# Set any variables if your DAG requires them
22+
models.Variable.set('gcs_bucket', 'example_bucket')
23+
from . import my_dag as module
24+
# Check for DAG validity
25+
internal_unit_testing.assert_has_valid_dag(module)
26+
```
27+
28+
For more examples, refer to the [`workflows`](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/master/composer/workflows) directory.
29+
30+

composer/workflows/unit_testing.py renamed to composer/dag_test_utils/internal_unit_testing.py

File renamed without changes.

‎composer/dag_test_utils/setup.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from setuptools import find_packages
16+
from setuptools import setup
17+
18+
setup(
19+
name="dag_test_utils",
20+
version="0.0.1",
21+
url="[email protected]:GoogleCloudPlatform/python-docs-samples.git#egg=dag_test_utils&subdirectory=composer/dag_test_utils",
22+
author="Google Cloud Platform",
23+
description="Utility used to unit test example Apache Airflow DAGs",
24+
packages=find_packages(),
25+
py_modules=['internal_unit_testing'],
26+
install_requires=['apache-airflow[gcp]']
27+
)

‎composer/workflows/airflow_db_cleanup_test.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
from . import unit_testing
15+
16+
import internal_unit_testing
1617

1718

1819
def test_dag_import():
@@ -23,4 +24,4 @@ def test_dag_import():
2324
docs: https://airflow.incubator.apache.org/tutorial.html#testing
2425
"""
2526
from . import airflow_db_cleanup as module
26-
unit_testing.assert_has_valid_dag(module)
27+
internal_unit_testing.assert_has_valid_dag(module)

‎composer/workflows/bashoperator_python2_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
from . import unit_testing
15+
import internal_unit_testing
1616

1717

1818
def test_dag_import():
@@ -23,4 +23,4 @@ def test_dag_import():
2323
docs: https://airflow.incubator.apache.org/tutorial.html#testing
2424
"""
2525
from . import bashoperator_python2 as module
26-
unit_testing.assert_has_valid_dag(module)
26+
internal_unit_testing.assert_has_valid_dag(module)

‎composer/workflows/bq_copy_across_locations_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from airflow import models
1919

20-
from . import unit_testing
20+
import internal_unit_testing
2121

2222

2323
def test_dag():
@@ -34,4 +34,4 @@ def test_dag():
3434
models.Variable.set('gcs_source_bucket', 'example-project')
3535
models.Variable.set('gcs_dest_bucket', 'us-central1-f')
3636
from . import bq_copy_across_locations as module
37-
unit_testing.assert_has_valid_dag(module)
37+
internal_unit_testing.assert_has_valid_dag(module)

‎composer/workflows/bq_notify_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
from airflow import models
1616

17-
from . import unit_testing
17+
import internal_unit_testing
1818

1919

2020
def test_dag_import():
@@ -29,4 +29,4 @@ def test_dag_import():
2929
models.Variable.set('gce_zone', 'us-central1-f')
3030
models.Variable.set('email', '[email protected]')
3131
from . import bq_notify as module
32-
unit_testing.assert_has_valid_dag(module)
32+
internal_unit_testing.assert_has_valid_dag(module)

‎composer/workflows/connections_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
from . import unit_testing
15+
import internal_unit_testing
1616

1717

1818
def test_dag_import():
@@ -23,4 +23,4 @@ def test_dag_import():
2323
docs: https://airflow.incubator.apache.org/tutorial.html#testing
2424
"""
2525
from . import connections as module
26-
unit_testing.assert_has_valid_dag(module)
26+
internal_unit_testing.assert_has_valid_dag(module)

‎composer/workflows/dataflowtemplateoperator_tutorial_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
from airflow import models
1616

17-
from . import unit_testing
17+
import internal_unit_testing
1818

1919

2020
def test_dag_import():
@@ -30,4 +30,4 @@ def test_dag_import():
3030
models.Variable.set("gce_region", "us-central1-f")
3131
from . import dataflowtemplateoperator_tutorial as module
3232

33-
unit_testing.assert_has_valid_dag(module)
33+
internal_unit_testing.assert_has_valid_dag(module)

‎composer/workflows/dataproc_workflow_template_instantiate_operator_tutorial_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
from airflow import models
1616

17-
from . import unit_testing
17+
import internal_unit_testing
1818

1919

2020
def test_dag_import():
@@ -27,4 +27,4 @@ def test_dag_import():
2727
models.Variable.set("project_id", "example-project")
2828
from . import dataproc_workflow_template_instantiate_operator_tutorial as module
2929

30-
unit_testing.assert_has_valid_dag(module)
30+
internal_unit_testing.assert_has_valid_dag(module)

0 commit comments

Comments
 (0)