Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 6 additions & 20 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v4.6.0
hooks:
- id: check-added-large-files
args: ['--maxkb=100']
Expand All @@ -17,33 +17,19 @@ repos:
- id: python-no-log-warn
- id: python-use-type-annotations
- id: text-unicode-replacement-char
- repo: https://github.com/asottile/reorder-python-imports
rev: v3.12.0
hooks:
- id: reorder-python-imports
args: [--py37-plus, --add-import, 'from __future__ import annotations']
- repo: https://github.com/asottile/setup-cfg-fmt
rev: v2.5.0
hooks:
- id: setup-cfg-fmt
- repo: https://github.com/psf/black
rev: 23.9.1
hooks:
- id: black
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.0.292
rev: v0.3.7
hooks:
- id: ruff
- id: ruff-format
- repo: https://github.com/dosisod/refurb
rev: v1.21.0
rev: v2.0.0
hooks:
- id: refurb
args: [--ignore, FURB126]
- repo: https://github.com/econchick/interrogate
rev: 1.5.0
hooks:
- id: interrogate
args: [-v, --fail-under=40, src, tests]
- repo: https://github.com/executablebooks/mdformat
rev: 0.7.17
hooks:
Expand All @@ -58,7 +44,7 @@ repos:
hooks:
- id: codespell
- repo: https://github.com/pre-commit/mirrors-mypy
rev: 'v1.5.1'
rev: 'v1.9.0'
hooks:
- id: mypy
additional_dependencies: [
Expand All @@ -74,7 +60,7 @@ repos:
hooks:
- id: check-manifest
args: [--no-build-isolation]
additional_dependencies: [setuptools-scm, toml]
additional_dependencies: [setuptools-scm, wheel, toml]
- repo: meta
hooks:
- id: check-hooks-apply
Expand Down
6 changes: 2 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,7 @@ Use the `serializer` keyword arguments of the `@pytask.mark.r` decorator with

```python
@pytask.mark.r(script="script.r", serializer="yaml")
def task_example():
...
def task_example(): ...
```

And in your R script use
Expand All @@ -214,8 +213,7 @@ import json


@pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json")
def task_example():
...
def task_example(): ...
```

### Configuration
Expand Down
37 changes: 10 additions & 27 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@
requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"]
build-backend = "setuptools.build_meta"


[tool.setuptools_scm]
write_to = "src/pytask_r/_version.py"


[tool.mypy]
files = ["src", "tests"]
check_untyped_defs = true
Expand All @@ -17,56 +15,41 @@ no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true


[[tool.mypy.overrides]]
module = "tests.*"
disallow_untyped_defs = false
ignore_errors = true


[tool.ruff]
target-version = "py38"
select = ["ALL"]
fix = true
unsafe-fixes = true

[tool.ruff.lint]
extend-ignore = [
"TCH",
"TRY",
# Numpy docstyle
"D107",
"D203",
"D212",
"D213",
"D402",
"D413",
"D415",
"D416",
"D417",
# Others.
"D404", # Do not start module docstring with "This".
"RET504", # unnecessary variable assignment before return.
"S101", # raise errors for asserts.
"B905", # strict parameter for zip that was implemented in py310.
"I", # ignore isort
"ANN101", # type annotating self
"ANN102", # type annotating cls
"FBT", # flake8-boolean-trap
"EM", # flake8-errmsg
"ANN401", # flake8-annotate typing.Any
"PD", # pandas-vet
"COM812", # trailing comma missing, but black takes care of that
"COM812", # Comply with ruff-format
"ISC001", # Comply with ruff-format
]
select = ["ALL"]


[tool.ruff.per-file-ignores]
[tool.ruff.lint.per-file-ignores]
"tests/*" = ["D", "ANN"]


[tool.ruff.pydocstyle]
[tool.ruff.lint.pydocstyle]
convention = "numpy"

[tool.ruff.lint.isort]
force-single-line = true

[tool.pytest.ini_options]
# Do not add src since it messes with the loading of pytask-parallel as a plugin.
testpaths = ["tests"]
markers = [
"wip: Tests that are work-in-progress.",
Expand Down
3 changes: 2 additions & 1 deletion src/pytask_r/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""This module contains the main namespace of pytask-r."""
"""Contains the main namespace of pytask-r."""

from __future__ import annotations

try:
Expand Down
30 changes: 18 additions & 12 deletions src/pytask_r/collect.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,37 @@
"""Collect tasks."""

from __future__ import annotations

import subprocess
import warnings
from pathlib import Path
from typing import Any

from pytask import has_mark
from pytask import hookimpl
from pytask import is_task_function
from pytask import Mark
from pytask import NodeInfo
from pytask import parse_dependencies_from_task_function
from pytask import parse_products_from_task_function
from pytask import PathNode
from pytask import PTask
from pytask import PythonNode
from pytask import remove_marks
from pytask import Session
from pytask import Task
from pytask import TaskWithoutPath
from pytask_r.serialization import create_path_to_serialized
from pytask import has_mark
from pytask import hookimpl
from pytask import is_task_function
from pytask import parse_dependencies_from_task_function
from pytask import parse_products_from_task_function
from pytask import remove_marks

from pytask_r.serialization import SERIALIZERS
from pytask_r.serialization import create_path_to_serialized
from pytask_r.shared import r


def run_r_script(
_script: Path, _options: list[str], _serialized: Path, **kwargs: Any # noqa: ARG001
_script: Path,
_options: list[str],
_serialized: Path,
**kwargs: Any, # noqa: ARG001
) -> None:
"""Run an R script."""
cmd = ["Rscript", _script.as_posix(), *_options, str(_serialized)]
Expand All @@ -49,10 +54,11 @@ def pytask_collect_task(
# Parse @pytask.mark.r decorator.
obj, marks = remove_marks(obj, "r")
if len(marks) > 1:
raise ValueError(
msg = (
f"Task {name!r} has multiple @pytask.mark.r marks, but only one is "
"allowed."
)
raise ValueError(msg)

mark = _parse_r_mark(
mark=marks[0],
Expand Down Expand Up @@ -88,10 +94,11 @@ def pytask_collect_task(
)

if not (isinstance(script_node, PathNode) and script_node.path.suffix == ".r"):
raise ValueError(
msg = (
"The 'script' keyword of the @pytask.mark.r decorator must point "
f"to Julia file with the .r suffix, but it is {script_node}."
)
raise ValueError(msg)

options_node = session.hook.pytask_collect_node(
session=session,
Expand Down Expand Up @@ -181,5 +188,4 @@ def _parse_r_mark(
)
parsed_kwargs["suffix"] = suffix or proposed_suffix # type: ignore[assignment]

mark = Mark("r", (), parsed_kwargs)
return mark
return Mark("r", (), parsed_kwargs)
8 changes: 6 additions & 2 deletions src/pytask_r/config.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
"""Configure pytask."""

from __future__ import annotations

from typing import Any

from pytask import hookimpl

from pytask_r.serialization import SERIALIZERS


Expand All @@ -13,10 +15,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None:
config["markers"]["r"] = "Tasks which are executed with Rscript."
config["r_serializer"] = config.get("r_serializer", "json")
if config["r_serializer"] not in SERIALIZERS:
raise ValueError(
msg = (
f"'r_serializer' is {config['r_serializer']} and not one of "
f"{list(SERIALIZERS)}."
)
raise ValueError(msg)
config["r_suffix"] = config.get("r_suffix", "")
config["r_options"] = _parse_value_or_whitespace_option(config.get("r_options"))

Expand All @@ -27,4 +30,5 @@ def _parse_value_or_whitespace_option(value: Any) -> list[str] | None:
return None
if isinstance(value, list):
return list(map(str, value))
raise ValueError(f"'r_options' is {value} and not a list.")
msg = f"'r_options' is {value} and not a list."
raise ValueError(msg)
13 changes: 8 additions & 5 deletions src/pytask_r/execute.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
"""Execute tasks."""

from __future__ import annotations

import shutil
from typing import Any

from pytask import get_marks
from pytask import hookimpl
from pytask import PPathNode
from pytask import PTask
from pytask import PythonNode
from pytask import get_marks
from pytask import hookimpl
from pytask.tree_util import tree_map

from pytask_r.serialization import serialize_keyword_arguments
from pytask_r.shared import r

Expand All @@ -20,9 +22,10 @@ def pytask_execute_task_setup(task: PTask) -> None:
marks = get_marks(task, "r")
if marks:
if shutil.which("Rscript") is None:
raise RuntimeError(
msg = (
"Rscript is needed to run R scripts, but it is not found on your PATH."
)
raise RuntimeError(msg)

assert len(marks) == 1

Expand All @@ -31,9 +34,9 @@ def pytask_execute_task_setup(task: PTask) -> None:
assert suffix

serialized_node: PythonNode = task.depends_on["_serialized"] # type: ignore[assignment]
serialized_node.value.parent.mkdir(parents=True, exist_ok=True)
serialized_node.value.parent.mkdir(parents=True, exist_ok=True) # type: ignore[union-attr]
kwargs = collect_keyword_arguments(task)
serialize_keyword_arguments(serializer, serialized_node.value, kwargs)
serialize_keyword_arguments(serializer, serialized_node.value, kwargs) # type: ignore[arg-type]


def collect_keyword_arguments(task: PTask) -> dict[str, Any]:
Expand Down
8 changes: 7 additions & 1 deletion src/pytask_r/plugin.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
"""Register hook specifications and implementations."""

from __future__ import annotations

from pluggy import PluginManager
from typing import TYPE_CHECKING

from pytask import hookimpl

from pytask_r import collect
from pytask_r import config
from pytask_r import execute

if TYPE_CHECKING:
from pluggy import PluginManager


@hookimpl
def pytask_add_hooks(pm: PluginManager) -> None:
Expand Down
14 changes: 6 additions & 8 deletions src/pytask_r/serialization.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""This module contains the code to serialize keyword arguments to the task."""
"""Contains the code to serialize keyword arguments to the task."""

from __future__ import annotations

import json
Expand All @@ -9,7 +10,6 @@
from pytask import PTask
from pytask import PTaskWithPath


_HIDDEN_FOLDER = ".pytask/pytask-r"


Expand All @@ -29,8 +29,7 @@ def create_path_to_serialized(task: PTask, suffix: str) -> Path:
"""Create path to serialized."""
parent = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
file_name = create_file_name(task, suffix)
path = parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix)
return path
return parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix)


def create_file_name(task: PTask, suffix: str) -> str:
Expand Down Expand Up @@ -59,11 +58,10 @@ def serialize_keyword_arguments(
if callable(serializer):
serializer_func = serializer
elif isinstance(serializer, str) and serializer in SERIALIZERS:
serializer_func = SERIALIZERS[serializer][
"serializer"
] # type: ignore[assignment]
serializer_func = SERIALIZERS[serializer]["serializer"] # type: ignore[assignment]
else:
raise ValueError(f"Serializer {serializer!r} is not known.")
msg = f"Serializer {serializer!r} is not known."
raise ValueError(msg)

serialized = serializer_func(kwargs)
path_to_serialized.write_text(serialized)
Loading