diff --git a/.github/workflows/monodocs_build.yml b/.github/workflows/monodocs_build.yml deleted file mode 100644 index 6ecaa2cf87..0000000000 --- a/.github/workflows/monodocs_build.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Monodocs Build - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -on: - push: - branches: - - master - pull_request: - branches: - - master -jobs: - docs: - name: Monodocs Build - runs-on: ubuntu-latest - steps: - - name: Fetch flytekit code - uses: actions/checkout@v4 - - name: 'Clear action cache' - uses: ./.github/actions/clear-action-cache - - name: Fetch flyte code - uses: actions/checkout@v4 - with: - repository: flyteorg/flyte - path: "${{ github.workspace }}/flyte" - - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - python-version: 3.9 - - shell: bash -el {0} - working-directory: ${{ github.workspace }}/flyte - run: | - conda install -c conda-forge conda-lock - conda-lock install -n monodocs-env monodocs-environment.lock.yaml - - shell: bash -el {0} - working-directory: ${{ github.workspace }}/flyte - run: | - conda activate monodocs-env - export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0" - pip install -e ./flyteidl - - shell: bash -el {0} - run: | - conda activate monodocs-env - pip install -e . - conda info - conda list - conda config --show-sources - conda config --show - printenv | sort - - name: Build the documentation - working-directory: ${{ github.workspace }}/flyte - shell: bash -el {0} - env: - FLYTEKIT_LOCAL_PATH: ${{ github.workspace }} - DOCSEARCH_API_KEY: fake_docsearch_api_key # must be set to get doc build to succeed - run: | - conda activate monodocs-env - make -C docs clean html SPHINXOPTS="-W -vvv" diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 74a50aff7f..d638d592a8 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -1,208 +1,102 @@ """ -===================== -Core Flytekit -===================== - -.. currentmodule:: flytekit - This package contains all of the most common abstractions you'll need to write Flyte workflows and extend Flytekit. -Basic Authoring -=============== +## Basic Authoring -These are the essentials needed to get started writing tasks and workflows. -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ +These are the essentials needed to get started writing tasks and workflows. - task - workflow - kwtypes - current_context - ExecutionParameters - FlyteContext - map_task - ~core.workflow.ImperativeWorkflow - ~core.node_creation.create_node - ~core.promise.NodeOutput - FlyteContextManager +- task +- workflow +- kwtypes +- current_context +- ExecutionParameters +- FlyteContext +- map_task +- ImperativeWorkflow +- create_node +- NodeOutput +- FlyteContextManager -.. important:: +> [!NOTE] +> **Local Execution** +> +> Tasks and Workflows can both be locally run, assuming the relevant tasks are capable of local execution. +> This is useful for unit testing. - Tasks and Workflows can both be locally run, assuming the relevant tasks are capable of local execution. - This is useful for unit testing. +### Branching and Conditionals -Branching and Conditionals -========================== Branches and conditionals can be expressed explicitly in Flyte. These conditions are evaluated -in the flyte engine and hence should be used for control flow. ``dynamic workflows`` can be used to perform custom conditional logic not supported by flytekit +in the flyte engine and hence should be used for control flow. "dynamic workflows" can be used to perform custom conditional logic not supported by flytekit. -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - conditional +### Customizing Tasks & Workflows +- TaskMetadata - Wrapper object that allows users to specify Task +- Resources - Things like CPUs/Memory, etc. +- WorkflowFailurePolicy - Customizes what happens when a workflow fails. +- PodTemplate - Custom PodTemplate for a task. -Customizing Tasks & Workflows -============================== +#### Dynamic and Nested Workflows -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ +See the Dynamic module for more information. - TaskMetadata - Wrapper object that allows users to specify Task - Resources - Things like CPUs/Memory, etc. - WorkflowFailurePolicy - Customizes what happens when a workflow fails. - PodTemplate - Custom PodTemplate for a task. -Dynamic and Nested Workflows -============================== -See the :py:mod:`Dynamic ` module for more information. +##### Signaling -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ +- approve +- sleep +- wait_for_input - dynamic +Scheduling -Signaling -========= +- CronSchedule +- FixedRate -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ +##### Notifications - approve - sleep - wait_for_input +- Email +- PagerDuty +- Slack -Scheduling -============================ - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - CronSchedule - FixedRate - -Notifications -============================ - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - Email - PagerDuty - Slack - -Reference Entities -==================== - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - get_reference_entity - LaunchPlanReference - TaskReference - WorkflowReference - reference_task - reference_workflow - reference_launch_plan - -Core Task Types -================= - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - SQLTask - ContainerTask - PythonFunctionTask - PythonInstanceTask - LaunchPlan - -Secrets and SecurityContext -============================ - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - Secret - SecurityContext - - -Common Flyte IDL Objects -========================= - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - AuthRole - Labels - Annotations - WorkflowExecutionPhase - Blob - BlobMetadata - Literal - Scalar - LiteralType - BlobType - -Task Utilities -============== - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - HashMethod - Cache - CachePolicy - VersionParameters - -Artifacts -========= - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - Artifact - -Documentation -============= - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - Description - Documentation - SourceCode +##### Reference Entities + +- get_reference_entity +- LaunchPlanReference +- TaskReference +- WorkflowReference +- reference_task +- reference_workflow +- reference_launch_plan + +##### Core Task Types + +- SQLTask +- ContainerTask +- PythonFunctionTask +- PythonInstanceTask +- LaunchPlan + +##### Secrets and SecurityContext + +- Secret +- SecurityContext + + +##### Common Flyte IDL Objects + +- AuthRole +- Labels +- Annotations +- WorkflowExecutionPhase +- Blob +- BlobMetadata +- Literal +- Scalar +- LiteralType +- BlobType """ @@ -276,11 +170,11 @@ def current_context() -> ExecutionParameters: Usage - .. code-block:: python + ```python + flytekit.current_context().logging.info(...) + ``` - flytekit.current_context().logging.info(...) - - Available params are documented in :py:class:`flytekit.core.context_manager.ExecutionParams`. + Available params are documented in {{< py_class_ref flytekit.core.context_manager.ExecutionParams >}}. There are some special params, that should be available """ return FlyteContextManager.current_context().execution_state.user_space_params @@ -293,31 +187,38 @@ def new_context() -> Generator[FlyteContext, None, None]: def load_implicit_plugins(): """ This method allows loading all plugins that have the entrypoint specification. This uses the plugin loading - behavior as explained `here <>`_. + behavior. This is an opt in system and plugins that have an implicit loading requirement should add the implicit loading entrypoint specification to their setup.py. The following example shows how we can autoload a module called fsspec (whose init files contains the necessary plugin registration step) - .. code-block:: - # note the group is always ``flytekit.plugins`` - setup( + > [!NOTE] + > The group is always ``flytekit.plugins`` + + + ```python + setup( ... entry_points={'flytekit.plugins': 'fsspec=flytekitplugins.fsspec'}, ... - ) - + ) + ``` This works as long as the fsspec module has - .. code-block:: + > [!NOTE] + > For data persistence plugins: - # For data persistence plugins - DataPersistencePlugins.register_plugin(f"{k}://", FSSpecPersistence, force=True) - # OR for type plugins - TypeEngine.register(PanderaTransformer()) - # etc + ```python + DataPersistencePlugins.register_plugin(f"{k}://", FSSpecPersistence, force=True) + ``` + OR for type plugins: + ```python + TypeEngine.register(PanderaTransformer()) + # etc + ``` """ discovered_plugins = entry_points(group="flytekit.plugins") for p in discovered_plugins: diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 3011544dec..0fe93e8e9e 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -44,7 +44,10 @@ from flytekit.deck.deck import _output_deck from flytekit.exceptions.base import FlyteException from flytekit.exceptions.system import FlyteNonRecoverableSystemException -from flytekit.exceptions.user import FlyteRecoverableException, FlyteUserRuntimeException +from flytekit.exceptions.user import ( + FlyteRecoverableException, + FlyteUserRuntimeException, +) from flytekit.interfaces.stats.taggable import get_stats as _get_stats from flytekit.loggers import logger, user_space_logger from flytekit.models import dynamic_job as _dynamic_job @@ -52,7 +55,9 @@ from flytekit.models.core import errors as _error_models from flytekit.models.core import execution as _execution_models from flytekit.models.core import identifier as _identifier -from flytekit.tools.fast_registration import download_distribution as _download_distribution +from flytekit.tools.fast_registration import ( + download_distribution as _download_distribution, +) from flytekit.tools.module_loader import load_object_from_module from flytekit.utils.pbhash import compute_hash_string @@ -84,10 +89,9 @@ def _build_error_file_name() -> str: For distributed tasks, all workers upload error files which must not overwrite each other, leading to a race condition. A uuid is included to prevent this. - Returns - ------- - str - Name of the error file. + :rtype: str + :return: Name of the error file. + """ dist_error_strategy = get_one_of("FLYTE_INTERNAL_DIST_ERROR_STRATEGY", "_F_DES") if not dist_error_strategy: @@ -102,10 +106,7 @@ def _get_worker_name() -> str: For distributed tasks, the backend plugin can set a worker name to be used for error reporting. - Returns - ------- - str - Name of the worker + :return: Name of the worker """ dist_error_strategy = get_one_of("FLYTE_INTERNAL_DIST_ERROR_STRATEGY", "_F_DES") if not dist_error_strategy: @@ -139,11 +140,17 @@ def _dispatch_execute( ): """ Dispatches execute to PythonTask + Step1: Download inputs and load into a literal map + Step2: Invoke task - dispatch_execute + Step3: + a: [Optional] Record outputs to output_prefix + b: OR if IgnoreOutputs is raised, then ignore uploading outputs + c: OR if an unhandled exception is retrieved - record it as an errors.pb :param ctx: FlyteContext @@ -234,7 +241,10 @@ def _dispatch_execute( offloaded_literals[offloaded_filename] = v outputs = _literal_models.LiteralMap(literals=literal_map_copy) - output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs, **offloaded_literals} + output_file_dict = { + _constants.OUTPUT_FILE_NAME: outputs, + **offloaded_literals, + } elif isinstance(outputs, _dynamic_job.DynamicJobSpec): output_file_dict = {_constants.FUTURES_FILE_NAME: outputs} else: @@ -324,7 +334,10 @@ def _dispatch_execute( logger.info(f"Engine folder written successfully to the output prefix {output_prefix}") if task_def is not None and not getattr(task_def, "disable_deck", True): - _output_deck(task_name=task_def.name.split(".")[-1], new_user_params=ctx.user_space_params) + _output_deck( + task_name=task_def.name.split(".")[-1], + new_user_params=ctx.user_space_params, + ) logger.debug("Finished _dispatch_execute") @@ -365,15 +378,8 @@ def get_container_error_timestamp(e: Optional[Exception] = None) -> Timestamp: If a flyte exception is passed, use its timestamp, otherwise, use the current time. - Parameters - ---------- - e : Exception, optional - Exception that has occurred. - - Returns - ------- - Timestamp - Timestamp to be reported in ContainerError + :param e: Exception that has occurred. Optional. + :return: Timestamp to be reported in ContainerError """ timestamp = None if isinstance(e, FlyteException): @@ -619,7 +625,12 @@ def _execute_map_task( raise ValueError(f"Resolver args cannot be <1, got {resolver_args}") with setup_execution( - raw_output_data_prefix, output_prefix, checkpoint_path, prev_checkpoint, dynamic_addl_distro, dynamic_dest_dir + raw_output_data_prefix, + output_prefix, + checkpoint_path, + prev_checkpoint, + dynamic_addl_distro, + dynamic_dest_dir, ) as ctx: working_dir = os.getcwd() if all(os.path.realpath(path) != working_dir for path in sys.path): @@ -647,7 +658,9 @@ def load_task(): def normalize_inputs( - raw_output_data_prefix: Optional[str], checkpoint_path: Optional[str], prev_checkpoint: Optional[str] + raw_output_data_prefix: Optional[str], + checkpoint_path: Optional[str], + prev_checkpoint: Optional[str], ): # Backwards compatibility - if Propeller hasn't filled this in, then it'll come through here as the original # template string, so let's explicitly set it to None so that the downstream functions will know to fall back @@ -739,7 +752,14 @@ def fast_execute_task_cmd(additional_distribution: str, dest_dir: str, task_exec cmd = [] for arg in task_execute_cmd: if arg == "--resolver": - cmd.extend(["--dynamic-addl-distro", additional_distribution, "--dynamic-dest-dir", dest_dir]) + cmd.extend( + [ + "--dynamic-addl-distro", + additional_distribution, + "--dynamic-dest-dir", + dest_dir, + ] + ) cmd.append(arg) # Use the commandline to run the task execute command rather than calling it directly in python code diff --git a/flytekit/clients/__init__.py b/flytekit/clients/__init__.py index 1b08e1c567..c2bcfa0706 100644 --- a/flytekit/clients/__init__.py +++ b/flytekit/clients/__init__.py @@ -1,19 +1,3 @@ """ -===================== -Clients -===================== - -.. currentmodule:: flytekit.clients - This module provides lower level access to a Flyte backend. - -.. _clients_module: - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~friendly.SynchronousFlyteClient - ~raw.RawSynchronousFlyteClient """ diff --git a/flytekit/clients/auth_helper.py b/flytekit/clients/auth_helper.py index 317a16c5f4..c46517dd31 100644 --- a/flytekit/clients/auth_helper.py +++ b/flytekit/clients/auth_helper.py @@ -118,6 +118,7 @@ def upgrade_channel_to_proxy_authenticated(cfg: PlatformConfig, in_channel: grpc """ If activated in the platform config, given a grpc.Channel, preferably a secure channel, it returns a composed channel that uses Interceptor to perform authentication with a proxy in front of Flyte + :param cfg: PlatformConfig :param in_channel: grpc.Channel Precreated channel :return: grpc.Channel. New composite channel @@ -179,19 +180,24 @@ def get_channel(cfg: PlatformConfig, **kwargs) -> grpc.Channel: Creates a new grpc.Channel given a platformConfig. It is possible to pass additional options to the underlying channel. Examples for various options are as below - .. code-block:: python + ```python + get_channel(cfg=PlatformConfig(...)) + ``` - get_channel(cfg=PlatformConfig(...)) + > [!NOTE] + > Additional options to insecure / secure channel. Example `options` and `compression` refer to grpc guide - .. code-block:: python - :caption: Additional options to insecure / secure channel. Example `options` and `compression` refer to grpc guide + ```python + get_channel(cfg=PlatformConfig(...), options=..., compression=...) + ``` - get_channel(cfg=PlatformConfig(...), options=..., compression=...) - .. code-block:: python - :caption: Create secure channel with custom `grpc.ssl_channel_credentials` + > [!NOTE] + > Create secure channel with custom `grpc.ssl_channel_credentials` - get_channel(cfg=PlatformConfig(insecure=False,...), credentials=...) + ```python + get_channel(cfg=PlatformConfig(insecure=False,...), credentials=...) + ``` :param cfg: PlatformConfig @@ -233,7 +239,8 @@ def wrap_exceptions_channel(cfg: PlatformConfig, in_channel: grpc.Channel) -> gr Wraps the input channel with RetryExceptionWrapperInterceptor. This wrapper will cover all exceptions and raise Exception from the Family flytekit.exceptions - .. note:: This channel should be usually the outermost channel. This channel will raise a FlyteException + > [!NOTE] + > This channel should be usually the outermost channel. This channel will raise a FlyteException :param cfg: PlatformConfig :param in_channel: grpc.Channel diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index f52b9afe36..af9c708477 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -38,11 +38,13 @@ class SynchronousFlyteClient(_RawSynchronousFlyteClient): """ This is a low-level client that users can use to make direct gRPC service calls to the control plane. See the :std:doc:`service spec `. This is more user-friendly interface than the - :py:class:`raw client ` so users should try to use this class - first. Create a client by :: + {{< py_class_ref flytekit.clients.raw.RawSynchronousFlyteClient >}} so users should try to use this class + first. Create a client by - SynchronousFlyteClient("your.domain:port", insecure=True) - # insecure should be True if your flyteadmin deployment doesn't have SSL enabled + ```python + SynchronousFlyteClient("your.domain:port", insecure=True) + # insecure should be True if your flyteadmin deployment doesn't have SSL enabled + ``` """ @@ -65,11 +67,10 @@ def create_task(self, task_identifer, task_spec): This will create a task definition in the Admin database. Once successful, the task object can be retrieved via the client or viewed via the UI or command-line interfaces. - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in - the database must match the existing definition exactly. Furthermore, as long as the request - remains identical, calling this method multiple times will result in success. + > [!NOTE] + > Overwrites are not supported so any request for a given project, domain, name, and version that exists in + the database must match the existing definition exactly. Furthermore, as long as the request + remains identical, calling this method multiple times will result in success. :param flytekit.models.core.identifier.Identifier task_identifer: The identifier for this task. :param flytekit.models.task.TaskSpec task_spec: This is the actual definition of the task that @@ -88,14 +89,12 @@ def list_task_ids_paginated(self, project, domain, limit=100, token=None, sort_b This returns a page of identifiers for the tasks for a given project and domain. Filters can also be specified. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param Text project: The namespace of the project to list. @@ -129,14 +128,12 @@ def list_tasks_paginated(self, identifier, limit=100, token=None, filters=None, This returns a page of task metadata for tasks in a given project and domain. Optionally, specifying a name will limit the results to only tasks with that name in the given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param flytekit.models.common.NamedEntityIdentifier identifier: NamedEntityIdentifier to list. @@ -193,9 +190,8 @@ def create_workflow(self, workflow_identifier, workflow_spec): This will create a workflow definition in the Admin database. Once successful, the workflow object can be retrieved via the client or viewed via the UI or command-line interfaces. - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in + > [!NOTE] + > Overwrites are not supported so any request for a given project, domain, name, and version that exists in the database must match the existing definition exactly. Furthermore, as long as the request remains identical, calling this method multiple times will result in success. @@ -218,14 +214,12 @@ def list_workflow_ids_paginated(self, project, domain, limit=100, token=None, so This returns a page of identifiers for the workflows for a given project and domain. Filters can also be specified. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param: Text project: The namespace of the project to list. @@ -259,14 +253,12 @@ def list_workflows_paginated(self, identifier, limit=100, token=None, filters=No This returns a page of workflow meta-information for workflows in a given project and domain. Optionally, specifying a name will limit the results to only workflows with that name in the given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param flytekit.models.common.NamedEntityIdentifier identifier: NamedEntityIdentifier to list. @@ -323,9 +315,8 @@ def create_launch_plan(self, launch_plan_identifer, launch_plan_spec): This will create a launch plan definition in the Admin database. Once successful, the launch plan object can be retrieved via the client or viewed via the UI or command-line interfaces. - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in + > [!NOTE] + > Overwrites are not supported so any request for a given project, domain, name, and version that exists in the database must match the existing definition exactly. This also means that as long as the request remains identical, calling this method multiple times will result in success. @@ -375,14 +366,12 @@ def list_launch_plan_ids_paginated(self, project, domain, limit=100, token=None, This returns a page of identifiers for the launch plans for a given project and domain. Filters can also be specified. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param: Text project: The namespace of the project to list. @@ -416,14 +405,12 @@ def list_launch_plans_paginated(self, identifier, limit=100, token=None, filters This returns a page of launch plan meta-information for launch plans in a given project and domain. Optionally, specifying a name will limit the results to only workflows with that name in the given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param flytekit.models.common.NamedEntityIdentifier identifier: NamedEntityIdentifier to list. @@ -463,14 +450,12 @@ def list_active_launch_plans_paginated( This returns a page of currently active launch plan meta-information for launch plans in a given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param Text project: @@ -611,14 +596,12 @@ def list_executions_paginated(self, project, domain, limit=100, token=None, filt """ This returns a page of executions in a given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param Text project: Project in which to list executions. @@ -873,14 +856,12 @@ def list_projects_paginated(self, limit=100, token=None, filters=None, sort_by=N """ This returns a page of projects. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param int limit: [Optional] The maximum number of entries to return. Must be greater than 0. The maximum diff --git a/flytekit/clients/raw.py b/flytekit/clients/raw.py index 0ffbc4ce58..3216fe0c0e 100644 --- a/flytekit/clients/raw.py +++ b/flytekit/clients/raw.py @@ -28,11 +28,11 @@ class RawSynchronousFlyteClient(object): This client should be usable regardless of environment in which this is used. In other words, configurations should be explicit as opposed to inferred from the environment or a configuration file. To create a client, - .. code-block:: python - - from flytekit.configuration import PlatformConfig - RawSynchronousFlyteClient(PlatformConfig(endpoint="a.b.com", insecure=True)) # or - SynchronousFlyteClient(PlatformConfig(endpoint="a.b.com", insecure=True)) + ```python + from flytekit.configuration import PlatformConfig + RawSynchronousFlyteClient(PlatformConfig(endpoint="a.b.com", insecure=True)) # or + SynchronousFlyteClient(PlatformConfig(endpoint="a.b.com", insecure=True)) + ``` """ _dataproxy_stub: DataProxyServiceStub @@ -89,9 +89,8 @@ def create_task(self, task_create_request): This will create a task definition in the Admin database. Once successful, the task object can be retrieved via the client or viewed via the UI or command-line interfaces. - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in + > [!NOTE] + > Overwrites are not supported so any request for a given project, domain, name, and version that exists in the database must match the existing definition exactly. This also means that as long as the request remains identical, calling this method multiple times will result in success. @@ -109,18 +108,15 @@ def list_task_ids_paginated(self, identifier_list_request): This returns a page of identifiers for the tasks for a given project and domain. Filters can also be specified. - .. note :: - - The name field in the TaskListRequest is ignored. - - .. note :: + > [!NOTE] + > The name field in the TaskListRequest is ignored. - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param: flyteidl.admin.common_pb2.NamedEntityIdentifierListRequest identifier_list_request: @@ -134,14 +130,12 @@ def list_tasks_paginated(self, resource_list_request): This returns a page of task metadata for tasks in a given project and domain. Optionally, specifying a name will limit the results to only tasks with that name in the given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param: flyteidl.admin.common_pb2.ResourceListRequest resource_list_request: @@ -183,9 +177,8 @@ def create_workflow(self, workflow_create_request): This will create a workflow definition in the Admin database. Once successful, the workflow object can be retrieved via the client or viewed via the UI or command-line interfaces. - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in + > [!NOTE] + > Overwrites are not supported so any request for a given project, domain, name, and version that exists in the database must match the existing definition exactly. This also means that as long as the request remains identical, calling this method multiple times will result in success. @@ -203,18 +196,15 @@ def list_workflow_ids_paginated(self, identifier_list_request): This returns a page of identifiers for the workflows for a given project and domain. Filters can also be specified. - .. note :: - - The name field in the WorkflowListRequest is ignored. + > [!NOTE] + > The name field in the WorkflowListRequest is ignored. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param: flyteidl.admin.common_pb2.NamedEntityIdentifierListRequest identifier_list_request: @@ -228,14 +218,12 @@ def list_workflows_paginated(self, resource_list_request): This returns a page of workflow meta-information for workflows in a given project and domain. Optionally, specifying a name will limit the results to only workflows with that name in the given project and domain. - .. note :: - - This is a paginated API. Use the token field in the request to specify a page offset token. + > [!NOTE] + > This is a paginated API. Use the token field in the request to specify a page offset token. The user of the API is responsible for providing this token. - .. note :: - - If entries are added to the database between requests for different pages, it is possible to receive + > [!NOTE] + > If entries are added to the database between requests for different pages, it is possible to receive entries on the second page that also appeared on the first. :param: flyteidl.admin.common_pb2.ResourceListRequest resource_list_request: @@ -265,9 +253,8 @@ def create_launch_plan(self, launch_plan_create_request): This will create a launch plan definition in the Admin database. Once successful, the launch plan object can be retrieved via the client or viewed via the UI or command-line interfaces. - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in + > [!NOTE] + > Overwrites are not supported so any request for a given project, domain, name, and version that exists in the database must match the existing definition exactly. This also means that as long as the request remains identical, calling this method multiple times will result in success. diff --git a/flytekit/clis/sdk_in_container/serialize.py b/flytekit/clis/sdk_in_container/serialize.py index 85a089e7e1..7329a99ad7 100644 --- a/flytekit/clis/sdk_in_container/serialize.py +++ b/flytekit/clis/sdk_in_container/serialize.py @@ -37,15 +37,19 @@ def serialize_all( env: typing.Optional[typing.Dict[str, str]] = None, ): """ - This function will write to the folder specified the following protobuf types :: - flyteidl.admin.launch_plan_pb2.LaunchPlan - flyteidl.admin.workflow_pb2.WorkflowSpec - flyteidl.admin.task_pb2.TaskSpec - - These can be inspected by calling (in the launch plan case) :: - flyte-cli parse-proto -f filename.pb -p flyteidl.admin.launch_plan_pb2.LaunchPlan - - See :py:class:`flytekit.models.core.identifier.ResourceType` to match the trailing index in the file name with the + This function will write to the folder specified the following protobuf types + ``` + flyteidl.admin.launch_plan_pb2.LaunchPlan + flyteidl.admin.workflow_pb2.WorkflowSpec + flyteidl.admin.task_pb2.TaskSpec + ``` + + These can be inspected by calling (in the launch plan case) + ```bash + flyte-cli parse-proto -f filename.pb -p flyteidl.admin.launch_plan_pb2.LaunchPlan + ``` + + See {{< py_class_ref flytekit.models.core.identifier.ResourceType >}} to match the trailing index in the file name with the entity type. :param pkgs: Dot-delimited Python packages/subpackages to look into for serialization. :param local_source_root: Where to start looking for the code. diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index bee6feb8be..d27e07c886 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -1,68 +1,78 @@ """ -===================== -Configuration -===================== +# Configuration -.. currentmodule:: flytekit.configuration - -Flytekit Configuration Sources -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +## Flytekit Configuration Sources There are multiple ways to configure flytekit settings: -**Command Line Arguments**: This is the recommended way of setting configuration values for many cases. -For example, see `pyflyte package `_ command. +### Command Line Arguments +This is the recommended way of setting configuration values for many cases. For example, see `pyflyte package` command. + +### Python Config Object +A `Config` object can be used directly, e.g. when initializing a `FlyteRemote` object. See the Control Plane design docs for examples on how to specify a `Config` object. + +### Environment Variables +Users can specify these at compile time, but when your task is run, Flyte Propeller will also set configuration to ensure correct interaction with the platform. The environment variables must be specified with the format `FLYTE_{SECTION}_{OPTION}`, all in upper case. For example, to specify the `PlatformConfig.endpoint` setting, the environment variable would be `FLYTE_PLATFORM_URL`. -**Python Config Object**: A :py:class:`~flytekit.configuration.Config` object can by used directly, e.g. when -initializing a :py:class:`~flytefit.remote.remote.FlyteRemote` object. See :doc:`here ` for examples on -how to specify a ``Config`` object. +> [!NOTE] +> Environment variables won't work for image configuration, which need to be specified with the `pyflyte package --image ...` option or in a configuration file. -**Environment Variables**: Users can specify these at compile time, but when your task is run, Flyte Propeller will -also set configuration to ensure correct interaction with the platform. The environment variables must be specified -with the format ``FLYTE_{SECTION}_{OPTION}``, all in upper case. For example, to specify the -:py:class:`PlatformConfig.endpoint ` setting, the environment variable would -be ``FLYTE_PLATFORM_URL``. +### YAML Format Configuration File +A configuration file that contains settings for both `flytectl` and `flytekit`. This is the recommended configuration file format. Invoke the `flytectl config init` command to create a boilerplate `~/.flyte/config.yaml` file, and `flytectl --help` to learn about all of the configuration yaml options. -.. note:: +Example `config.yaml` file: - Environment variables won't work for image configuration, which need to be specified with the - `pyflyte package --image ... `_ option or in a configuration - file. +```YAML +# Sample config file -**YAML Format Configuration File**: A configuration file that contains settings for both -`flytectl `__ and ``flytekit``. This is the recommended configuration -file format. Invoke the :ref:`flytectl config init ` command to create a boilerplate -``~/.flyte/config.yaml`` file, and ``flytectl --help`` to learn about all of the configuration yaml options. +admin: + # For GRPC endpoints you might want to use dns:///flyte.myexample.com + endpoint: dns:///localhost:8089 + authType: Pkce + insecure: true -.. dropdown:: See example ``config.yaml`` file - :animate: fade-in-slide-down +logger: + show-source: true + level: 0 - .. literalinclude:: ../../tests/flytekit/unit/configuration/configs/sample.yaml - :language: yaml - :caption: config.yaml +console: + endpoint: http://localhost:8080 + insecure: true -**INI Format Configuration File**: A configuration file for ``flytekit``. By default, ``flytekit`` will look for a -file in two places: +# This section is used only in the control plane to trigger a remote execution +storage: + type: minio + stow: + kind: s3 + config: + auth_type: accesskey + access_key_id: minio + secret_key: miniostorage + endpoint: http://localhost:9000 + region: us-east-1 + disable_ssl: true + addressing_style: "path" +``` -1. First, a file named ``flytekit.config`` in the Python interpreter's working directory. -2. A file in ``~/.flyte/config`` in the home directory as detected by Python. +### INI Format Configuration File +A configuration file for `flytekit`. By default, `flytekit` will look for a file in two places: -.. dropdown:: See example ``flytekit.config`` file - :animate: fade-in-slide-down +1. First, a file named `flytekit.config` in the Python interpreter's working directory. +2. A file in `~/.flyte/config` in the home directory as detected by Python. - .. literalinclude:: ../../tests/flytekit/unit/configuration/configs/images.config - :language: ini - :caption: flytekit.config +Example `flytekit.config` file: -.. warning:: +```ini +[sdk] +workflow_packages=my_cool_workflows, other_workflows +``` - The INI format configuration is considered a legacy configuration format. We recommend using the yaml format - instead if you're using a configuration file. +> [!WARNING] +> The INI format configuration is considered a legacy configuration format. We recommend using the yaml format instead if you're using a configuration file. -How is configuration used? -^^^^^^^^^^^^^^^^^^^^^^^^^^ +## How is configuration used? -Configuration usage can roughly be bucketed into the following areas, +Configuration usage can roughly be bucketed into the following areas: - **Compile-time settings**: these are settings like the default image and named images, where to look for Flyte code, etc. - **Platform settings**: Where to find the Flyte backend (Admin DNS, whether to use SSL) @@ -70,60 +80,31 @@ - **Data access settings**: Is there a custom S3 endpoint in use? Backoff/retry behavior for accessing S3/GCS, key and password, etc. - **Other settings** - Statsd configuration, which is a run-time applicable setting but is not necessarily relevant to the Flyte platform. -Configuration Objects ---------------------- - -The following objects are encapsulated in a parent object called ``Config``. - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~Config +## Configuration Objects -.. _configuration-compile-time-settings: +The following objects are encapsulated in a parent object called `Config`: -Serialization Time Settings -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +### Serialization Time Settings -These are serialization/compile-time settings that are used when using commands like -`pyflyte package `_ or `pyflyte register `_. These -configuration settings are typically passed in as flags to the above CLI commands. +These are serialization/compile-time settings that are used when using commands like `pyflyte package` or `pyflyte register`. These configuration settings are typically passed in as flags to the above CLI commands. -The image configurations are typically either passed in via an `--image `_ flag, -or can be specified in the ``yaml`` or ``ini`` configuration files (see examples above). +The image configurations are typically either passed in via an `--image` flag, or can be specified in the `yaml` or `ini` configuration files (see examples above). -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: +- **Image**: Represents a container image with optional configuration overrides. +- **ImageConfig**: Represents an image configuration for a given project/domain combination. +- **SerializationSettings**: Controls how to serialize Flyte entities when registering with Admin. +- **FastSerializationSettings**: Configuration for faster serialization settings. - ~Image - ~ImageConfig - ~SerializationSettings - ~FastSerializationSettings +### Execution Time Settings -.. _configuration-execution-time-settings: +Users typically shouldn't be concerned with these configurations, as they are typically set by FlytePropeller or FlyteAdmin. The configurations below are useful for authenticating to a Flyte backend, configuring data access credentials, secrets, and statsd metrics. -Execution Time Settings -^^^^^^^^^^^^^^^^^^^^^^^ - -Users typically shouldn't be concerned with these configurations, as they are typically set by FlytePropeller or -FlyteAdmin. The configurations below are useful for authenticating to a Flyte backend, configuring data access -credentials, secrets, and statsd metrics. - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~PlatformConfig - ~StatsConfig - ~SecretsConfig - ~S3Config - ~GCSConfig - ~DataConfig +- **PlatformConfig**: Configuration for how to connect to the Flyte platform. +- **StatsConfig**: Configuration for how to emit statsd metrics. +- **SecretsConfig**: Configuration for how to access secrets. +- **S3Config**: Amazon S3 specific configuration. +- **GCSConfig**: Google Cloud Storage specific configuration. +- **DataConfig**: Configuration for data access. """ @@ -358,15 +339,15 @@ def from_images(cls, default_image: str, m: typing.Optional[typing.Dict[str, str Allows you to programmatically create an ImageConfig. Usually only the default_image is required, unless your workflow uses multiple images - .. code:: python - - ImageConfig.from_dict( - "ghcr.io/flyteorg/flytecookbook:v1.0.0", - { - "spark": "ghcr.io/flyteorg/myspark:...", - "other": "...", - } - ) + ```python + ImageConfig.from_dict( + "ghcr.io/flyteorg/flytecookbook:v1.0.0", + { + "spark": "ghcr.io/flyteorg/myspark:...", + "other": "...", + } + ) + ``` :return: """ diff --git a/flytekit/configuration/file.py b/flytekit/configuration/file.py index 521bc72f61..f36e285ded 100644 --- a/flytekit/configuration/file.py +++ b/flytekit/configuration/file.py @@ -285,9 +285,8 @@ def set_if_exists(d: dict, k: str, v: typing.Any) -> dict: Given a dict ``d`` sets the key ``k`` with value of config ``v``, if the config value ``v`` is set and return the updated dictionary. - .. note:: - - The input dictionary ``d`` will be mutated. + > [!NOTE] + > The input dictionary ``d`` will be mutated. """ if _exists(v): d[k] = v diff --git a/flytekit/core/annotation.py b/flytekit/core/annotation.py index b4a70a6469..ce0cb4693d 100644 --- a/flytekit/core/annotation.py +++ b/flytekit/core/annotation.py @@ -14,11 +14,11 @@ class FlyteAnnotation: For a task definition: - .. code-block:: python - - @task - def x(a: typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]): - return + ```python + @task + def x(a: typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]): + return + ``` """ diff --git a/flytekit/core/array_node_map_task.py b/flytekit/core/array_node_map_task.py index 96b062ded0..9df005ac33 100644 --- a/flytekit/core/array_node_map_task.py +++ b/flytekit/core/array_node_map_task.py @@ -426,9 +426,9 @@ def array_node_map_task( ): """Map task that uses the ``ArrayNode`` construct.. - .. important:: + > [!IMPORTANT] - This is an experimental drop-in replacement for :py:func:`~flytekit.map_task`. + > This is an experimental drop-in replacement for `flytekit.map_task`. :param task_function: This argument is implicitly passed and represents the repeatable function :param concurrency: If specified, this limits the number of mapped tasks than can run in parallel to the given batch @@ -454,14 +454,14 @@ class ArrayNodeMapTaskResolver(tracker.TrackedInstance, TaskResolverMixin): But in cases in which `j` is bound to a fixed value by using `functools.partial` we need a way to ensure that the interface is not simply interpolated, but only the unbound inputs are interpolated. - .. code-block:: python + ```python + def foo((i: int, j: str) -> str: + ... - def foo((i: int, j: str) -> str: - ... + mt = map_task(functools.partial(foo, j=10)) - mt = map_task(functools.partial(foo, j=10)) - - print(mt.interface) + print(mt.interface) + ``` output: diff --git a/flytekit/core/base_sql_task.py b/flytekit/core/base_sql_task.py index 500e19c260..c3d839b2ed 100644 --- a/flytekit/core/base_sql_task.py +++ b/flytekit/core/base_sql_task.py @@ -9,8 +9,8 @@ class SQLTask(PythonTask[T]): """ - Base task types for all SQL tasks. See :py:class:`flytekit.extras.sqlite3.task.SQLite3Task` - and :py:class:`flytekitplugins.athena.task.AthenaTask` for examples of how to use it as a base class. + Base task types for all SQL tasks. See `flytekit.extras.sqlite3.task.SQLite3Task` + and `flytekitplugins.athena.task.AthenaTask` for examples of how to use it as a base class. .. autoclass:: flytekit.extras.sqlite3.task.SQLite3Task :noindex: @@ -31,7 +31,7 @@ def __init__( ): """ This SQLTask should mostly just be used as a base class for other SQL task types and should not be used - directly. See :py:class:`flytekit.extras.sqlite3.task.SQLite3Task` + directly. See `flytekit.extras.sqlite3.task.SQLite3Task` """ super().__init__( task_type=task_type, diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 41da032fee..46fce167dd 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -1,20 +1,24 @@ """ -============================== -:mod:`flytekit.core.base_task` -============================== +# flytekit.core.base_task -.. currentmodule:: flytekit.core.base_task +This module provides the core task-related functionality in Flytekit. -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ +## Core Components - kwtypes - PythonTask - Task - TaskResolverMixin - IgnoreOutputs +### kwtypes +Utility for creating keyword type annotations for tasks. + +### PythonTask +Base class for Python-based task implementations. + +### Task +The base class for all Flyte tasks. + +### TaskResolverMixin +Mixin class that helps resolve a task implementation. + +### IgnoreOutputs +Exception that can be raised to ignore task outputs. """ @@ -97,9 +101,9 @@ def kwtypes(**kwargs) -> OrderedDict[str, Type]: """ This is a small helper function to convert the keyword arguments to an OrderedDict of types. - .. code-block:: python - - kwtypes(a=int, b=str) + ```python + kwtypes(a=int, b=str) + ``` """ d = collections.OrderedDict() for k, v in kwargs.items(): @@ -455,7 +459,7 @@ def execute(self, **kwargs) -> Any: class PythonTask(TrackedInstance, Task, Generic[T]): """ Base Class for all Tasks with a Python native ``Interface``. This should be directly used for task types, that do - not have a python function to be executed. Otherwise refer to :py:class:`flytekit.PythonFunctionTask`. + not have a python function to be executed. Otherwise refer to {{< py_class_ref flytekit.PythonFunctionTask >}}. """ def __init__( diff --git a/flytekit/core/checkpointer.py b/flytekit/core/checkpointer.py index d0fdf129e4..5f96faf0de 100644 --- a/flytekit/core/checkpointer.py +++ b/flytekit/core/checkpointer.py @@ -21,10 +21,9 @@ def restore(self, path: typing.Union[Path, str]) -> typing.Optional[Path]: Given a path, if a previous checkpoint exists, will be downloaded to this path. If download is successful the downloaded path is returned - .. note: - - Download will not be performed, if the checkpoint was previously restored. The method will return the - previously downloaded path. + > [!NOTE] + > Download will not be performed, if the checkpoint was previously restored. The method will return the + previously downloaded path. """ raise NotImplementedError("Use one of the derived classes") @@ -37,10 +36,10 @@ def save(self, cp: typing.Union[Path, str, io.BufferedReader]): Usage: If you have a io.BufferedReader then the following should work - .. code-block: python - - with input_file.open(mode="rb") as b: - checkpointer.save(b) + ```python + with input_file.open(mode="rb") as b: + checkpointer.save(b) + ``` """ raise NotImplementedError("Use one of the derived classes") diff --git a/flytekit/core/condition.py b/flytekit/core/condition.py index 50403574c1..2ab390dff5 100644 --- a/flytekit/core/condition.py +++ b/flytekit/core/condition.py @@ -38,16 +38,14 @@ class ConditionalSection: for Compilation mode. It is advised to derive the class and re-implement the `start_branch` and `end_branch` methods to override the compilation behavior - .. note:: - - Conditions can only be used within a workflow context. + > [!NOTE] + > Conditions can only be used within a workflow context. Usage: - .. code-block:: python - - v = conditional("fractions").if_((my_input > 0.1) & (my_input < 1.0)).then(...)... - + ```python + v = conditional("fractions").if_((my_input > 0.1) & (my_input < 1.0)).then(...)... + ``` """ def __init__(self, name: str): @@ -488,25 +486,25 @@ def conditional(name: str) -> ConditionalSection: Example of a condition usage. Note the nesting and the assignment to a LHS variable - .. code-block:: python - - v = ( - conditional("fractions") - .if_((my_input > 0.1) & (my_input < 1.0)) - .then( - conditional("inner_fractions") - .if_(my_input < 0.5) - .then(double(n=my_input)) - .elif_((my_input > 0.5) & (my_input < 0.7)) - .then(square(n=my_input)) - .else_() - .fail("Only <0.7 allowed") - ) - .elif_((my_input > 1.0) & (my_input < 10.0)) - .then(square(n=my_input)) - .else_() - .then(double(n=my_input)) - ) + ```python + v = ( + conditional("fractions") + .if_((my_input > 0.1) & (my_input < 1.0)) + .then( + conditional("inner_fractions") + .if_(my_input < 0.5) + .then(double(n=my_input)) + .elif_((my_input > 0.5) & (my_input < 0.7)) + .then(square(n=my_input)) + .else_() + .fail("Only <0.7 allowed") + ) + .elif_((my_input > 1.0) & (my_input < 10.0)) + .then(square(n=my_input)) + .else_() + .then(double(n=my_input)) + ) + ``` """ ctx = FlyteContextManager.current_context() diff --git a/flytekit/core/context_manager.py b/flytekit/core/context_manager.py index 6378f42706..e7ffcbe3b4 100644 --- a/flytekit/core/context_manager.py +++ b/flytekit/core/context_manager.py @@ -1,14 +1,5 @@ """ - -.. autoclass:: flytekit.core.context_manager::ExecutionState.Mode - :noindex: -.. autoclass:: flytekit.core.context_manager::ExecutionState.Mode.TASK_EXECUTION - :noindex: -.. autoclass:: flytekit.core.context_manager::ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION - :noindex: -.. autoclass:: flytekit.core.context_manager::ExecutionState.Mode.LOCAL_TASK_EXECUTION - :noindex: - +These classes provide functionality related context management. """ from __future__ import annotations @@ -68,17 +59,18 @@ class ExecutionParameters(object): """ This is a run-time user-centric context object that is accessible to every @task method. It can be accessed using - .. code-block:: python - flytekit.current_context() + ```python + flytekit.current_context() + ``` - This object provides the following + This object provides the following objections * a statsd handler * a logging handler - * the execution ID as an :py:class:`flytekit.models.core.identifier.WorkflowExecutionIdentifier` object + * the execution ID as an `flytekit.models.core.identifier.WorkflowExecutionIdentifier` object * a working directory for the user to write arbitrary files to - Please do not confuse this object with the :py:class:`flytekit.FlyteContext` object. + Please do not confuse this object with the `flytekit.FlyteContext` object. """ @dataclass(init=False) @@ -254,9 +246,8 @@ def execution_date(self) -> datetime: This is a datetime representing the time at which a workflow was started. This is consistent across all tasks executed in a workflow or sub-workflow. - .. note:: - - Do NOT use this execution_date to drive any production logic. It might be useful as a tag for data to help + > [!NOTE] + > Do NOT use this execution_date to drive any production logic. It might be useful as a tag for data to help in debugging. """ return self._execution_date @@ -267,9 +258,8 @@ def execution_id(self) -> _identifier.WorkflowExecutionIdentifier: This is the identifier of the workflow execution within the underlying engine. It will be consistent across all task executions in a workflow or sub-workflow execution. - .. note:: - - Do NOT use this execution_id to drive any production logic. This execution ID should only be used as a tag + > [!NOTE] + > Do NOT use this execution_id to drive any production logic. This execution ID should only be used as a tag on output data to link back to the workflow run that created it. """ return self._execution_id @@ -468,8 +458,8 @@ class CompilationState(object): prefix (str): This is because we may one day want to be able to have subworkflows inside other workflows. If users choose to not specify their node names, then we can end up with multiple "n0"s. This prefix allows us to give those nested nodes a distinct name, as well as properly identify them in the workflow. - mode (int): refer to :py:class:`flytekit.extend.ExecutionState.Mode` - task_resolver (Optional[TaskResolverMixin]): Please see :py:class:`flytekit.extend.TaskResolverMixin` + mode (int): refer to `flytekit.extend.ExecutionState.Mode` + task_resolver (Optional[TaskResolverMixin]): Please see `flytekit.extend.TaskResolverMixin` nodes (Optional[List]): Stores currently compiled nodes so far. """ @@ -688,9 +678,9 @@ class FlyteContext(object): compile workflows, serialize Flyte entities, etc. Even though this object as a ``current_context`` function on it, it should not be called directly. Please use the - :py:class:`flytekit.FlyteContextManager` object instead. + `flytekit.FlyteContextManager` object instead. - Please do not confuse this object with the :py:class:`flytekit.ExecutionParameters` object. + Please do not confuse this object with the `flytekit.ExecutionParameters` object. """ file_access: FileAccessProvider @@ -785,7 +775,7 @@ def current_context() -> FlyteContext: ``FlyteContextManager.current_context()`` instead. Users of flytekit should be wary not to confuse the object returned from this function - with :py:func:`flytekit.current_context` + with {{< py_func_ref flytekit.current_context >}} """ return FlyteContextManager.current_context() @@ -796,18 +786,18 @@ def get_deck(self) -> typing.Union[str, "IPython.core.display.HTML"]: # type:ig The return value depends on the execution environment. In a notebook, the return value is compatible with IPython.display and should be rendered in the notebook. - .. code-block:: python - - with flytekit.new_context() as ctx: - my_task(...) - ctx.get_deck() + ```python + with flytekit.new_context() as ctx: + my_task(...) + ctx.get_deck() + ``` OR if you wish to explicitly display - .. code-block:: python - - from IPython import display - display(ctx.get_deck()) + ```python + from IPython import display + display(ctx.get_deck()) + ``` """ from flytekit.deck.deck import _get_deck @@ -918,16 +908,16 @@ class FlyteContextManager(object): Typical usage is - .. code-block:: python - - FlyteContextManager.initialize() - with FlyteContextManager.with_context(o) as ctx: - pass + ```python + FlyteContextManager.initialize() + with FlyteContextManager.with_context(o) as ctx: + pass - # If required - not recommended you can use - FlyteContextManager.push_context() - # but correspondingly a pop_context should be called - FlyteContextManager.pop_context() + # If required - not recommended you can use + FlyteContextManager.push_context() + # but correspondingly a pop_context should be called + FlyteContextManager.pop_context() + ``` """ signal_handlers: typing.List[typing.Callable[[int, FrameType], typing.Any]] = [] diff --git a/flytekit/core/data_persistence.py b/flytekit/core/data_persistence.py index b7c9deda46..321851210e 100644 --- a/flytekit/core/data_persistence.py +++ b/flytekit/core/data_persistence.py @@ -1,21 +1,7 @@ """ -====================================== -:mod:`flytekit.core.data_persistence` -====================================== - -.. currentmodule:: flytekit.core.data_persistence - The Data persistence module is used by core flytekit and most of the core TypeTransformers to manage data fetch & store, between the durable backend store and the runtime environment. This is designed to be a pluggable system, with a default simple implementation that ships with the core. - -.. autosummary:: - :toctree: generated/ - :template: custom.rst - :nosignatures: - - FileAccessProvider - """ import asyncio diff --git a/flytekit/core/dynamic_workflow_task.py b/flytekit/core/dynamic_workflow_task.py index a9ff5055db..59b50fb57a 100644 --- a/flytekit/core/dynamic_workflow_task.py +++ b/flytekit/core/dynamic_workflow_task.py @@ -1,7 +1,7 @@ """ Dynamic Workflows ----------------------------- -Dynamic workflows are one of the powerful aspects of Flyte. Please take a look at the :py:func:`flytekit.dynamic` documentation first to get started. +Dynamic workflows are one of the powerful aspects of Flyte. Please take a look at the {{< py_func_ref flytekit.dynamic >}} documentation first to get started. Caveats when using a dynamic workflow @@ -20,7 +20,7 @@ dynamic = functools.partial(task.task, execution_mode=PythonFunctionTask.ExecutionBehavior.DYNAMIC) # type: ignore[var-annotated] dynamic.__doc__ = """ -Please first see the comments for :py:func:`flytekit.task` and :py:func:`flytekit.workflow`. This ``dynamic`` +Please first see the comments for {{< py_func_ref flytekit.task >}} and {{< py_func_ref flytekit.workflow >}}. This ``dynamic`` concept is an amalgamation of both and enables the user to pursue some :std:ref:`pretty incredible ` constructs. @@ -31,24 +31,24 @@ The resulting workflow is passed back to the Flyte engine and is run as a :std:ref:`subworkflow `. Simple usage -.. code-block:: - - @dynamic - def my_dynamic_subwf(a: int) -> (typing.List[str], int): - s = [] - for i in range(a): - s.append(t1(a=i)) - return s, 5 +```python +@dynamic +def my_dynamic_subwf(a: int) -> (typing.List[str], int): + s = [] + for i in range(a): + s.append(t1(a=i)) + return s, 5 +``` Note in the code block that we call the Python ``range`` operator on the input. This is typically not allowed in a workflow but it is here. You can even express dependencies between tasks. -.. code-block:: - - @dynamic - def my_dynamic_subwf(a: int, b: int) -> int: - x = t1(a=a) - return t2(b=b, x=x) +```python +@dynamic +def my_dynamic_subwf(a: int, b: int) -> int: + x = t1(a=a) + return t2(b=b, x=x) +``` See the :std:ref:`cookbook ` for a longer discussion. """ # noqa: W293 diff --git a/flytekit/core/launch_plan.py b/flytekit/core/launch_plan.py index 05ba393dd4..7dd5ba6f86 100644 --- a/flytekit/core/launch_plan.py +++ b/flytekit/core/launch_plan.py @@ -25,27 +25,19 @@ class LaunchPlan(object): Every workflow is registered with a default launch plan, which is just a launch plan with none of the additional attributes set - no default values, fixed values, schedules, etc. Assuming you have the following workflow - .. code-block:: python - - @workflow - def wf(a: int, c: str) -> str: + ```python + @workflow + def wf(a: int, c: str) -> str: ... - + ``` Create the default launch plan with - .. code-block:: python - - LaunchPlan.get_or_create(workflow=my_wf) - + ```python + LaunchPlan.get_or_create(workflow=my_wf) + ``` If you specify additional parameters, you'll also have to give the launch plan a unique name. Default and fixed inputs can be expressed as Python native values like so: - .. literalinclude:: ../../../tests/flytekit/unit/core/test_launch_plan.py - :start-after: # fixed_and_default_start - :end-before: # fixed_and_default_end - :language: python - :dedent: 4 - Additionally, a launch plan can be configured to run on a schedule and emit notifications. @@ -53,23 +45,10 @@ def wf(a: int, c: str) -> str: To configure the remaining parameters, you'll need to import the relevant model objects as well. - .. literalinclude:: ../../../tests/flytekit/unit/core/test_launch_plan.py - :start-after: # schedule_start - :end-before: # schedule_end - :language: python - :dedent: 4 - - .. code-block:: python - - from flytekit.models.common import Annotations, AuthRole, Labels, RawOutputDataConfig - - Then use as follows - - .. literalinclude:: ../../../tests/flytekit/unit/core/test_launch_plan.py - :start-after: # auth_role_start - :end-before: # auth_role_end - :language: python - :dedent: 4 + ```python + from flytekit.models.common import Annotations, AuthRole, Labels, RawOutputDataConfig + ``` + Then use as follows: """ diff --git a/flytekit/core/legacy_map_task.py b/flytekit/core/legacy_map_task.py index 82f5fb1da7..8efba55c12 100644 --- a/flytekit/core/legacy_map_task.py +++ b/flytekit/core/legacy_map_task.py @@ -30,8 +30,8 @@ class MapPythonTask(PythonTask): """ - A MapPythonTask defines a :py:class:`flytekit.PythonTask` which specifies how to run - an inner :py:class:`flytekit.PythonFunctionTask` across a range of inputs in parallel. + A MapPythonTask defines a {{< py_class_ref flytekit.PythonTask >}} which specifies how to run + an inner {{< py_class_ref flytekit.PythonFunctionTask >}} across a range of inputs in parallel. """ def __init__( @@ -308,20 +308,36 @@ def map_task( ): """ Use a map task for parallelizable tasks that run across a list of an input type. A map task can be composed of - any individual :py:class:`flytekit.PythonFunctionTask`. + any individual {{}}. - Invoke a map task with arguments using the :py:class:`list` version of the expected input. + Invoke a map task with arguments using {{}} version of the expected input. Usage: + + + ```python + @task + def my_mappable_task(a: int) -> typing.Optional[str]: + return str(a) + + @workflow + def my_wf(x: typing.List[int]) -> typing.List[typing.Optional[str]]: + return map_task( + my_mappable_task, + metadata=TaskMetadata(retries=1), + concurrency=10, + min_success_ratio=0.75, + )(a=x).with_overrides(requests=Resources(cpu="10M")) + ``` At run time, the underlying map task will be run for every value in the input collection. Attributes - such as :py:class:`flytekit.TaskMetadata` and ``with_overrides`` are applied to individual instances + such as {{}} and ``with_overrides`` are applied to individual instances of the mapped task. **Map Task Plugins** @@ -329,9 +345,9 @@ def map_task( There are two plugins to run maptasks that ship as part of flyteplugins: 1. K8s Array - 2. `AWS batch `_ + 2. [`AWS batch`](https://docs.flyte.org/en/latest/deployment/plugin_setup/aws/batch.html) - Enabling a plugin is controlled in the plugin configuration at `values-sandbox.yaml `_. + Enabling a plugin is controlled in the plugin configuration at [`values-sandbox.yaml`](https://github.com/flyteorg/flyte/blob/10cee9f139824512b6c5be1667d321bdbc8835fa/charts/flyte/values-sandbox.yaml#L152-L162). **K8s Array** @@ -339,7 +355,7 @@ def map_task( **AWS batch** - Learn more about ``AWS batch`` setup configuration `here `_. + Learn more about ``AWS batch`` setup configuration [`here`](https://docs.flyte.org/en/latest/deployment/plugin_setup/aws/batch.html#deployment-plugin-setup-aws-array). A custom plugin can also be implemented to handle the task type. @@ -367,14 +383,14 @@ class MapTaskResolver(TrackedInstance, TaskResolverMixin): But in cases in which `j` is bound to a fixed value by using `functools.partial` we need a way to ensure that the interface is not simply interpolated, but only the unbound inputs are interpolated. - .. code-block:: python - - def foo((i: int, j: str) -> str: - ... + ```python + def foo((i: int, j: str) -> str: + ... - mt = map_task(functools.partial(foo, j=10)) + mt = map_task(functools.partial(foo, j=10)) - print(mt.interface) + print(mt.interface) + ``` output: diff --git a/flytekit/core/local_fsspec.py b/flytekit/core/local_fsspec.py index 91fe93ad6f..bd904b3349 100644 --- a/flytekit/core/local_fsspec.py +++ b/flytekit/core/local_fsspec.py @@ -1,19 +1,4 @@ -""" -====================================== -:mod:`flytekit.core.local_fsspec` -====================================== - -.. currentmodule:: flytekit.core.local_fsspec - - -.. autosummary:: - :toctree: generated/ - :template: custom.rst - :nosignatures: - - FlyteLocalFileSystem - -""" +""" """ import os diff --git a/flytekit/core/notification.py b/flytekit/core/notification.py index c964c67568..6a8eb2e1b0 100644 --- a/flytekit/core/notification.py +++ b/flytekit/core/notification.py @@ -2,18 +2,10 @@ Notifications are primarily used when defining Launch Plans (also can be used when launching executions) and will trigger the Flyte platform to send emails when a workflow run reaches certain stages (fails or succeeds, etc.). -.. note:: - - Notifications require some setup and configuration on the Flyte platform side. Please contact your Flyte platform - admins to get this feature enabled. See :std:ref:`cookbook:setting up workflow notifications` - -Each notification type takes a list of :py:class:`flytekit.models.core.execution.WorkflowExecutionPhase` and a list of +Each notification type takes a list of {{< py_class_ref flytekit.models.core.execution.WorkflowExecutionPhase >}} and a list of emails. Even though there are different notification classes in this module, they all just send email. The differentiation offers semantic meaning to the end-user but do not functionally behave differently. Successful integration with Slack and Pagerduty is incumbent on those email API being set-up correctly. - -.. autoclass:: flytekit.core.notification.Notification - """ from typing import List @@ -60,11 +52,11 @@ class PagerDuty(Notification): """ This notification should be used when sending emails to the PagerDuty service. - .. code-block:: python + ```python + from flytekit.models.core.execution import WorkflowExecutionPhase - from flytekit.models.core.execution import WorkflowExecutionPhase - - PagerDuty(phases=[WorkflowExecutionPhase.SUCCEEDED], recipients_email=["my-team@email.com"]) + PagerDuty(phases=[WorkflowExecutionPhase.SUCCEEDED], recipients_email=["my-team@email.com"]) + ``` """ def __init__(self, phases: List[int], recipients_email: List[str]): @@ -72,6 +64,7 @@ def __init__(self, phases: List[int], recipients_email: List[str]): :param list[int] phases: A required list of phases for which to fire the event. Events can only be fired for terminal phases. Phases should be as defined in: flytekit.models.core.execution.WorkflowExecutionPhase :param list[str] recipients_email: A required non-empty list of recipients for the notification. + """ super(PagerDuty, self).__init__(phases, pager_duty=_common_model.PagerDutyNotification(recipients_email)) @@ -80,17 +73,17 @@ class Email(Notification): """ This notification should be used when sending regular emails to people. - .. code-block:: python + ```python + from flytekit.models.core.execution import WorkflowExecutionPhase - from flytekit.models.core.execution import WorkflowExecutionPhase - - Email(phases=[WorkflowExecutionPhase.SUCCEEDED], recipients_email=["my-team@email.com"]) + Email(phases=[WorkflowExecutionPhase.SUCCEEDED], recipients_email=["my-team@email.com"]) + ``` """ def __init__(self, phases: List[int], recipients_email: List[str]): """ :param list[int] phases: A required list of phases for which to fire the event. Events can only be fired for - terminal phases. Phases should be as defined in: :py:class:`flytekit.models.core.execution.WorkflowExecutionPhase` + terminal phases. Phases should be as defined in: {{< py_class_ref flytekit.models.core.execution.WorkflowExecutionPhase >}} :param list[str] recipients_email: A required non-empty list of recipients for the notification. """ super(Email, self).__init__(phases, email=_common_model.EmailNotification(recipients_email)) @@ -100,11 +93,11 @@ class Slack(Notification): """ This notification should be used when sending emails to the Slack. - .. code-block:: python - - from flytekit.models.core.execution import WorkflowExecutionPhase + ```python + from flytekit.models.core.execution import WorkflowExecutionPhase - Slack(phases=[WorkflowExecutionPhase.SUCCEEDED], recipients_email=["my-team@email.com"]) + Slack(phases=[WorkflowExecutionPhase.SUCCEEDED], recipients_email=["my-team@email.com"]) + ``` """ def __init__(self, phases: List[int], recipients_email: List[str]): diff --git a/flytekit/core/promise.py b/flytekit/core/promise.py index b9b8a150e4..e92094ca04 100644 --- a/flytekit/core/promise.py +++ b/flytekit/core/promise.py @@ -84,7 +84,7 @@ def my_wf(in1: int, in2: int) -> int: :param ctx: Context needed in case a non-primitive literal needs to be translated to a Flyte literal (like a file) :param incoming_values: This is a map of your task's input or wf's output kwargs basically - :param flyte_interface_types: One side of an :py:class:`flytekit.models.interface.TypedInterface` basically. + :param flyte_interface_types: One side of an {{< py_class_ref flytekit.models.interface.TypedInterface >}} basically. :param native_types: Map to native Python type. """ if incoming_values is None: diff --git a/flytekit/core/python_auto_container.py b/flytekit/core/python_auto_container.py index aa0327299b..9e7fe224d8 100644 --- a/flytekit/core/python_auto_container.py +++ b/flytekit/core/python_auto_container.py @@ -74,10 +74,10 @@ def __init__( to provide secrets and if secrets are available in the configured secrets store. Possible options for secret stores are - - `Vault `__ - - `Confidant `__ - - `Kube secrets `__ - - `AWS Parameter store `__ + - [`Vault`](https://www.vaultproject.io/) + - [`Confidant`](https://lyft.github.io/confidant) + - [`Kube secrets`](https://kubernetes.io/docs/concepts/configuration/secret) + - [`AWS Parameter store`](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) :param pod_template: Custom PodTemplate for this task. :param pod_template_name: The name of the existing PodTemplate resource which will be used in this task. :param accelerator: The accelerator to use for this task. diff --git a/flytekit/core/python_customized_container_task.py b/flytekit/core/python_customized_container_task.py index ff014bd479..9dafc014ea 100644 --- a/flytekit/core/python_customized_container_task.py +++ b/flytekit/core/python_customized_container_task.py @@ -25,7 +25,7 @@ class PythonCustomizedContainerTask(ExecutableTemplateShimTask, PythonTask[TC]): # type: ignore """ - Please take a look at the comments for :py:class`flytekit.extend.ExecutableTemplateShimTask` as well. This class + Please take a look at the comments for {{< py_class_ref flytekit.extend.ExecutableTemplateShimTask >}} as well. This class should be subclassed and a custom Executor provided as a default to this parent class constructor when building a new external-container flytekit-only plugin. @@ -84,10 +84,10 @@ def __init__( The key values will be available from runtime, if the backend is configured to provide secrets and if secrets are available in the configured secrets store. Possible options for secret stores are - - `Vault `__ - - `Confidant `__ - - `Kube secrets `__ - - `AWS Parameter store `__ + - [`Vault`](https://www.vaultproject.io) + - [`Confidant`](https://lyft.github.io/confidant) + - [`Kube secrets`](https://kubernetes.io/docs/concepts/configuration/secret) + - [`AWS Parameter store`](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) """ sec_ctx = None if secret_requests: diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index 48d29e6625..ae7116da96 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -1,20 +1,3 @@ -""" -========================================= -:mod:`flytekit.core.python_function_task` -========================================= - -.. currentmodule:: flytekit.core.python_function_task - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - PythonFunctionTask - PythonInstanceTask - -""" - from __future__ import annotations import inspect @@ -81,12 +64,12 @@ class PythonInstanceTask(PythonAutoContainerTask[T], ABC): # type: ignore a platform defined execute method. (Execute needs to be overridden). This base class ensures that the module loader will invoke the right class automatically, by capturing the module name and variable in the module name. - .. code-block: python - - x = MyInstanceTask(name="x", .....) + ```python + x = MyInstanceTask(name="x", .....) - # this can be invoked as - x(a=5) # depending on the interface of the defined task + # this can be invoked as + x(a=5) # depending on the interface of the defined task + ``` """ @@ -112,12 +95,11 @@ class PythonFunctionTask(PythonAutoContainerTask[T]): # type: ignore It is advised this task is used using the @task decorator as follows - .. code-block: python - - @task - def my_func(a: int) -> str: - ... - + ```python + @task + def my_func(a: int) -> str: + ... + ``` In the above code, the name of the function, the module, and the interface (inputs = int and outputs = str) will be auto detected. """ diff --git a/flytekit/core/reference.py b/flytekit/core/reference.py index 6a88549c43..7832fcaafc 100644 --- a/flytekit/core/reference.py +++ b/flytekit/core/reference.py @@ -19,19 +19,32 @@ def get_reference_entity( outputs: Dict[str, Type], ): """ - See the documentation for :py:class:`flytekit.reference_task` and :py:class:`flytekit.reference_workflow` as well. + See the documentation for {{< py_class_ref flytekit.reference_task >}} and {{< py_class_ref flytekit.reference_workflow >}} as well. This function is the general form of the two aforementioned functions. It's better for programmatic usage, as the interface is passed in as arguments instead of analyzed from type annotations. + + ```python + ref_entity = get_reference_entity( + _identifier_model.ResourceType.WORKFLOW, + "project", + "dev", + "my.other.workflow", + "abc123", + inputs=kwtypes(a=str, b=int), + outputs={}, + ) + ``` :param resource_type: This is the type of entity it is. Must be one of - :py:class:`flytekit.models.core.identifier.ResourceType` + {{< py_class_ref flytekit.models.core.identifier.ResourceType >}} :param project: The project the entity you're looking for has been registered in. :param domain: The domain the entity you're looking for has been registered in. :param name: The name of the registered entity diff --git a/flytekit/core/resources.py b/flytekit/core/resources.py index 27e781e7ad..36b9c256d0 100644 --- a/flytekit/core/resources.py +++ b/flytekit/core/resources.py @@ -18,28 +18,26 @@ class Resources(DataClassJSONMixin): """ This class is used to specify both resource requests and resource limits. - .. code-block:: python - - Resources(cpu="1", mem="2048") # This is 1 CPU and 2 KB of memory - Resources(cpu="100m", mem="2Gi") # This is 1/10th of a CPU and 2 gigabytes of memory - Resources(cpu=0.5, mem=1024) # This is 500m CPU and 1 KB of memory - - # For Kubernetes-based tasks, pods use ephemeral local storage for scratch space, caching, and for logs. - # This allocates 1Gi of such local storage. - Resources(ephemeral_storage="1Gi") - + ```python + Resources(cpu="1", mem="2048") # This is 1 CPU and 2 KB of memory + Resources(cpu="100m", mem="2Gi") # This is 1/10th of a CPU and 2 gigabytes of memory + Resources(cpu=0.5, mem=1024) # This is 500m CPU and 1 KB of memory + + # For Kubernetes-based tasks, pods use ephemeral local storage for scratch space, caching, and for logs. + # This allocates 1Gi of such local storage. + Resources(ephemeral_storage="1Gi") + ``` When used together with `@task(resources=)`, you a specific the request and limits with one object. When the value is set to a tuple or list, the first value is the request and the second value is the limit. If the value is a single value, then both the requests and limit is set to that value. For example, the `Resource(cpu=("1", "2"), mem=1024)` will set the cpu request to 1, cpu limit to 2, mem limit and request to 1024. - .. note:: - - Persistent storage is not currently supported on the Flyte backend. + > [!NOTE] + > Persistent storage is not currently supported on the Flyte backend. Please see the :std:ref:`User Guide ` for detailed examples. - Also refer to the `K8s conventions. `__ + Also refer to the [`K8s conventions.`](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) """ cpu: Optional[Union[str, int, float, list, tuple]] = None diff --git a/flytekit/core/schedule.py b/flytekit/core/schedule.py index 891fb17a24..d9bb621cf0 100644 --- a/flytekit/core/schedule.py +++ b/flytekit/core/schedule.py @@ -1,7 +1,5 @@ """ -.. autoclass:: flytekit.core.schedule.CronSchedule - :noindex: - +These classes provide functionality related to schedules. """ import datetime @@ -23,14 +21,15 @@ def to_flyte_idl(self, *args, **kwargs) -> google_message.Message: ... class CronSchedule(_schedule_models.Schedule): """ Use this when you have a launch plan that you want to run on a cron expression. - This uses standard `cron format `__ + This uses standard [`cron format`](https://docs.flyte.org/en/latest/concepts/schedules.html#cron-expression-table) in case where you are using default native scheduler using the schedule attribute. - .. code-block:: + ``` CronSchedule( schedule="*/1 * * * *", # Following schedule runs every min ) + ``` See the :std:ref:`User Guide ` for further examples. """ @@ -68,7 +67,7 @@ def __init__( """ :param str cron_expression: This should be a cron expression in AWS style.Shouldn't be used in case of native scheduler. :param str schedule: This takes a cron alias (see ``_VALID_CRON_ALIASES``) or a croniter parseable schedule. - Only one of this or ``cron_expression`` can be set, not both. This uses standard `cron format `_ + Only one of this or ``cron_expression`` can be set, not both. This uses standard [`cron format`](https://docs.flyte.org/en/latest/concepts/schedules.html#cron-expression) and is supported by native scheduler :param str offset: :param str kickoff_time_input_arg: This is a convenient argument to use when your code needs to know what time @@ -160,11 +159,11 @@ class FixedRate(_schedule_models.Schedule): """ Use this class to schedule a fixed-rate interval for a launch plan. - .. code-block:: python - - from datetime import timedelta + ```python + from datetime import timedelta - FixedRate(duration=timedelta(minutes=10)) + FixedRate(duration=timedelta(minutes=10)) + ``` See the :std:ref:`fixed rate intervals` chapter in the cookbook for additional usage examples. """ diff --git a/flytekit/core/shim_task.py b/flytekit/core/shim_task.py index b205bbab08..35e42c858a 100644 --- a/flytekit/core/shim_task.py +++ b/flytekit/core/shim_task.py @@ -25,10 +25,9 @@ class ExecutableTemplateShimTask(object): Basically at execution time (both locally and on a Flyte cluster), the task template is given to the executor, which is responsible for computing and returning the results. - .. note:: - - The interface at execution time will have to derived from the Flyte IDL interface, which means it may be lossy. - This is because when a task is serialized from Python into the ``TaskTemplate`` some information is lost because + > [!NOTE] + > The interface at execution time will have to derived from the Flyte IDL interface, which means it may be lossy. + This is because when a task is serialized from Python into the ``TaskTemplate`` some information is lost because Flyte IDL can't keep track of every single Python type (or Java type if writing in the Java flytekit). This class also implements the ``dispatch_execute`` and ``execute`` functions to make it look like a ``PythonTask`` diff --git a/flytekit/core/task.py b/flytekit/core/task.py index 25c12c4b4a..8710d7baeb 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -34,17 +34,16 @@ class TaskPlugins(object): Every task that the user wishes to use should be available in this factory. Usage - .. code-block:: python - - TaskPlugins.register_pythontask_plugin(config_object_type, plugin_object_type) - # config_object_type is any class that will be passed to the plugin_object as task_config - # Plugin_object_type is a derivative of ``PythonFunctionTask`` - + ```python + TaskPlugins.register_pythontask_plugin(config_object_type, plugin_object_type) + # config_object_type is any class that will be passed to the plugin_object as task_config + # Plugin_object_type is a derivative of ``PythonFunctionTask`` + ``` Examples of available task plugins include different query-based plugins such as - :py:class:`flytekitplugins.athena.task.AthenaTask` and :py:class:`flytekitplugins.hive.task.HiveTask`, kubeflow - operators like :py:class:`plugins.kfpytorch.flytekitplugins.kfpytorch.task.PyTorchFunctionTask` and - :py:class:`plugins.kftensorflow.flytekitplugins.kftensorflow.task.TensorflowFunctionTask`, and generic plugins like - :py:class:`flytekitplugins.pod.task.PodFunctionTask` which doesn't integrate with third party tools or services. + {{< py_class_ref flytekitplugins.athena.task.AthenaTask >}} and {{< py_class_ref flytekitplugins.hive.task.HiveTask >}}, kubeflow + operators like {{< py_class_ref plugins.kfpytorch.flytekitplugins.kfpytorch.task.PyTorchFunctionTask >}} and + {{< py_class_ref plugins.kftensorflow.flytekitplugins.kftensorflow.task.TensorflowFunctionTask >}}, and generic plugins like + {{< py_class_ref flytekitplugins.pod.task.PodFunctionTask >}} which doesn't integrate with third party tools or services. The `task_config` is different for every task plugin type. This is filled out by users when they define a task to specify plugin-specific behavior and features. For example, with a query type task plugin, the config might store @@ -61,11 +60,11 @@ def register_pythontask_plugin(cls, plugin_config_type: type, plugin: Type[Pytho """ Use this method to register a new plugin into Flytekit. Usage :: - .. code-block:: python - - TaskPlugins.register_pythontask_plugin(config_object_type, plugin_object_type) - # config_object_type is any class that will be passed to the plugin_object as task_config - # Plugin_object_type is a derivative of ``PythonFunctionTask`` + ```python + TaskPlugins.register_pythontask_plugin(config_object_type, plugin_object_type) + # config_object_type is any class that will be passed to the plugin_object as task_config + # Plugin_object_type is a derivative of ``PythonFunctionTask`` + ``` """ if plugin_config_type in cls._PYTHONFUNCTION_TASK_PLUGINS: found = cls._PYTHONFUNCTION_TASK_PLUGINS[plugin_config_type] @@ -231,20 +230,19 @@ def task( For a simple python task, - .. code-block:: python - - @task - def my_task(x: int, y: typing.Dict[str, str]) -> str: - ... + ```python + @task + def my_task(x: int, y: typing.Dict[str, str]) -> str: + ... + ``` For specific task types - .. code-block:: python - - @task(task_config=Spark(), retries=3) - def my_task(x: int, y: typing.Dict[str, str]) -> str: - ... - + ```python + @task(task_config=Spark(), retries=3) + def my_task(x: int, y: typing.Dict[str, str]) -> str: + ... + ``` Please see some cookbook :std:ref:`task examples ` for additional information. :param _task_function: This argument is implicitly passed and represents the decorated function @@ -281,35 +279,34 @@ def my_task(x: int, y: typing.Dict[str, str]) -> str: bloat because of various dependencies and a dependency is only required for this or a set of tasks, and they vary from the default. - .. code-block:: python - - # Use default image name `fqn` and alter the tag to `tag-{{default.tag}}` tag of the default image - # with a prefix. In this case, it is assumed that the image like - # flytecookbook:tag-gitsha is published alongwith the default of flytecookbook:gitsha - @task(container_image='{{.images.default.fqn}}:tag-{{images.default.tag}}') - def foo(): - ... - - # Refer to configurations to configure fqns for other images besides default. In this case it will - # lookup for an image named xyz - @task(container_image='{{.images.xyz.fqn}}:{{images.default.tag}}') - def foo2(): - ... + ```python + # Use default image name `fqn` and alter the tag to `tag-{{default.tag}}` tag of the default image + # with a prefix. In this case, it is assumed that the image like + # flytecookbook:tag-gitsha is published alongwith the default of flytecookbook:gitsha + @task(container_image='{{.images.default.fqn}}:tag-{{images.default.tag}}') + def foo(): + ... + + # Refer to configurations to configure fqns for other images besides default. In this case it will + # lookup for an image named xyz + @task(container_image='{{.images.xyz.fqn}}:{{images.default.tag}}') + def foo2(): + ... + ``` :param environment: Environment variables that should be added for this tasks execution :param requests: Specify compute resource requests for your task. For Pod-plugin tasks, these values will apply only to the primary container. :param limits: Compute limits. Specify compute resource limits for your task. For Pod-plugin tasks, these values - will apply only to the primary container. For more information, please see :py:class:`flytekit.Resources`. + will apply only to the primary container. For more information, please see {{< py_class_ref flytekit.Resources >}}. :param secret_requests: Keys that can identify the secrets supplied at runtime. Ideally the secret keys should also be semi-descriptive. The key values will be available from runtime, if the backend is configured to provide secrets and if secrets are available in the configured secrets store. Possible options for secret stores are - Vault, Confidant, Kube secrets, AWS KMS etc - Refer to :py:class:`Secret` to understand how to specify the request for a secret. It + Refer to {{< py_class_ref Secret >}} to understand how to specify the request for a secret. It may change based on the backend provider. - .. note:: - - During local execution, the secrets will be pulled from the local environment variables + > [!NOTE] + > During local execution, the secrets will be pulled from the local environment variables with the format `{GROUP}_{GROUP_VERSION}_{KEY}`, where all the characters are capitalized and the prefix is not used. @@ -322,20 +319,20 @@ def foo2(): For example this is useful to run launchplans dynamically, because launchplans must be registered on flyteadmin before they can be run. Tasks and workflows do not have this requirement. - .. code-block:: python - - @workflow - def workflow0(): - ... + ```python + @workflow + def workflow0(): + ... - launchplan0 = LaunchPlan.get_or_create(workflow0) + launchplan0 = LaunchPlan.get_or_create(workflow0) - # Specify node_dependency_hints so that launchplan0 will be registered on flyteadmin, despite this being a - # dynamic task. - @dynamic(node_dependency_hints=[launchplan0]) - def launch_dynamically(): - # To run a sub-launchplan it must have previously been registered on flyteadmin. - return [launchplan0]*10 + # Specify node_dependency_hints so that launchplan0 will be registered on flyteadmin, despite this being a + # dynamic task. + @dynamic(node_dependency_hints=[launchplan0]) + def launch_dynamically(): + # To run a sub-launchplan it must have previously been registered on flyteadmin. + return [launchplan0]*10 + ``` :param task_resolver: Provide a custom task resolver. :param disable_deck: (deprecated) If true, this task will not output deck html file :param enable_deck: If true, this task will output deck html file @@ -492,10 +489,25 @@ def reference_task( If at registration time the interface provided causes an issue with compilation, an error will be returned. Example: - + + ```python + @reference_task( + project="flytesnacks", + domain="development", + name="recipes.aaa.simple.join_strings", + version="553018f39e519bdb2597b652639c30ce16b99c79", + ) + def ref_t1(a: typing.List[str]) -> str: + ''' + The empty function acts as a convenient skeleton to make it intuitive to call/reference this task from workflows. + The interface of the task must match that of the remote task. Otherwise, remote compilation of the workflow will + fail. + ''' + return "hello" + ``` """ def wrapper(fn) -> ReferenceTask: @@ -572,75 +584,71 @@ def eager( This type of task will execute all Flyte entities within it eagerly, meaning that all python constructs can be used inside of an ``@eager``-decorated function. This is because eager workflows use a - :py:class:`~flytekit.remote.remote.FlyteRemote` object to kick off executions when a flyte entity needs to produce a + {{< py_class_ref flytekit.remote.remote.FlyteRemote >}} object to kick off executions when a flyte entity needs to produce a value. Basically think about it as: every Flyte entity that is called(), the stack frame is an execution with its own Flyte URL. Results (or the error) are fetched when the execution is finished. For example: - .. code-block:: python - - from flytekit import task, eager - - @task - def add_one(x: int) -> int: - return x + 1 + ```python + from flytekit import task, eager - @task - def double(x: int) -> int: - return x * 2 + @task + def add_one(x: int) -> int: + return x + 1 - @eager - async def eager_workflow(x: int) -> int: - out = add_one(x=x) - return double(x=out) - - # run locally with asyncio - if __name__ == "__main__": - import asyncio + @task + def double(x: int) -> int: + return x * 2 - result = asyncio.run(eager_workflow(x=1)) - print(f"Result: {result}") # "Result: 4" + @eager + async def eager_workflow(x: int) -> int: + out = add_one(x=x) + return double(x=out) - Unlike :py:func:`dynamic workflows `, eager workflows are not compiled into a workflow spec, but - uses python's `async `__ capabilities to execute flyte entities. + # run locally with asyncio + if __name__ == "__main__": + import asyncio - .. note:: + result = asyncio.run(eager_workflow(x=1)) + print(f"Result: {result}") # "Result: 4" + ``` + Unlike {{< py_func_ref dynamic workflows flytekit.dynamic >}}, eager workflows are not compiled into a workflow spec, but + uses python's [`async`](https://docs.python.org/3/library/asyncio.html) capabilities to execute flyte entities. - Eager workflows only support `@task`, `@workflow`, and `@eager` entities. Conditionals are not supported, use a + > [!NOTE] + > Eager workflows only support `@task`, `@workflow`, and `@eager` entities. Conditionals are not supported, use a plain Python if statement instead. - .. important:: - - A ``client_secret_group`` and ``client_secret_key`` is needed for authenticating via - :py:class:`~flytekit.remote.remote.FlyteRemote` using the ``client_credentials`` authentication, which is - configured via :py:class:`~flytekit.configuration.PlatformConfig`. - - .. code-block:: python + > [!IMPORTANT] + > A ``client_secret_group`` and ``client_secret_key`` is needed for authenticating via + {{< py_class_ref flytekit.remote.remote.FlyteRemote >}} using the ``client_credentials`` authentication, which is + configured via {{< py_class_ref flytekit.configuration.PlatformConfig >}}. - from flytekit.remote import FlyteRemote - from flytekit.configuration import Config - - @eager( - remote=FlyteRemote(config=Config.auto(config_file="config.yaml")), - client_secret_group="my_client_secret_group", - client_secret_key="my_client_secret_key", - ) - async def eager_workflow(x: int) -> int: - out = await add_one(x) - return await double(one) + ```python + from flytekit.remote import FlyteRemote + from flytekit.configuration import Config + @eager( + remote=FlyteRemote(config=Config.auto(config_file="config.yaml")), + client_secret_group="my_client_secret_group", + client_secret_key="my_client_secret_key", + ) + async def eager_workflow(x: int) -> int: + out = await add_one(x) + return await double(one) + ``` Where ``config.yaml`` contains is a flytectl-compatible config file. - For more details, see `here `__. + For more details, see [`here`](https://docs.flyte.org/en/latest/flytectl/overview.html#configuration). When using a sandbox cluster started with ``flytectl demo start``, however, the ``client_secret_group`` and ``client_secret_key`` are not needed, : - .. code-block:: python - - @eager - async def eager_workflow(x: int) -> int: - ... + ```python + @eager + async def eager_workflow(x: int) -> int: + ... + ``` """ if _fn is None: diff --git a/flytekit/core/testing.py b/flytekit/core/testing.py index 4eabfaddd6..505cf6d99b 100644 --- a/flytekit/core/testing.py +++ b/flytekit/core/testing.py @@ -20,16 +20,16 @@ def task_mock(t: PythonTask) -> typing.Generator[MagicMock, None, None]: Usage: - .. code-block:: python - - @task - def t1(i: int) -> int: - pass - - with task_mock(t1) as m: - m.side_effect = lambda x: x - t1(10) - # The mock is valid only within this context + ```python + @task + def t1(i: int) -> int: + pass + + with task_mock(t1) as m: + m.side_effect = lambda x: x + t1(10) + # The mock is valid only within this context + ``` """ if not isinstance(t, PythonTask) and not isinstance(t, WorkflowBase) and not isinstance(t, ReferenceEntity): diff --git a/flytekit/core/tracker.py b/flytekit/core/tracker.py index 8ead705cd4..ac718e7153 100644 --- a/flytekit/core/tracker.py +++ b/flytekit/core/tracker.py @@ -24,7 +24,7 @@ def import_module_from_file(module_name, file): class InstanceTrackingMeta(type): """ - Please see the original class :py:class`flytekit.common.mixins.registerable._InstanceTracker` also and also look + Please see the original class :flytekit.common.mixins.registerable._InstanceTracker` also and also look at the tests in the ``tests/flytekit/unit/core/tracker/test_tracking/`` folder to see how it's used. Basically, this will make instances of classes that use this metaclass aware of the module (the .py file) that @@ -92,8 +92,8 @@ class TrackedInstance(metaclass=InstanceTrackingMeta): This functionality has two use-cases currently, * Keep track of naming for non-function ``PythonAutoContainerTasks``. That is, things like the - :py:class:`flytekit.extras.sqlite3.task.SQLite3Task` task. - * Task resolvers, because task resolvers are instances of :py:class:`flytekit.core.python_auto_container.TaskResolverMixin` + {{< py_class_ref flytekit.extras.sqlite3.task.SQLite3Task >}} task. + * Task resolvers, because task resolvers are instances of {{< py_class_ref flytekit.core.python_auto_container.TaskResolverMixin >}} classes, not the classes themselves, which means we need to look on the left hand side of them to see how to find them at task execution time. """ @@ -178,12 +178,12 @@ def isnested(func: Callable) -> bool: This would essentially be any function with a `..` (defined within a function) e.g. - .. code:: python - - def foo(): - def foo_inner(): - pass + ```python + def foo(): + def foo_inner(): pass + pass + ``` In the above example `foo_inner` is the local function or a nested function. """ @@ -194,33 +194,33 @@ def foo_inner(): def is_functools_wrapped_module_level(func: Callable) -> bool: """Returns true if the function is a functools.wraps-updated function that is defined in the module-level scope. - .. code:: python - - import functools + ```python + import functools - def decorator(fn): - @functools.wraps(fn) - def wrapper(*args, **kwargs): - return fn(*arks, **kwargs) + def decorator(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + return fn(*arks, **kwargs) - return wrapper + return wrapper - @decorator - def foo(): - ... + @decorator + def foo(): + ... - def define_inner_wrapped_fn(): + def define_inner_wrapped_fn(): - @decorator - def foo_inner(*args, **kwargs): - return fn(*arks, **kwargs) + @decorator + def foo_inner(*args, **kwargs): + return fn(*arks, **kwargs) - return foo_inner + return foo_inner - bar = define_inner_wrapped_fn() + bar = define_inner_wrapped_fn() - is_functools_wrapped_module_level(foo) # True - is_functools_wrapped_module_level(bar) # False + is_functools_wrapped_module_level(foo) # True + is_functools_wrapped_module_level(bar) # False + ``` In this case, applying this function to ``foo`` returns true because ``foo`` was defined in the module-level scope. Applying this function to ``bar`` returns false because it's being assigned to ``foo_inner``, which is a diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index c53884eced..f7aaff662d 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -452,56 +452,55 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: class DataclassTransformer(TypeTransformer[object]): """ - The Dataclass Transformer provides a type transformer for dataclasses. + The Dataclass Transformer provides a type transformer for dataclasses. - The dataclass is converted to and from MessagePack Bytes by the mashumaro library - and is transported between tasks using the Binary IDL representation. - Also, the type declaration will try to extract the JSON Schema for the - object, if possible, and pass it with the definition. + The dataclass is converted to and from MessagePack Bytes by the mashumaro library + and is transported between tasks using the Binary IDL representation. + Also, the type declaration will try to extract the JSON Schema for the + object, if possible, and pass it with the definition. - The lifecycle of the dataclass in the Flyte type system is as follows: + The lifecycle of the dataclass in the Flyte type system is as follows: - 1. Serialization: The dataclass transformer converts the dataclass to MessagePack Bytes. - (1) Handle dataclass attributes to make them serializable with mashumaro. - (2) Use the mashumaro API to serialize the dataclass to MessagePack Bytes. - (3) Use MessagePack Bytes to create a Flyte Literal. - (4) Serialize the Flyte Literal to a Binary IDL Object. + 1. Serialization: The dataclass transformer converts the dataclass to MessagePack Bytes. + (1) Handle dataclass attributes to make them serializable with mashumaro. + (2) Use the mashumaro API to serialize the dataclass to MessagePack Bytes. + (3) Use MessagePack Bytes to create a Flyte Literal. + (4) Serialize the Flyte Literal to a Binary IDL Object. - 2. Deserialization: The dataclass transformer converts the MessagePack Bytes back to a dataclass. - (1) Convert MessagePack Bytes to a dataclass using mashumaro. - (2) Handle dataclass attributes to ensure they are of the correct types. + 2. Deserialization: The dataclass transformer converts the MessagePack Bytes back to a dataclass. + (1) Convert MessagePack Bytes to a dataclass using mashumaro. + (2) Handle dataclass attributes to ensure they are of the correct types. - For Json Schema, we use https://github.com/fuhrysteve/marshmallow-jsonschema library. + For Json Schema, we use https://github.com/fuhrysteve/marshmallow-jsonschema library. - Example - - .. code-block:: python + Example + ```python @dataclass class Test(DataClassJsonMixin): - a: int - b: str + a: int + b: str from marshmallow_jsonschema import JSONSchema t = Test(a=10,b="e") JSONSchema().dump(t.schema()) + ``` - Output will look like - - .. code-block:: json + Output will look like + ```python {'$schema': 'http://json-schema.org/draft-07/schema#', - 'definitions': {'TestSchema': {'properties': {'a': {'title': 'a', - 'type': 'number', - 'format': 'integer'}, + 'definitions': {'TestSchema': {'properties': {'a': {'title': 'a', + 'type': 'number', + 'format': 'integer'}, 'b': {'title': 'b', 'type': 'string'}}, - 'type': 'object', - 'additionalProperties': False}}, - '$ref': '#/definitions/TestSchema'} - - .. note:: + 'type': 'object', + 'additionalProperties': False}}, + '$ref': '#/definitions/TestSchema'} + ``` - The schema support is experimental and is useful for auto-completing in the UI/CLI + > [!NOTE] + > The schema support is experimental and is useful for auto-completing in the UI/CLI """ diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 3422a5bade..5feffbf998 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -403,27 +403,63 @@ class ImperativeWorkflow(WorkflowBase): Assuming you have some tasks like so + + ```python + @task + def t1(a: str) -> str: + return a + " world" + + @task + def t2(): + print("side effect") + ``` You could create a workflow imperatively like so + + ```python + # Create the workflow with a name. This needs to be unique within the project and takes the place of the function + # name that's used for regular decorated function-based workflows. + wb = Workflow(name="my_workflow") + # Adds a top level input to the workflow. This is like an input to a workflow function. + wb.add_workflow_input("in1", str) + # Call your tasks. + node = wb.add_entity(t1, a=wb.inputs["in1"]) + wb.add_entity(t2) + # This is analogous to a return statement + wb.add_workflow_output("from_n0t1", node.outputs["o0"]) + ``` This workflow would be identical on the back-end to + + ```python + nt = typing.NamedTuple("wf_output", [("from_n0t1", str)]) + + @workflow + def my_workflow(in1: str) -> nt: + x = t1(a=in1) + t2() + return nt(x) + ``` Note that the only reason we need the ``NamedTuple`` is so we can name the output the same thing as in the imperative example. The imperative paradigm makes the naming of workflow outputs easier, but this isn't a big @@ -702,7 +738,7 @@ class PythonFunctionWorkflow(WorkflowBase, ClassStorageTaskResolver): """ Please read :std:ref:`flyte:divedeep-workflows` first for a high-level understanding of what workflows are in Flyte. This Python object represents a workflow defined by a function and decorated with the - :py:func:`@workflow ` decorator. Please see notes on that object for additional information. + {{< py_func_ref `@workflow ` >}} decorator. Please see notes on that object for additional information. """ def __init__( @@ -940,8 +976,528 @@ def workflow( Example: + + + ```python + import os + import sys + import typing + from collections import OrderedDict + from unittest.mock import patch + + import pytest + from typing_extensions import Annotated # type: ignore + + import flytekit.configuration + from flytekit import FlyteContextManager, StructuredDataset, kwtypes + from flytekit.configuration import Image, ImageConfig + from flytekit.core import context_manager + from flytekit.core.condition import conditional + from flytekit.core.task import task + from flytekit.core.workflow import WorkflowFailurePolicy, WorkflowMetadata, WorkflowMetadataDefaults, workflow + from flytekit.exceptions.user import FlyteValidationException, FlyteValueException, FlyteMissingReturnValueException + from flytekit.tools.translator import get_serializable + from flytekit.types.error.error import FlyteError + + default_img = Image(name="default", fqn="test", tag="tag") + serialization_settings = flytekit.configuration.SerializationSettings( + project="project", + domain="domain", + version="version", + env=None, + image_config=ImageConfig(default_image=default_img, images=[default_img]), + ) + + def test_metadata_values(): + with pytest.raises(FlyteValidationException): + WorkflowMetadata(on_failure=0) + + wm = WorkflowMetadata(on_failure=WorkflowFailurePolicy.FAIL_IMMEDIATELY) + assert wm.on_failure == WorkflowFailurePolicy.FAIL_IMMEDIATELY + + + def test_default_metadata_values(): + with pytest.raises(FlyteValidationException): + WorkflowMetadataDefaults(3) + + wm = WorkflowMetadataDefaults(interruptible=False) + assert wm.interruptible is False + + + def test_workflow_values(): + @task + def t1(a: int) -> typing.NamedTuple("OutputsBC", [("t1_int_output", int), ("c", str)]): + a = a + 2 + return a, "world-" + str(a) + + @workflow(interruptible=True, failure_policy=WorkflowFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE) + def wf(a: int) -> typing.Tuple[str, str]: + x, y = t1(a=a) + _, v = t1(a=x) + return y, v + + wf_spec = get_serializable(OrderedDict(), serialization_settings, wf) + assert wf_spec.template.metadata_defaults.interruptible + assert wf_spec.template.metadata.on_failure == 1 + + def test_default_values(): + @task + def t() -> bool: + return True + + @task + def f() -> bool: + return False + + @workflow + def wf(a: bool = True) -> bool: + return conditional("bool").if_(a.is_true()).then(t()).else_().then(f()) # type: ignore + + assert wf() is True + assert wf(a=False) is False + + + def test_list_output_wf(): + @task + def t1(a: int) -> int: + a = a + 5 + return a + + @workflow + def list_output_wf() -> typing.List[int]: + v = [] + for i in range(2): + v.append(t1(a=i)) + return v + + x = list_output_wf() + assert x == [5, 6] + + + def test_sub_wf_single_named_tuple(): + nt = typing.NamedTuple("SingleNamedOutput", [("named1", int)]) + + @task + def t1(a: int) -> nt: + a = a + 2 + return nt(a) + + @workflow + def subwf(a: int) -> nt: + return t1(a=a) + + @workflow + def wf(b: int) -> nt: + out = subwf(a=b) + return t1(a=out.named1) + + x = wf(b=3) + assert x == (7,) + + + def test_sub_wf_multi_named_tuple(): + nt = typing.NamedTuple("Multi", [("named1", int), ("named2", int)]) + + @task + def t1(a: int) -> nt: + a = a + 2 + return nt(a, a) + + @workflow + def subwf(a: int) -> nt: + return t1(a=a) + + @workflow + def wf(b: int) -> nt: + out = subwf(a=b) + return t1(a=out.named1) + + x = wf(b=3) + assert x == (7, 7) + + + def test_sub_wf_varying_types(): + @task + def t1l( + a: typing.List[typing.Dict[str, typing.List[int]]], + b: typing.Dict[str, typing.List[int]], + c: typing.Union[typing.List[typing.Dict[str, typing.List[int]]], typing.Dict[str, typing.List[int]], int], + d: int, + ) -> str: + xx = ",".join([f"{k}:{v}" for d in a for k, v in d.items()]) + yy = ",".join([f"{k}: {i}" for k, v in b.items() for i in v]) + if isinstance(c, list): + zz = ",".join([f"{k}:{v}" for d in c for k, v in d.items()]) + elif isinstance(c, dict): + zz = ",".join([f"{k}: {i}" for k, v in c.items() for i in v]) + else: + zz = str(c) + return f"First: {xx} Second: {yy} Third: {zz} Int: {d}" + + @task + def get_int() -> int: + return 1 + + @workflow + def subwf( + a: typing.List[typing.Dict[str, typing.List[int]]], + b: typing.Dict[str, typing.List[int]], + c: typing.Union[typing.List[typing.Dict[str, typing.List[int]]], typing.Dict[str, typing.List[int]]], + d: int, + ) -> str: + return t1l(a=a, b=b, c=c, d=d) + + @workflow + def wf() -> str: + ds = [ + {"first_map_a": [42], "first_map_b": [get_int(), 2]}, + { + "second_map_c": [33], + "second_map_d": [9, 99], + }, + ] + ll = { + "ll_1": [get_int(), get_int(), get_int()], + "ll_2": [4, 5, 6], + } + out = subwf(a=ds, b=ll, c=ds, d=get_int()) + return out + + wf.compile() + x = wf() + expected = ( + "First: first_map_a:[42],first_map_b:[1, 2],second_map_c:[33],second_map_d:[9, 99] " + "Second: ll_1: 1,ll_1: 1,ll_1: 1,ll_2: 4,ll_2: 5,ll_2: 6 " + "Third: first_map_a:[42],first_map_b:[1, 2],second_map_c:[33],second_map_d:[9, 99] " + "Int: 1" + ) + assert x == expected + wf_spec = get_serializable(OrderedDict(), serialization_settings, wf) + assert set(wf_spec.template.nodes[5].upstream_node_ids) == {"n2", "n1", "n0", "n4", "n3"} + + @workflow + def wf() -> str: + ds = [ + {"first_map_a": [42], "first_map_b": [get_int(), 2]}, + { + "second_map_c": [33], + "second_map_d": [9, 99], + }, + ] + ll = { + "ll_1": [get_int(), get_int(), get_int()], + "ll_2": [4, 5, 6], + } + out = subwf(a=ds, b=ll, c=ll, d=get_int()) + return out + + x = wf() + expected = ( + "First: first_map_a:[42],first_map_b:[1, 2],second_map_c:[33],second_map_d:[9, 99] " + "Second: ll_1: 1,ll_1: 1,ll_1: 1,ll_2: 4,ll_2: 5,ll_2: 6 " + "Third: ll_1: 1,ll_1: 1,ll_1: 1,ll_2: 4,ll_2: 5,ll_2: 6 " + "Int: 1" + ) + assert x == expected + + + def test_unexpected_outputs(): + @task + def t1(a: int) -> int: + a = a + 5 + return a + + @workflow + def no_outputs_wf(): + return t1(a=3) + + # Should raise an exception because the workflow returns something when it shouldn't + with pytest.raises(FlyteValueException): + no_outputs_wf() + + @pytest.mark.skipif(sys.version_info < (3, 10, 10), reason="inspect module does not work correctly with Python <3.10.10. https://github.com/python/cpython/issues/102647#issuecomment-1466868212") + def test_missing_return_value(): + @task + def t1(a: int) -> int: + a = a + 5 + return a + + # Should raise an exception because it doesn't return something when it should + with pytest.raises(FlyteMissingReturnValueException): + + @workflow + def one_output_wf() -> int: # type: ignore + t1(a=3) + + one_output_wf() + + + def test_custom_wrapper(): + def our_task( + _task_function: typing.Optional[typing.Callable] = None, + **kwargs, + ): + def wrapped(_func: typing.Callable): + return task(_task_function=_func) + + if _task_function: + return wrapped(_task_function) + else: + return wrapped + + @our_task( + foo={ + "bar1": lambda x: print(x), + "bar2": lambda x: print(x), + }, + ) + def missing_func_body() -> str: + return "foo" + + + def test_wf_no_output(): + @task + def t1(a: int) -> int: + a = a + 5 + return a + + @workflow + def no_outputs_wf(): + t1(a=3) + + assert no_outputs_wf() is None + + + def test_wf_nested_comp(exec_prefix): + @task + def t1(a: int) -> int: + a = a + 5 + return a + + @workflow + def outer() -> typing.Tuple[int, int]: + # You should not do this. This is just here for testing. + @workflow + def wf2() -> int: + return t1(a=5) + + return t1(a=3), wf2() + + assert (8, 10) == outer() + entity_mapping = OrderedDict() + + model_wf = get_serializable(entity_mapping, serialization_settings, outer) + + assert len(model_wf.template.interface.outputs) == 2 + assert len(model_wf.template.nodes) == 2 + assert model_wf.template.nodes[1].workflow_node is not None + + sub_wf = model_wf.sub_workflows[0] + assert len(sub_wf.nodes) == 1 + assert sub_wf.nodes[0].id == "n0" + assert sub_wf.nodes[0].task_node.reference_id.name == f"{exec_prefix}tests.flytekit.unit.core.test_workflows.t1" + + + @task + def add_5(a: int) -> int: + a = a + 5 + return a + + + @workflow + def simple_wf() -> int: + return add_5(a=1) + + @workflow + def my_wf_example(a: int) -> typing.Tuple[int, int]: + '''example + + Workflows can have inputs and return outputs of simple or complex types. + + ''' + + x = add_5(a=a) + + # You can use outputs of a previous task as inputs to other nodes. + z = add_5(a=x) + + # You can call other workflows from within this workflow + d = simple_wf() + + # You can add conditions that can run on primitive types and execute different branches + e = conditional("bool").if_(a == 5).then(add_5(a=d)).else_().then(add_5(a=z)) + + # Outputs of the workflow have to be outputs returned by prior nodes. + # No outputs and single or multiple outputs are supported + return x, e + + def test_workflow_lhs(): + assert my_wf_example._lhs == "my_wf_example" + + + def test_all_node_types(): + assert my_wf_example(a=1) == (6, 16) + entity_mapping = OrderedDict() + + model_wf = get_serializable(entity_mapping, serialization_settings, my_wf_example) + + assert len(model_wf.template.interface.outputs) == 2 + assert len(model_wf.template.nodes) == 4 + assert model_wf.template.nodes[2].workflow_node is not None + + sub_wf = model_wf.sub_workflows[0] + assert len(sub_wf.nodes) == 1 + assert sub_wf.nodes[0].id == "n0" + assert sub_wf.nodes[0].task_node.reference_id.name == "tests.flytekit.unit.core.test_workflows.add_5" + + + def test_wf_docstring(): + model_wf = get_serializable(OrderedDict(), serialization_settings, my_wf_example) + + assert len(model_wf.template.interface.outputs) == 2 + assert model_wf.template.interface.outputs["o0"].description == "outputs" + assert model_wf.template.interface.outputs["o1"].description == "outputs" + assert len(model_wf.template.interface.inputs) == 1 + assert model_wf.template.interface.inputs["a"].description == "input a" + + + @pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.") + def test_structured_dataset_wf(): + import pandas as pd + from pandas.testing import assert_frame_equal + + from flytekit.types.schema import FlyteSchema + + superset_cols = kwtypes(Name=str, Age=int, Height=int) + subset_cols = kwtypes(Name=str) + superset_df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22], "Height": [160, 178]}) + subset_df = pd.DataFrame({"Name": ["Tom", "Joseph"]}) + + @task + def t1() -> Annotated[pd.DataFrame, superset_cols]: + return superset_df + + @task + def t2(df: Annotated[pd.DataFrame, subset_cols]) -> Annotated[pd.DataFrame, subset_cols]: + return df + + @task + def t3(df: FlyteSchema[superset_cols]) -> FlyteSchema[superset_cols]: + return df + + @task + def t4() -> FlyteSchema[superset_cols]: + return superset_df + + @task + def t5(sd: Annotated[StructuredDataset, subset_cols]) -> Annotated[pd.DataFrame, subset_cols]: + return sd.open(pd.DataFrame).all() + + @workflow + def sd_wf() -> Annotated[pd.DataFrame, subset_cols]: + # StructuredDataset -> StructuredDataset + df = t1() + return t2(df=df) + + @workflow + def sd_to_schema_wf() -> pd.DataFrame: + # StructuredDataset -> schema + df = t1() + return t3(df=df) + + @workflow + def schema_to_sd_wf() -> typing.Tuple[pd.DataFrame, pd.DataFrame]: + # schema -> StructuredDataset + df = t4() + return t2(df=df), t5(sd=df) # type: ignore + + assert_frame_equal(sd_wf(), subset_df) + assert_frame_equal(sd_to_schema_wf(), superset_df) + assert_frame_equal(schema_to_sd_wf()[0], subset_df) + assert_frame_equal(schema_to_sd_wf()[1], subset_df) + + + @pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.") + def test_compile_wf_at_compile_time(): + import pandas as pd + + from flytekit.types.schema import FlyteSchema + + superset_cols = kwtypes(Name=str, Age=int, Height=int) + superset_df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22], "Height": [160, 178]}) + + ctx = FlyteContextManager.current_context() + with FlyteContextManager.with_context( + ctx.with_execution_state( + ctx.new_execution_state().with_params(mode=context_manager.ExecutionState.Mode.TASK_EXECUTION) + ) + ): + + @task + def t4() -> FlyteSchema[superset_cols]: + return superset_df + + @workflow + def wf(): + t4() + + assert ctx.compilation_state is None + + + @pytest.mark.parametrize( + "error_message", [ + "Fail!", + None, + "", + ("big", "boom!") + ] + ) + @patch("builtins.print") + def test_failure_node_local_execution(mock_print, error_message, exec_prefix): + @task + def clean_up(name: str, err: typing.Optional[FlyteError] = None): + print(f"Deleting cluster {name} due to {err}") + print("This is err:", str(err)) + + @task + def create_cluster(name: str): + print(f"Creating cluster: {name}") + + @task + def delete_cluster(name: str, err: typing.Optional[FlyteError] = None): + print(f"Deleting cluster {name}") + print(err) + + @task + def t1(a: int, b: str): + print(f"{a} {b}") + raise ValueError(error_message) + + @workflow(on_failure=clean_up) + def wf(name: str = "flyteorg"): + c = create_cluster(name=name) + t = t1(a=1, b="2") + d = delete_cluster(name=name) + c >> t >> d + + with pytest.raises(ValueError): + wf() + + # Adjusted the error message to match the one in the failure + expected_error_message = str( + FlyteError(message=f"Error encountered while executing '{exec_prefix}tests.flytekit.unit.core.test_workflows.t1':\n {error_message}", failed_node_id="fn0") + ) + + assert mock_print.call_count > 0 + + mock_print.assert_any_call("Creating cluster: flyteorg") + mock_print.assert_any_call("1 2") + mock_print.assert_any_call(f"Deleting cluster flyteorg due to {expected_error_message}") + mock_print.assert_any_call("This is err:", expected_error_message) + ``` + Again, users should keep in mind that even though the body of the function looks like regular Python, it is actually not. When flytekit scans the workflow function, the objects being passed around between the tasks are not @@ -1012,8 +1568,16 @@ def reference_workflow( Example: + + ```python + @reference_workflow(project="proj", domain="development", name="wf_name", version="abc") + def ref_wf1(a: int) -> typing.Tuple[str, str]: + ... + return "hello", "world" + ``` """ def wrapper(fn) -> ReferenceWorkflow: diff --git a/flytekit/deck/__init__.py b/flytekit/deck/__init__.py index 58da56cf64..953ca7d4a2 100644 --- a/flytekit/deck/__init__.py +++ b/flytekit/deck/__init__.py @@ -1,16 +1,11 @@ """ -========== -Flyte Deck -========== - -.. currentmodule:: flytekit.deck +# Flyte Deck Contains deck renderers provided by flytekit. -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ + +# This module provides the following classes: + Deck TopFrameRenderer diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index c8f9fd6644..5df4b2fdc3 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -41,26 +41,26 @@ class Deck: scatter plots or Markdown text. In addition, users can create new decks to render their data with custom renderers. - .. code-block:: python - - iris_df = px.data.iris() - - @task() - def t1() -> str: - md_text = '#Hello Flyte##Hello Flyte###Hello Flyte' - m = MarkdownRenderer() - s = BoxRenderer("sepal_length") - deck = flytekit.Deck("demo", s.to_html(iris_df)) - deck.append(m.to_html(md_text)) - default_deck = flytekit.current_context().default_deck - default_deck.append(m.to_html(md_text)) - return md_text - - - # Use Annotated to override default renderer - @task() - def t2() -> Annotated[pd.DataFrame, TopFrameRenderer(10)]: - return iris_df + ```python + iris_df = px.data.iris() + + @task() + def t1() -> str: + md_text = '#Hello Flyte##Hello Flyte###Hello Flyte' + m = MarkdownRenderer() + s = BoxRenderer("sepal_length") + deck = flytekit.Deck("demo", s.to_html(iris_df)) + deck.append(m.to_html(md_text)) + default_deck = flytekit.current_context().default_deck + default_deck.append(m.to_html(md_text)) + return md_text + + + # Use Annotated to override default renderer + @task() + def t2() -> Annotated[pd.DataFrame, TopFrameRenderer(10)]: + return iris_df + ``` """ def __init__(self, name: str, html: Optional[str] = "", auto_add_to_deck: bool = True): diff --git a/flytekit/exceptions/eager.py b/flytekit/exceptions/eager.py index 806c66884d..39df2f52af 100644 --- a/flytekit/exceptions/eager.py +++ b/flytekit/exceptions/eager.py @@ -1,31 +1,31 @@ class EagerException(Exception): """Raised when a node in an eager workflow encounters an error. - This exception should be used in an :py:func:`@eager ` workflow function to + This exception should be used in an {{< py_func_ref `@eager ` >}} workflow function to catch exceptions that are raised by tasks or subworkflows. - .. code-block:: python + ```python + from flytekit import task + from flytekit.exceptions.eager import EagerException - from flytekit import task - from flytekit.exceptions.eager import EagerException + @task + def add_one(x: int) -> int: + if x < 0: + raise ValueError("x must be positive") + return x + 1 - @task - def add_one(x: int) -> int: - if x < 0: - raise ValueError("x must be positive") - return x + 1 + @task + def double(x: int) -> int: + return x * 2 - @task - def double(x: int) -> int: - return x * 2 - - @eager - async def eager_workflow(x: int) -> int: - try: - out = await add_one(x=x) - except EagerException: - # The ValueError error is caught - # and raised as an EagerException - raise - return await double(x=out) + @eager + async def eager_workflow(x: int) -> int: + try: + out = await add_one(x=x) + except EagerException: + # The ValueError error is caught + # and raised as an EagerException + raise + return await double(x=out) + ``` """ diff --git a/flytekit/extend/__init__.py b/flytekit/extend/__init__.py index 73ac51e0ab..0dd96985ce 100644 --- a/flytekit/extend/__init__.py +++ b/flytekit/extend/__init__.py @@ -1,33 +1,10 @@ """ -================== -Extending Flytekit -================== -.. currentmodule:: flytekit.extend +## Extending Flytekit -This package contains things that are useful when extending Flytekit. -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ +This package contains things that are useful when extending Flytekit. - get_serializable - context_manager - IgnoreOutputs - ExecutionState - Image - ImageConfig - Interface - Promise - TaskPlugins - DictTransformer - T - TypeEngine - TypeTransformer - PythonCustomizedContainerTask - ExecutableTemplateShimTask - ShimTaskExecutor """ from flytekit.configuration import Image, ImageConfig, SerializationSettings diff --git a/flytekit/extras/accelerators.py b/flytekit/extras/accelerators.py index 624fd8192a..62b638e0ff 100644 --- a/flytekit/extras/accelerators.py +++ b/flytekit/extras/accelerators.py @@ -1,8 +1,7 @@ """ -Specifying Accelerators -========================== +## Specifying Accelerators -.. tags:: MachineLearning, Advanced, Hardware +tags: MachineLearning, Advanced, Hardware Flyte allows you to specify `gpu` resources for a given task. However, in some cases, you may want to use a different accelerator type, such as TPU, specific variations of GPUs, or fractional GPUs. You can configure the Flyte backend to @@ -12,85 +11,53 @@ If you want to use a specific GPU device, you can pass the device name directly to the task decorator, e.g.: -.. code-block:: +```python +@task( + limits=Resources(gpu="1"), + accelerator=GPUAccelerator("nvidia-tesla-v100"), +) +def my_task() -> None: + ... +``` - @task( - limits=Resources(gpu="1"), - accelerator=GPUAccelerator("nvidia-tesla-v100"), - ) - def my_task() -> None: - ... +### Base Classes - -Base Classes ------------- These classes can be used to create custom accelerator type constants. For example, you can create a TPU accelerator. - - -.. currentmodule:: flytekit.extras.accelerators - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - BaseAccelerator - GPUAccelerator - MultiInstanceGPUAccelerator - But, often, you may want to use a well known accelerator type, and to simplify this, flytekit provides a set of predefined accelerator constants, as described in the next section. -Predefined Accelerator Constants --------------------------------- +### Predefined Accelerator Constants + The `flytekit.extras.accelerators` module provides some constants for known accelerators, listed below, but this is not a complete list. If you know the name of the accelerator, you can pass the string name to the task decorator directly. If using the constants, you can import them directly from the module, e.g.: -.. code-block:: - - from flytekit.extras.accelerators import T4 - - @task( - limits=Resources(gpu="1"), - accelerator=T4, - ) - def my_task() -> None: - ... +```python +from flytekit.extras.accelerators import T4 +@task( + limits=Resources(gpu="1"), + accelerator=T4, +) +def my_task() -> None: + ... +``` if you want to use a fractional GPU, you can use the ``partitioned`` method on the accelerator constant, e.g.: -.. code-block:: - - from flytekit.extras.accelerators import A100 - - @task( - limits=Resources(gpu="1"), - accelerator=A100.partition_2g_10gb, - ) - def my_task() -> None: - ... - -.. currentmodule:: flytekit.extras.accelerators - -.. autosummary:: - :toctree: generated/ - :nosignatures: +```python +from flytekit.extras.accelerators import A100 - A10G - L4 - K80 - M60 - P4 - P100 - T4 - V100 - A100 - A100_80GB +@task( + limits=Resources(gpu="1"), + accelerator=A100.partition_2g_10gb, +) +def my_task() -> None: + ... +``` """ @@ -128,39 +95,39 @@ def to_flyte_idl(self) -> tasks_pb2.GPUAccelerator: #: use this constant to specify that the task should run on an -#: `NVIDIA A10 Tensor Core GPU `_ +#: `NVIDIA A10 Tensor Core GPU https://www.nvidia.com/en-us/data-center/products/a10-gpu`_ A10G = GPUAccelerator("nvidia-a10g") #: use this constant to specify that the task should run on an -#: `NVIDIA L4 Tensor Core GPU `_ +#: `NVIDIA L4 Tensor Core GPU https://www.nvidia.com/en-us/data-center/l4 L4 = GPUAccelerator("nvidia-l4") #: use this constant to specify that the task should run on an -#: `NVIDIA L4 Tensor Core GPU `_ +#: `NVIDIA L4 Tensor Core GPU https://www.nvidia.com/en-us/data-center/l4 L4_VWS = GPUAccelerator("nvidia-l4-vws") #: use this constant to specify that the task should run on an -#: `NVIDIA Tesla K80 GPU `_ +#: `NVIDIA Tesla K80 GPU https://www.nvidia.com/en-gb/data-center/tesla-k80 K80 = GPUAccelerator("nvidia-tesla-k80") #: use this constant to specify that the task should run on an -#: `NVIDIA Tesla M60 GPU `_ +#: `NVIDIA Tesla M60 GPU https://images.nvidia.com/content/tesla/pdf/188417-Tesla-M60-DS-A4-fnl-Web.pdf M60 = GPUAccelerator("nvidia-tesla-m60") #: use this constant to specify that the task should run on an -#: `NVIDIA Tesla P4 GPU `_ +#: `NVIDIA Tesla P4 GPU https://images.nvidia.com/content/pdf/tesla/184457-Tesla-P4-Datasheet-NV-Final-Letter-Web.pdf P4 = GPUAccelerator("nvidia-tesla-p4") #: use this constant to specify that the task should run on an -#: `NVIDIA Tesla P100 GPU `_ +#: `NVIDIA Tesla P100 GPU https://images.nvidia.com/content/tesla/pdf/nvidia-tesla-p100-datasheet.pdf P100 = GPUAccelerator("nvidia-tesla-p100") #: use this constant to specify that the task should run on an -#: `NVIDIA Tesla T4 GPU `_ +#: `NVIDIA Tesla T4 GPU https://www.nvidia.com/en-us/data-center/tesla-t4 T4 = GPUAccelerator("nvidia-tesla-t4") #: use this constant to specify that the task should run on an -#: `NVIDIA Tesla V100 GPU `_ +#: `NVIDIA Tesla V100 GPU https://images.nvidia.com/content/technologies/volta/pdf/tesla-volta-v100-datasheet-letter-fnl-web.pdf V100 = GPUAccelerator("nvidia-tesla-v100") @@ -204,10 +171,10 @@ class _A100_Base(MultiInstanceGPUAccelerator): class _A100(_A100_Base): """ - Class that represents an `NVIDIA A100 GPU `_. It is possible + Class that represents an `NVIDIA A100 GPU https://www.nvidia.com/en-us/data-center/a100. It is possible to specify a partition of an A100 GPU by using the provided partitions on the class. For example, to specify a 10GB partition, use ``A100.partition_2g_10gb``. - Refer to `Partitioned GPUs `_ + Refer to `Partitioned GPUs https://docs.nvidia.com/datacenter/tesla/mig-user-guide/index.html#partitioning """ partition_1g_5gb = _A100_Base.partitioned("1g.5gb") @@ -233,7 +200,7 @@ class _A100(_A100_Base): #: Use this constant to specify that the task should run on an entire -#: `NVIDIA A100 GPU `_. Fractional partitions are also available. +#: `NVIDIA A100 GPU https://www.nvidia.com/en-us/data-center/a100. Fractional partitions are also available. #: #: Use pre-defined partitions (as instance attributes). For example, to specify a 10GB partition, use #: ``A100.partition_2g_10gb``. @@ -250,7 +217,7 @@ class _A100_80GB_Base(MultiInstanceGPUAccelerator): class _A100_80GB(_A100_80GB_Base): """ - Partitions of an `NVIDIA A100 80GB GPU `_. + Partitions of an [`NVIDIA A100 80GB GPU`](https://www.nvidia.com/en-us/data-center/a100). """ partition_1g_10gb = _A100_80GB_Base.partitioned("1g.10gb") @@ -276,7 +243,7 @@ class _A100_80GB(_A100_80GB_Base): #: use this constant to specify that the task should run on an entire -#: `NVIDIA A100 80GB GPU `_. Fractional partitions are also available. +#: `NVIDIA A100 80GB GPU https://www.nvidia.com/en-us/data-center/a100. Fractional partitions are also available. #: #: Use pre-defined partitions (as instance attributes). For example, to specify a 10GB partition, use #: ``A100.partition_2g_10gb``. @@ -293,7 +260,7 @@ class _V5E_Base(MultiInstanceGPUAccelerator): class _V5E(_V5E_Base): """ - Slices of a `Google Cloud TPU v5e `_. + Slices of a [`Google Cloud TPU v5e](https://cloud.google.com/tpu/docs/v5e). """ slice_1x1 = _V5E_Base.partitioned("1x1") @@ -331,7 +298,7 @@ class _V5E(_V5E_Base): #: use this constant to specify that the task should run on V5E TPU. -#: `Google V5E Cloud TPU `_. +#: `Google V5E Cloud TPU https://cloud.google.com/tpu/docs/v5e>`_. #: #: Use pre-defined slices (as instance attributes). For example, to specify a 2x4 slice, use #: ``V5E.slice_2x4``. @@ -348,7 +315,7 @@ class _V5P_Base(MultiInstanceGPUAccelerator): class _V5P(_V5P_Base): """ - Slices of a `Google Cloud TPU v5p `_. + Slices of a [`Google Cloud TPU v5p`](https://cloud.google.com/tpu/docs/v5p). """ slice_2x2x1 = _V5P_Base.partitioned("2x2x1") @@ -408,7 +375,7 @@ class _V5P(_V5P_Base): #: Use this constant to specify that the task should run on V5P TPU. -#: `Google V5P Cloud TPU `_. +#: `Google V5P Cloud TPU https://cloud.google.com/tpu/docs/v5p. #: #: Use pre-defined slices (as instance attributes). For example, to specify a 2x4x4 slice, use #: ``V5P.slice_2x4x4``. @@ -425,7 +392,7 @@ class _V6E_Base(MultiInstanceGPUAccelerator): class _V6E(_V6E_Base): """ - Slices of a `Google Cloud TPU v6e `_. + Slices of a [`Google Cloud TPU v6e`](https://cloud.google.com/tpu/docs/v6e). """ slice_1x1 = _V6E_Base.partitioned("1x1") @@ -470,7 +437,7 @@ class _V6E(_V6E_Base): #: Use this constant to specify that the task should run on V6E TPU. -#: `Google V6E Cloud TPU `_. +#: `Google V6E Cloud TPU https://cloud.google.com/tpu/docs/v6e. #: #: Use pre-defined slices (as instance attributes). For example, to specify a 2x4 slice, use #: ``V6E.slice_2x4``. diff --git a/flytekit/extras/pytorch/__init__.py b/flytekit/extras/pytorch/__init__.py index 12c507afb9..4776b59d8f 100644 --- a/flytekit/extras/pytorch/__init__.py +++ b/flytekit/extras/pytorch/__init__.py @@ -1,14 +1,5 @@ """ -.. currentmodule:: flytekit.extras.pytorch - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - PyTorchCheckpoint - PyTorchCheckpointTransformer - PyTorchModuleTransformer - PyTorchTensorTransformer +This module provides functionality related tp PyTorch """ from flytekit.loggers import logger diff --git a/flytekit/extras/sklearn/__init__.py b/flytekit/extras/sklearn/__init__.py index d22546dbe2..9ae037003c 100644 --- a/flytekit/extras/sklearn/__init__.py +++ b/flytekit/extras/sklearn/__init__.py @@ -1,11 +1,5 @@ """ -.. currentmodule:: flytekit.extras.sklearn - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - SklearnEstimatorTransformer +This module provides functionality related to Scikit-learn """ from flytekit.loggers import logger diff --git a/flytekit/extras/sqlite3/__init__.py b/flytekit/extras/sqlite3/__init__.py index cc016ac4af..0ca06f13ed 100644 --- a/flytekit/extras/sqlite3/__init__.py +++ b/flytekit/extras/sqlite3/__init__.py @@ -1,12 +1,3 @@ """ -Flytekit SQLite3Task -========================================= -.. currentmodule:: flytekit.extras.sqlite3 - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - ~task.SQLite3Task - ~task.SQLite3Config +This module provides functionality related to SQLite3 """ diff --git a/flytekit/extras/sqlite3/task.py b/flytekit/extras/sqlite3/task.py index 51dcaccd61..e7c03899f0 100644 --- a/flytekit/extras/sqlite3/task.py +++ b/flytekit/extras/sqlite3/task.py @@ -57,19 +57,33 @@ class SQLite3Task(PythonCustomizedContainerTask[SQLite3Config], SQLTask[SQLite3C """ Run client side SQLite3 queries that optionally return a FlyteSchema object. - .. note:: - - This is a pre-built container task. That is, your user container will not be used at task execution time. + > [!NOTE] + > This is a pre-built container task. That is, your user container will not be used at task execution time. Instead the image defined in this task definition will be used instead. + + ```python + sql_task = SQLite3Task( + "test", + query_template="select TrackId, Name from tracks limit {{.inputs.limit}}", + inputs=kwtypes(limit=int), + output_schema_type=FlyteSchema[kwtypes(TrackId=int, Name=str)], + task_config=SQLite3Config( + uri=EXAMPLE_DB, + compressed=True, + ), + ) + ``` + See the :ref:`integrations guide ` for additional usage examples and - the base class :py:class:`flytekit.extend.PythonCustomizedContainerTask` as well. + the base class {{< py_class_ref flytekit.extend.PythonCustomizedContainerTask >}} as well. """ _SQLITE_TASK_TYPE = "sqlite" diff --git a/flytekit/extras/tasks/__init__.py b/flytekit/extras/tasks/__init__.py index 94ca9244fd..efc9ffb619 100644 --- a/flytekit/extras/tasks/__init__.py +++ b/flytekit/extras/tasks/__init__.py @@ -1,12 +1,3 @@ """ -Flytekit ShellTask -========================================= -.. currentmodule:: flytekit.extras.tasks - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - ~shell.ShellTask - ~shell.OutputLocation +This module provides functionality tasks """ diff --git a/flytekit/extras/tasks/shell.py b/flytekit/extras/tasks/shell.py index 32ae33fcc7..8da0bc154e 100644 --- a/flytekit/extras/tasks/shell.py +++ b/flytekit/extras/tasks/shell.py @@ -235,9 +235,9 @@ def __init__( task_config: Configuration for the task, can be either a Pod (or coming soon, BatchJob) config shell: Shell to use to run the script inputs: A Dictionary of input names to types - output_locs: A list of :py:class:`OutputLocations` + output_locs: A list of {{< py_class_ref OutputLocations >}} **kwargs: Other arguments that can be passed to - :py:class:`~flytekit.core.python_function_task.PythonInstanceTask` + {{< py_class_ref flytekit.core.python_function_task.PythonInstanceTask >}} """ if script and script_file: raise ValueError("Only either of script or script_file can be provided") @@ -394,12 +394,12 @@ def __init__( template. The template itself will export the desired environment variables, and subsequently execute the desired "raw" script with the specified arguments. - .. note:: - This means that within your workflow, you can dynamically control the env variables, arguments, and even the + > [!NOTE] + > This means that within your workflow, you can dynamically control the env variables, arguments, and even the actual script you want to run. - .. note:: - The downside is that a dynamic workflow will be required. The "raw" script passed in at execution time must + > [!NOTE] + > The downside is that a dynamic workflow will be required. The "raw" script passed in at execution time must be at the specified location. These args are forwarded directly to the parent `ShellTask` constructor as behavior does not diverge diff --git a/flytekit/extras/tensorflow/__init__.py b/flytekit/extras/tensorflow/__init__.py index 0c4d12f443..bc9742aa91 100644 --- a/flytekit/extras/tensorflow/__init__.py +++ b/flytekit/extras/tensorflow/__init__.py @@ -1,12 +1,5 @@ """ -.. currentmodule:: flytekit.extras.tensorflow - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - TensorFlowRecordFileTransformer - TensorFlowRecordsDirTransformer +This module provides functionality related to Tensorflow """ from flytekit.loggers import logger diff --git a/flytekit/extras/webhook/task.py b/flytekit/extras/webhook/task.py index ff11534256..d86f0b84a1 100644 --- a/flytekit/extras/webhook/task.py +++ b/flytekit/extras/webhook/task.py @@ -12,17 +12,17 @@ class WebhookTask(SyncConnectorExecutorMixin, PythonTask): """ - The WebhookTask is used to invoke a webhook. The webhook can be invoked with a POST or GET method. + The WebhookTask is used to invoke a webhook. The webhook can be invoked with a POST or GET method. - All the parameters can be formatted using python format strings. + All the parameters can be formatted using python format strings. - Example: - ```python - simple_get = WebhookTask( - name="simple-get", - url="http://localhost:8000/", - method=http.HTTPMethod.GET, - headers={"Content-Type": "application/json"}, + Example: + ```python + simple_get = WebhookTask( + name="simple-get", + url="http://localhost:8000/", + method=http.HTTPMethod.GET, + headers={"Content-Type": "application/json"}, ) get_with_params = WebhookTask( @@ -52,44 +52,47 @@ def wf(s: str) -> (dict, dict, dict): dynamic_inputs={"s": str}, ) return simple_get(), get_with_params(s=v, item_id=10), w(s=v) - ``` - - All the parameters can be formatted using python format strings. The following parameters are available for - formatting: - - dynamic_inputs: These are the dynamic inputs to the task. The keys are the names of the inputs and the values - are the values of the inputs. All inputs are available under the prefix `inputs.`. - For example, if the inputs are {"input1": 10, "input2": "hello"}, then you can - use {inputs.input1} and {inputs.input2} in the URL and the body. Define the dynamic_inputs argument in the - constructor to use these inputs. The dynamic inputs should not be actual values, but the types of the inputs. - - TODO Coming soon secrets support - - secrets: These are the secrets that are requested by the task. The keys are the names of the secrets and the - values are the values of the secrets. All secrets are available under the prefix `secrets.`. - For example, if the secret requested are Secret(name="secret1") and Secret(name="secret), then you can use - {secrets.secret1} and {secrets.secret2} in the URL and the body. Define the secret_requests argument in the - constructor to use these secrets. The secrets should not be actual values, but the types of the secrets. - - :param name: Name of this task, should be unique in the project - :param url: The endpoint or URL to invoke for this webhook. This can be a static string or a python format string, - where the format arguments are the dynamic_inputs to the task, secrets etc. Refer to the description for more + ``` + + All the parameters can be formatted using python format strings. The following parameters are available for + formatting: + - dynamic_inputs: These are the dynamic inputs to the task. The keys are the names of the inputs and the values + are the values of the inputs. All inputs are available under the prefix `inputs.`. + For example, if the inputs are {"input1": 10, "input2": "hello"}, then you can + use {inputs.input1} and {inputs.input2} in the URL and the body. Define the dynamic_inputs argument in the + constructor to use these inputs. The dynamic inputs should not be actual values, but the types of the inputs. + + TODO Coming soon secrets support + - secrets: These are the secrets that are requested by the task. The keys are the names of the secrets and the + values are the values of the secrets. All secrets are available under the prefix `secrets.`. + For example, if the secret requested are Secret(name="secret1") and Secret(name="secret), then you can use + {secrets.secret1} and {secrets.secret2} in the URL and the body. Define the secret_requests argument in the + constructor to use these secrets. The secrets should not be actual values, but the types of the secrets. + + Args: + name (str): Name of this task, should be unique in the project. + url (str): The endpoint or URL to invoke for this webhook. This can be a static string or a Python format string, + where the format arguments are the dynamic_inputs to the task, secrets, etc. Refer to the description for more details of available formatting parameters. - :param method: The HTTP method to use for the request. Default is POST. - :param headers: The headers to send with the request. This can be a static dictionary or a python format string, - where the format arguments are the dynamic_inputs to the task, secrets etc. Refer to the description for more + method (str): The HTTP method to use for the request. Default is POST. + headers (Optional[Dict[str, str]]): The headers to send with the request. This can be a static dictionary or a Python format string, + where the format arguments are the dynamic_inputs to the task, secrets, etc. Refer to the description for more details of available formatting parameters. - :param data: The body to send with the request. This can be a static dictionary or a python format string, - where the format arguments are the dynamic_inputs to the task, secrets etc. Refer to the description for more - details of available formatting parameters. the data should be a json serializable dictionary and will be - sent as the json body of the POST request and as the query parameters of the GET request. - :param dynamic_inputs: The dynamic inputs to the task. The keys are the names of the inputs and the values + data (Optional[Dict[str, Any]]): The body to send with the request. This can be a static dictionary or a Python format string, + where the format arguments are the dynamic_inputs to the task, secrets, etc. Refer to the description for more + details of available formatting parameters. The data should be a JSON-serializable dictionary and will be + sent as the JSON body of the POST request and as the query parameters of the GET request. + dynamic_inputs (Optional[Dict[str, Type]]): The dynamic inputs to the task. The keys are the names of the inputs and the values are the types of the inputs. These inputs are available under the prefix `inputs.` to be used in the URL, - headers and body and other formatted fields. - :param secret_requests: The secrets that are requested by the task. (TODO not yet supported) - :param show_data: If True, the body of the request will be logged in the UI as the output of the task. - :param show_url: If True, the URL of the request will be logged in the UI as the output of the task. - :param description: Description of the task - :param timeout: The timeout for the request (connection and read). Default is 10 seconds. If int value is provided, + headers, body, and other formatted fields. + show_data (bool): If True, the body of the request will be logged in the UI as the output of the task. + show_url (bool): If True, the URL of the request will be logged in the UI as the output of the task. + description (Optional[str]): Description of the task. + timeout (Union[int, timedelta]): The timeout for the request (connection and read). Default is 10 seconds. If an int is provided, it is considered as seconds. + + Raises: + ValueError: if the method is not 'POST' or 'GET'. """ def __init__( diff --git a/flytekit/image_spec/__init__.py b/flytekit/image_spec/__init__.py index c06c22c62b..a0499a9ce0 100644 --- a/flytekit/image_spec/__init__.py +++ b/flytekit/image_spec/__init__.py @@ -1,18 +1,5 @@ """ -========== -ImageSpec -========== - -.. currentmodule:: flytekit.image_spec - -This module contains the ImageSpec class parameters and methods. - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - ImageSpec +This module provides functionality related to IMageSPec """ from .default_builder import DefaultImageBuilder diff --git a/flytekit/interactive/__init__.py b/flytekit/interactive/__init__.py index 64393f6436..3418a5d52e 100644 --- a/flytekit/interactive/__init__.py +++ b/flytekit/interactive/__init__.py @@ -1,19 +1,5 @@ """ -.. -currentmodule:: flytekit.interactive - -This package contains flyteinteractive plugin for Flytekit. - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - vscode - VscodeConfig - DEFAULT_CODE_SERVER_DIR_NAME - DEFAULT_CODE_SERVER_REMOTE_PATH - DEFAULT_CODE_SERVER_EXTENSIONS - get_task_inputs +This module provides functionality related to Flytekit Interactive """ from .utils import get_task_inputs diff --git a/flytekit/lazy_import/lazy_module.py b/flytekit/lazy_import/lazy_module.py index 3112682d9d..f92bf11d9b 100644 --- a/flytekit/lazy_import/lazy_module.py +++ b/flytekit/lazy_import/lazy_module.py @@ -30,10 +30,11 @@ def is_imported(module_name): def lazy_module(fullname): """ This function is used to lazily import modules. It is used in the following way: - .. code-block:: python - from flytekit.lazy_import import lazy_module - sklearn = lazy_module("sklearn") - sklearn.svm.SVC() + ```python + from flytekit.lazy_import import lazy_module + sklearn = lazy_module("sklearn") + sklearn.svm.SVC() + ``` :param Text fullname: The full name of the module to import """ if fullname in sys.modules: diff --git a/flytekit/models/core/execution.py b/flytekit/models/core/execution.py index cf7d0a7e00..f24f69167d 100644 --- a/flytekit/models/core/execution.py +++ b/flytekit/models/core/execution.py @@ -8,7 +8,7 @@ class WorkflowExecutionPhase(object): """ - This class holds enum values used for setting notifications. See :py:class:`flytekit.Email` + This class holds enum values used for setting notifications. See {{< py_class_ref flytekit.Email >}} for sample usage. """ diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index a4b5a1d359..6271d13835 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -246,7 +246,7 @@ class Blob(_common.FlyteIdlEntity): def __init__(self, metadata, uri): """ This literal model is used to represent binary data offloaded to some storage location which is - identifiable with a unique string. See :py:class:`flytekit.FlyteFile` as an example. + identifiable with a unique string. See {{< py_class_ref flytekit.FlyteFile >}} as an example. :param BlobMetadata metadata: :param Text uri: The location of this blob diff --git a/flytekit/models/security.py b/flytekit/models/security.py index 9021f75496..18384293da 100644 --- a/flytekit/models/security.py +++ b/flytekit/models/security.py @@ -161,7 +161,7 @@ def from_flyte_idl(cls, pb2_object: _sec.OAuth2TokenRequest) -> "OAuth2TokenRequ class SecurityContext(_common.FlyteIdlEntity): """ This is a higher level wrapper object that for the most part users shouldn't have to worry about. You should - be able to just use :py:class:`flytekit.Secret` instead. + be able to just use {{< py_class_ref flytekit.Secret >}} instead. """ run_as: Optional[Identity] = None diff --git a/flytekit/remote/__init__.py b/flytekit/remote/__init__.py index dd92a813f2..0b2ac0c873 100644 --- a/flytekit/remote/__init__.py +++ b/flytekit/remote/__init__.py @@ -1,89 +1,48 @@ """ -===================== -Remote Access -===================== - -.. currentmodule:: flytekit.remote - -This module provides utilities for performing operations on tasks, workflows, launchplans, and executions, for example, -the following code fetches and executes a workflow: - -.. code-block:: python - - # create a remote object from flyte config and environment variables - FlyteRemote(config=Config.auto()) - FlyteRemote(config=Config.auto(config_file=....)) - FlyteRemote(config=Config(....)) - - # Or if you need to specify a custom cert chain - # (options and compression are also respected keyword arguments) - FlyteRemote(private_key=your_private_key_bytes, root_certificates=..., certificate_chain=...) - - # fetch a workflow from the flyte backend - remote = FlyteRemote(...) - flyte_workflow = remote.fetch_workflow(name="my_workflow", version="v1") - - # execute the workflow, wait=True will return the execution object after it's completed - workflow_execution = remote.execute(flyte_workflow, inputs={"a": 1, "b": 10}, wait=True) - - # inspect the execution's outputs - print(workflow_execution.outputs) - -.. _remote-entrypoint: - -Entrypoint -========== - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~remote.FlyteRemote - ~remote.Options - -.. _remote-flyte-entities: - -Entities -======== - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~entities.FlyteTask - ~entities.FlyteWorkflow - ~entities.FlyteLaunchPlan - -.. _remote-flyte-entity-components: - -Entity Components -================= - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~entities.FlyteNode - ~entities.FlyteTaskNode - ~entities.FlyteWorkflowNode - -.. _remote-flyte-execution-objects: - -Execution Objects -================= - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - :nosignatures: - - ~executions.FlyteWorkflowExecution - ~executions.FlyteTaskExecution - ~executions.FlyteNodeExecution - +# Remote Access + +This module provides utilities for performing operations on tasks, workflows, launchplans, and executions. For example, the following code fetches and executes a workflow: + +```python +# create a remote object from flyte config and environment variables +FlyteRemote(config=Config.auto()) +FlyteRemote(config=Config.auto(config_file=....)) +FlyteRemote(config=Config(....)) +``` +Or if you need to specify a custom cert chain +(options and compression are also respected keyword arguments) +```python +FlyteRemote(private_key=your_private_key_bytes, root_certificates=..., certificate_chain=...) + + +### fetch a workflow from the flyte backend +remote = FlyteRemote(...) +flyte_workflow = remote.fetch_workflow(name="my_workflow", version="v1") + +# execute the workflow, wait=True will return the execution object after it's completed +workflow_execution = remote.execute(flyte_workflow, inputs={"a": 1, "b": 10}, wait=True) + +# inspect the execution's outputs +print(workflow_execution.outputs) +``` + +## Entrypoint + +| Class | | Description | +|-------|-------------|-------------| +| {{< py_class_ref remote.FlyteRemote >}} | {{< py_class_docsum remote.FlyteRemote >}} | The main class for interacting with a Flyte backend. | +| {{< py_class_ref remote.Config >}} | {{< py_class_docsum remote.Config >}} | Configuration options for the FlyteRemote client. | +| {{< py_class_ref remote.Options >}} | {{< py_class_docsum remote.Options >}} | Configuration options for the FlyteRemote client. | +| {{< py_class_ref remote.FlyteTask >}} | {{< py_class_docsum remote.FlyteTask >}} | Represents a registered Flyte task. | +| {{< py_class_ref remote.FlyteWorkflow >}} | {{< py_class_docsum remote.FlyteWorkflow >}} | Represents a registered Flyte workflow. | +| {{< py_class_ref remote.FlyteLaunchPlan >}} | {{< py_class_docsum remote.FlyteLaunchPlan >}} | Represents a registered Flyte launch plan. | +| {{< py_class_ref remote.FlyteNode >}} | {{< py_class_docsum remote.FlyteNode >}} | Base class for nodes in a Flyte workflow. | +| {{< py_class_ref remote.FlyteTaskNode >}} | {{< py_class_docsum remote.FlyteTaskNode >}} | Represents a task node in a Flyte workflow. | +| {{< py_class_ref remote.FlyteWorkflowNode >}} | {{< py_class_docsum remote.FlyteWorkflowNode >}} | Represents a subworkflow node in a Flyte workflow. | +| {{< py_class_ref remote.FlyteWorkflowExecution >}} | {{< py_class_docsum remote.FlyteWorkflowExecution >}} | Represents an execution of a Flyte workflow. | +| {{< py_class_ref remote.FlyteTaskExecution >}} | {{< py_class_docsum remote.FlyteTaskExecution >}} | Represents an execution of a Flyte task. | +| {{< py_class_ref remote.FlyteNodeExecution >}} | {{< py_class_docsum remote.FlyteNodeExecution >}} | Represents an execution of a node within a workflow. | +| {{< py_class_ref remote.FlyteBranchNode >}} | {{< py_class_docsum remote.FlyteBranchNode >}} | Represents a branch node in a Flyte workflow. | """ from flytekit.remote.entities import ( diff --git a/flytekit/remote/backfill.py b/flytekit/remote/backfill.py index 166f2e4745..73052ccbd6 100644 --- a/flytekit/remote/backfill.py +++ b/flytekit/remote/backfill.py @@ -24,22 +24,23 @@ def create_backfill_workflow( the Backfill plan is generated as (start_date - exclusive, end_date inclusive) - .. code-block:: python - :caption: Correct usage for dates example - - lp = Launchplan.get_or_create(...) - start_date = datetime.datetime(2023, 1, 1) - end_date = start_date + datetime.timedelta(days=10) - wf = create_backfill_workflow(start_date, end_date, for_lp=lp) - - - .. code-block:: python - :caption: Incorrect date example - - wf = create_backfill_workflow(end_date, start_date, for_lp=lp) # end_date is before start_date - # OR - wf = create_backfill_workflow(start_date, start_date, for_lp=lp) # start and end date are same - + > [!NOTE] + > Correct usage for dates example + + ```python + lp = Launchplan.get_or_create(...) + start_date = datetime.datetime(2023, 1, 1) + end_date = start_date + datetime.timedelta(days=10) + wf = create_backfill_workflow(start_date, end_date, for_lp=lp) + ``` + > [!WARNING] + > Incorrect date example + + ```python + wf = create_backfill_workflow(end_date, start_date, for_lp=lp) # end_date is before start_date + # OR + wf = create_backfill_workflow(start_date, start_date, for_lp=lp) # start and end date are same + ``` :param start_date: datetime generate a backfill starting at this datetime (exclusive) :param end_date: datetime generate a backfill ending at this datetime (inclusive) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 6c2b065792..4d9cf34cd7 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -1714,9 +1714,8 @@ def execute( :param serialization_settings: Optionally provide serialization settings, in case the entity being run needs to first be registered. If not provided, a default will be used. - .. note: - - The ``name`` and ``version`` arguments do not apply to ``FlyteTask``, ``FlyteLaunchPlan``, and + > [!NOTE] + > The ``name`` and ``version`` arguments do not apply to ``FlyteTask``, ``FlyteLaunchPlan``, and ``FlyteWorkflow`` entity inputs. These values are determined by referencing the entity identifier values. """ if entity.python_interface: diff --git a/flytekit/testing/__init__.py b/flytekit/testing/__init__.py index ccb95e6d33..76f89c2545 100644 --- a/flytekit/testing/__init__.py +++ b/flytekit/testing/__init__.py @@ -1,20 +1,5 @@ """ -===================== -Unit Testing -===================== - -.. currentmodule:: flytekit.testing - -The imports exposed in this package will help you unit test your Flyte tasks. These are particularly helpful when -testing workflows that contain tasks that cannot run locally (a Hive task for instance). - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - patch - A decorator similar to the regular one you're probably used to - task_mock - Non-decorative function - +This module provides functionality related to testing """ from flytekit.core.context_manager import SecretsManager diff --git a/flytekit/tools/module_loader.py b/flytekit/tools/module_loader.py index 977a194fbd..c16faa548b 100644 --- a/flytekit/tools/module_loader.py +++ b/flytekit/tools/module_loader.py @@ -43,7 +43,7 @@ def just_load_modules(pkgs: List[str]): def load_object_from_module(object_location: str) -> Any: """ - # TODO: Handle corner cases, like where the first part is [] maybe + TODO: Handle corner cases, like where the first part is [] maybe """ class_obj = object_location.split(".") class_obj_mod = class_obj[:-1] # e.g. ['flytekit', 'core', 'python_auto_container'] diff --git a/flytekit/tools/repo.py b/flytekit/tools/repo.py index e2e46f49d3..d4d984a776 100644 --- a/flytekit/tools/repo.py +++ b/flytekit/tools/repo.py @@ -33,7 +33,7 @@ def serialize_load_only( local_source_root: typing.Optional[str] = None, ): """ - See :py:class:`flytekit.models.core.identifier.ResourceType` to match the trailing index in the file name with the + See {{< py_class_ref flytekit.models.core.identifier.ResourceType >}} to match the trailing index in the file name with the entity type. :param settings: SerializationSettings to be used :param pkgs: Dot-delimited Python packages/subpackages to look into for serialization. @@ -55,7 +55,7 @@ def serialize_get_control_plane_entities( is_registration: bool = False, ) -> typing.List[FlyteControlPlaneEntity]: """ - See :py:class:`flytekit.models.core.identifier.ResourceType` to match the trailing index in the file name with the + See {{< py_class_ref flytekit.models.core.identifier.ResourceType >}} to match the trailing index in the file name with the entity type. :param options: :param settings: SerializationSettings to be used diff --git a/flytekit/types/directory/__init__.py b/flytekit/types/directory/__init__.py index 83bb0c8fa8..0846bc6b17 100644 --- a/flytekit/types/directory/__init__.py +++ b/flytekit/types/directory/__init__.py @@ -1,17 +1,12 @@ """ -Flytekit Directory Type -========================================================== -.. currentmodule:: flytekit.types.directory +Similar to {{< py_class_ref flytekit.types.file.FlyteFile >}} there are some 'preformatted' directory types. -Similar to :py:class:`flytekit.types.file.FlyteFile` there are some 'preformatted' directory types. +| Class | Description | +|-------| ---- | +| {{< py_class_ref FlyteDirectory >}} | {{< py_class_docsum FlyteDirectory >}} | +| {{< py_class_ref TensorboardLogs >}} | {{< py_class_docsum TensorboardLogs >}} | +| {{< py_class_ref TFRecordsDirectory >}} | {{< py_class_docsum TFRecordsDirectory >}} | -.. autosummary:: - :toctree: generated/ - :template: file_types.rst - - FlyteDirectory - TensorboardLogs - TFRecordsDirectory """ import typing diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 699278b0b6..5f13bd35e5 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -43,13 +43,12 @@ def noop(): ... class FlyteDirectory(SerializableType, DataClassJsonMixin, os.PathLike, typing.Generic[T]): path: PathType = field(default=None, metadata=config(mm_field=fields.String())) # type: ignore """ - .. warning:: - - This class should not be used on very large datasets, as merely listing the dataset will cause + > [!WARNING] + > This class should not be used on very large datasets, as merely listing the dataset will cause the entire dataset to be downloaded. Listing on S3 and other backend object stores is not consistent and we should not need data to be downloaded to list. - Please first read through the comments on the :py:class:`flytekit.types.file.FlyteFile` class as the + Please first read through the comments on the {{< py_class_ref flytekit.types.file.FlyteFile >}} class as the implementation here is similar. One thing to note is that the ``os.PathLike`` type that comes with Python was used as a stand-in for ``FlyteFile``. @@ -345,17 +344,17 @@ def listdir(cls, directory: FlyteDirectory) -> typing.List[typing.Union[FlyteDir In addition, it will return a list of FlyteFile and FlyteDirectory objects that have ability to lazily download the contents of the file/folder. For example: - .. code-block:: python - - entity = FlyteDirectory.listdir(directory) - for e in entity: - print("s3 object:", e.remote_source) - # s3 object: s3://test-flytedir/file1.txt - # s3 object: s3://test-flytedir/file2.txt - # s3 object: s3://test-flytedir/sub_dir - - open(entity[0], "r") # This will download the file to the local disk. - open(entity[0], "r") # flytekit will read data from the local disk if you open it again. + ```python + entity = FlyteDirectory.listdir(directory) + for e in entity: + print("s3 object:", e.remote_source) + # s3 object: s3://test-flytedir/file1.txt + # s3 object: s3://test-flytedir/file2.txt + # s3 object: s3://test-flytedir/sub_dir + + open(entity[0], "r") # This will download the file to the local disk. + open(entity[0], "r") # flytekit will read data from the local disk if you open it again. + ``` """ final_path = directory.path @@ -449,9 +448,8 @@ class FlyteDirToMultipartBlobTransformer(AsyncTypeTransformer[FlyteDirectory]): This transformer handles conversion between the Python native FlyteDirectory class defined above, and the Flyte IDL literal/type of Multipart Blob. Please see the FlyteDirectory comments for additional information. - .. caution: - - The transformer will not check if the given path is actually a directory. This is because the path could be + > [!CAUTION] caution: + > The transformer will not check if the given path is actually a directory. This is because the path could be a remote reference. """ @@ -470,7 +468,8 @@ def _blob_type(format: str) -> _core_types.BlobType: def assert_type(self, t: typing.Type[FlyteDirectory], v: typing.Union[FlyteDirectory, os.PathLike, str]): if isinstance(v, FlyteDirectory) or isinstance(v, str) or isinstance(v, os.PathLike): """ - NOTE: we do not do a isdir check because the given path could be remote reference + >[!NOTE] + > we do not do a isdir check because the given path could be remote reference """ return raise TypeError( diff --git a/flytekit/types/error/__init__.py b/flytekit/types/error/__init__.py index 6714e88844..7a60832694 100644 --- a/flytekit/types/error/__init__.py +++ b/flytekit/types/error/__init__.py @@ -1,14 +1 @@ -""" -Flytekit Error Type -========================================================== -.. currentmodule:: flytekit.types.error - -.. autosummary:: - :nosignatures: - :template: custom.rst - :toctree: generated/ - - FlyteError -""" - from .error import FlyteError diff --git a/flytekit/types/file/__init__.py b/flytekit/types/file/__init__.py index 8d69247e10..e0cde777cb 100644 --- a/flytekit/types/file/__init__.py +++ b/flytekit/types/file/__init__.py @@ -1,24 +1,5 @@ """ -Flytekit File Type -========================================================== -.. currentmodule:: flytekit.types.file - -This list also contains a bunch of pre-formatted :py:class:`flytekit.types.file.FlyteFile` types. - -.. autosummary:: - :toctree: generated/ - :template: file_types.rst - - FlyteFile - HDF5EncodedFile - HTMLPage - JoblibSerializedFile - JPEGImageFile - PDFFile - PNGImageFile - PythonPickledFile - PythonNotebook - SVGImageFile +This module provides functionality related to FlyteFile """ import typing diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 4ba06c2008..a5eff0b68f 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -56,7 +56,7 @@ class FlyteFile(SerializableType, os.PathLike, typing.Generic[T], DataClassJSONM Files (and directories) differ from the primitive types like floats and string in that Flytekit typically uploads the contents of the files to the blob store connected with your Flyte installation. That is, the Python native literal that represents a file is typically just the path to the file on the local filesystem. However in Flyte, - an instance of a file is represented by a :py:class:`Blob ` literal, + an instance of a file is represented by a {{< py_class_ref Blob >}} literal, with the ``uri`` field set to the location in the Flyte blob store (AWS/GCS etc.). Take a look at the :std:ref:`data handling doc ` for a deeper discussion. @@ -74,7 +74,7 @@ class FlyteFile(SerializableType, os.PathLike, typing.Generic[T], DataClassJSONM You can also make it so that the upload does not happen. There are different types of task/workflow signatures. Keep in mind that in the backend, in Admin and in the blob store, there is only one type - that represents files, the :py:class:`Blob ` type. + that represents files, the {{< py_class_ref Blob >}} type. Whether the uploading happens or not, the behavior of the translation between Python native values and Flyte literal values depends on a few attributes: @@ -84,9 +84,9 @@ class FlyteFile(SerializableType, os.PathLike, typing.Generic[T], DataClassJSONM * :class:`python:os.PathLike` Note that ``os.PathLike`` is only a type in Python, you can't instantiate it. * The type of the Python native value we're returning. These can be - * :py:class:`flytekit.FlyteFile` - * :py:class:`pathlib.Path` - * :py:class:`str` + * {{< py_class_ref flytekit.FlyteFile >}} + * {{< py_class_ref pathlib.Path >}} + * {{< py_class_ref str >}} * Whether the value being converted is a "remote" path or not. For instance, if a task returns a value of "http://www.google.com" as a ``FlyteFile``, obviously it doesn't make sense for us to try to upload that to the Flyte blob store. So no remote paths are uploaded. Flytekit considers a path remote if it starts with ``s3://``, @@ -399,15 +399,15 @@ def open( ): """Returns a streaming File handle - .. code-block:: python - - @task - def copy_file(ff: FlyteFile) -> FlyteFile: - new_file = FlyteFile.new_remote_file() - with ff.open("rb", cache_type="readahead") as r: - with new_file.open("wb") as w: - w.write(r.read()) - return new_file + ```python + @task + def copy_file(ff: FlyteFile) -> FlyteFile: + new_file = FlyteFile.new_remote_file() + with ff.open("rb", cache_type="readahead") as r: + with new_file.open("wb") as w: + w.write(r.read()) + return new_file + ``` :param mode: Open mode. For example: 'r', 'w', 'rb', 'rt', 'wb', etc. :type mode: str diff --git a/flytekit/types/iterator/__init__.py b/flytekit/types/iterator/__init__.py index 3c1911394f..59d9b0ef20 100644 --- a/flytekit/types/iterator/__init__.py +++ b/flytekit/types/iterator/__init__.py @@ -1,16 +1,4 @@ -""" -Flytekit Iterator Type -====================== - -.. currentmodule:: flytekit.types.iterator - -.. autosummary:: - :nosignatures: - :toctree: generated/ - - FlyteIterator - JSON -""" +""" """ from .iterator import FlyteIterator from .json_iterator import JSON diff --git a/flytekit/types/pickle/__init__.py b/flytekit/types/pickle/__init__.py index 59833bdc84..5c89798a72 100644 --- a/flytekit/types/pickle/__init__.py +++ b/flytekit/types/pickle/__init__.py @@ -1,13 +1,3 @@ -""" -Flytekit Pickle Type -========================================================== -.. currentmodule:: flytekit.types.pickle - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - FlytePickle -""" +""" """ from .pickle import FlytePickle diff --git a/flytekit/types/structured/__init__.py b/flytekit/types/structured/__init__.py index 16c1812fc6..6cc10a281e 100644 --- a/flytekit/types/structured/__init__.py +++ b/flytekit/types/structured/__init__.py @@ -1,16 +1,4 @@ -""" -Flytekit StructuredDataset -========================================================== -.. currentmodule:: flytekit.types.structured - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - StructuredDataset - StructuredDatasetDecoder - StructuredDatasetEncoder -""" +""" """ import functools diff --git a/plugins/flytekit-geopandas/flytekitplugins/geopandas/gdf_transformers.py b/plugins/flytekit-geopandas/flytekitplugins/geopandas/gdf_transformers.py index c6be9a54b6..34c7ef8495 100644 --- a/plugins/flytekit-geopandas/flytekitplugins/geopandas/gdf_transformers.py +++ b/plugins/flytekit-geopandas/flytekitplugins/geopandas/gdf_transformers.py @@ -1,4 +1,3 @@ -import os import typing from pathlib import Path @@ -40,10 +39,12 @@ def encode( structured_dataset: StructuredDataset, structured_dataset_type: StructuredDatasetType, ) -> literals.StructuredDataset: - dir = ctx.file_access.get_random_remote_directory() - if not ctx.file_access.is_remote(dir): - Path(dir).mkdir(parents=True, exist_ok=True) - uri = os.path.join(str(dir), "data.parquet") + uri = typing.cast(str, structured_dataset.uri) or ctx.file_access.join( + ctx.file_access.raw_output_prefix, ctx.file_access.get_random_string() + ) + if not ctx.file_access.is_remote(uri): + Path(uri).mkdir(parents=True, exist_ok=True) + uri = str(Path(uri) / "data.parquet") df = typing.cast(gpd.GeoDataFrame, structured_dataset.dataframe) df.to_parquet(uri) structured_dataset_type.format = PARQUET diff --git a/plugins/flytekit-hive/flytekitplugins/hive/task.py b/plugins/flytekit-hive/flytekitplugins/hive/task.py index 76835a8d77..4b5d9a9451 100644 --- a/plugins/flytekit-hive/flytekitplugins/hive/task.py +++ b/plugins/flytekit-hive/flytekitplugins/hive/task.py @@ -124,7 +124,7 @@ def __init__( Args: select_query: Singular query that returns a Tabular dataset stage_query: optional query that should be executed before the actual ``select_query``. This can usually - be used for setting memory or the an alternate execution engine like `tez `__ + be used for setting memory or the an alternate execution engine like [`tez`](https://tez.apache.org) """ query_template = HiveSelectTask._HIVE_QUERY_FORMATTER.format( stage_query_str=stage_query or "", select_query_str=select_query.strip().strip(";") diff --git a/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py b/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py index a6a6ef3647..1eedd31f29 100644 --- a/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py +++ b/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py @@ -1,6 +1,6 @@ """ This Plugin adds the capability of running distributed MPI training to Flyte using backend plugins, natively on -Kubernetes. It leverages `MPI Job `_ Plugin from kubeflow. +Kubernetes. It leverages [`MPI Job`](https://github.com/kubeflow/mpi-operator) Plugin from kubeflow. """ from dataclasses import dataclass, field @@ -91,7 +91,7 @@ class Launcher: @dataclass class MPIJob(object): """ - Configuration for an executable `MPI Job `_. Use this + Configuration for an executable [`MPI Job`](https://github.com/kubeflow/mpi-operator). Use this to run distributed training on k8s with MPI Args: @@ -222,8 +222,8 @@ def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: @dataclass class HorovodJob(object): """ - Configuration for an executable `Horovod Job using MPI operator`_. Use this - to run distributed training on k8s with MPI. For more info, check out Running Horovod`_. + Configuration for an executable [`Horovod Job using MPI operator`](https://github.com/kubeflow/mpi-operator). Use this + to run distributed training on k8s with MPI. For more info, check out [`Running Horovod`](https://horovod.readthedocs.io/en/stable/summary_include.html#running-horovod). Args: worker: Worker configuration for the job. diff --git a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py index 9a77d6326e..62e4e0ccda 100644 --- a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py +++ b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py @@ -1,6 +1,6 @@ """ This Plugin adds the capability of running distributed pytorch training to Flyte using backend plugins, natively on -Kubernetes. It leverages `Pytorch Job `_ Plugin from kubeflow. +Kubernetes. It leverages [`Pytorch Job`](https://github.com/kubeflow/pytorch-operator) Plugin from kubeflow. """ import os @@ -95,7 +95,7 @@ class Master: @dataclass class PyTorch(object): """ - Configuration for an executable `PyTorch Job `_. Use this + Configuration for an executable [`PyTorch Job`](https://github.com/kubeflow/pytorch-operator). Use this to run distributed PyTorch training on Kubernetes. Please notice, in most cases, you should not worry about the configuration of the master and worker groups. The default configuration should work. The only field you should change is the number of workers. Both replicas will use the same image, and the same @@ -125,12 +125,12 @@ class PyTorch(object): @dataclass class Elastic(object): """ - Configuration for `torch elastic training `_. + Configuration for [`torch elastic training`](https://pytorch.org/docs/stable/elastic/run.html). Use this to run single- or multi-node distributed pytorch elastic training on k8s. Single-node elastic training is executed in a k8s pod when `nnodes` is set to 1. - Multi-node training is executed otherwise using a `Pytorch Job `_. + Multi-node training is executed otherwise using a [`Pytorch Job`](https://github.com/kubeflow/training-operator). Like `torchrun`, this plugin sets the environment variable `OMP_NUM_THREADS` to 1 if it is not set. Please see https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html for potential performance improvements. diff --git a/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py b/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py index 62cd482416..60cd0b8472 100644 --- a/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py +++ b/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py @@ -1,6 +1,6 @@ """ This Plugin adds the capability of running distributed tensorflow training to Flyte using backend plugins, natively on -Kubernetes. It leverages `TF Job `_ Plugin from kubeflow. +Kubernetes. It leverages [`TF Job`](https://github.com/kubeflow/tf-operator) Plugin from kubeflow. """ from dataclasses import dataclass, field @@ -98,7 +98,7 @@ class Evaluator: @dataclass class TfJob: """ - Configuration for an executable `TensorFlow Job `_. Use this + Configuration for an executable [`TensorFlow Job`](https://github.com/kubeflow/tf-operator). Use this to run distributed TensorFlow training on Kubernetes. Args: diff --git a/plugins/flytekit-papermill/flytekitplugins/papermill/task.py b/plugins/flytekit-papermill/flytekitplugins/papermill/task.py index 65281ca4c9..242cc452bc 100644 --- a/plugins/flytekit-papermill/flytekitplugins/papermill/task.py +++ b/plugins/flytekit-papermill/flytekitplugins/papermill/task.py @@ -50,7 +50,7 @@ class NotebookTask(PythonInstanceTask[T]): Property 2: For a notebook that produces outputs, that should be consumed by a subsequent notebook, use the method - :py:func:`record_outputs` in your notebook after the outputs are ready and pass all outputs. + {{< py_func_ref record_outputs >}} in your notebook after the outputs are ready and pass all outputs. Usage: @@ -67,7 +67,7 @@ class NotebookTask(PythonInstanceTask[T]): #cell end Step 2: Wrap in a task - Now point to the notebook and create an instance of :py:class:`NotebookTask` as follows + Now point to the notebook and create an instance of {{< py_class_ref NotebookTask >}} as follows Usage: diff --git a/plugins/flytekit-slurm/flytekitplugins/slurm/function/connector.py b/plugins/flytekit-slurm/flytekitplugins/slurm/function/connector.py index d2f24a2af9..e58730b647 100644 --- a/plugins/flytekit-slurm/flytekitplugins/slurm/function/connector.py +++ b/plugins/flytekit-slurm/flytekitplugins/slurm/function/connector.py @@ -7,7 +7,6 @@ from asyncssh import SSHClientConnection from asyncssh.sftp import SFTPNoSuchFile -from flytekit import logger from flytekit.extend.backend.base_connector import AsyncConnectorBase, ConnectorRegistry, Resource, ResourceMeta from flytekit.extend.backend.utils import convert_to_flyte_phase from flytekit.models.literals import LiteralMap @@ -85,7 +84,6 @@ async def get(self, resource_meta: SlurmJobMetadata, **kwargs) -> Resource: ) job_res = await conn.run(f"scontrol --json show job {resource_meta.job_id}", check=True) job_info = json.loads(job_res.stdout)["jobs"][0] - # Determine the current flyte phase from Slurm job state job_state = job_info["job_state"][0].strip().lower() cur_phase = convert_to_flyte_phase(job_state) diff --git a/plugins/flytekit-slurm/flytekitplugins/slurm/script/connector.py b/plugins/flytekit-slurm/flytekitplugins/slurm/script/connector.py index ac7dd9caa5..cbdd422ab9 100644 --- a/plugins/flytekit-slurm/flytekitplugins/slurm/script/connector.py +++ b/plugins/flytekit-slurm/flytekitplugins/slurm/script/connector.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type from asyncssh import SSHClientConnection -from asyncssh.sftp import SFTPError, SFTPNoSuchFile +from asyncssh.sftp import SFTPError import flytekit from flytekit.core.type_engine import TypeEngine diff --git a/pydoclint-errors-baseline.txt b/pydoclint-errors-baseline.txt index c6c02f2413..b0abfe55ba 100644 --- a/pydoclint-errors-baseline.txt +++ b/pydoclint-errors-baseline.txt @@ -133,10 +133,6 @@ flytekit/core/utils.py DOC301: Class `ClassDecorator`: __init__() should not have a docstring; please combine it with the docstring of the class -------------------- flytekit/core/workflow.py - DOC101: Function `workflow`: Docstring contains fewer arguments than in function signature. - DOC103: Function `workflow`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [_workflow_function: Optional[Callable[P, FuncOut]], default_options: Optional[Options], docs: Optional[Documentation], failure_policy: Optional[WorkflowFailurePolicy], interruptible: bool, on_failure: Optional[Union[WorkflowBase, Task]], pickle_untyped: bool]. - DOC201: Function `workflow` does not have a return section in docstring - DOC203: Function `workflow` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). DOC101: Function `reference_workflow`: Docstring contains fewer arguments than in function signature. DOC103: Function `reference_workflow`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [domain: str, name: str, project: str, version: str]. DOC201: Function `reference_workflow` does not have a return section in docstring diff --git a/tests/flytekit/integration/remote/workflows/basic/basic_workflow.py b/tests/flytekit/integration/remote/workflows/basic/basic_workflow.py index d63f96336b..6aa8f91a65 100644 --- a/tests/flytekit/integration/remote/workflows/basic/basic_workflow.py +++ b/tests/flytekit/integration/remote/workflows/basic/basic_workflow.py @@ -12,7 +12,7 @@ of ``t2`` in the workflow below. As such, the body of workflows is run at "registration" time. Please refer to the registration docs for additional information as well since it is actually a two-step process. -Take a look at the conceptual `discussion `__ +Take a look at the conceptual [`discussion`](https://lyft.github.io/flyte/user/concepts/workflows_nodes.html#workflows) behind workflows for additional information. """ @@ -33,7 +33,7 @@ def t2(a: str, b: str) -> str: # %% # You can treat the outputs of a task as you normally would a Python function. Assign the output to two variables -# and use them in subsequent tasks as normal. See :py:func:`flytekit.workflow` +# and use them in subsequent tasks as normal. See {{< py_func_ref flytekit.workflow >}} @workflow def my_wf(a: int, b: str) -> (int, str): x, y = t1(a=a) diff --git a/tests/flytekit/integration/remote/workflows/basic/hello_world.py b/tests/flytekit/integration/remote/workflows/basic/hello_world.py index da7e61536f..2f810002c1 100644 --- a/tests/flytekit/integration/remote/workflows/basic/hello_world.py +++ b/tests/flytekit/integration/remote/workflows/basic/hello_world.py @@ -18,7 +18,7 @@ def say_hello() -> str: # %% # You can treat the outputs of a task as you normally would a Python function. Assign the output to two variables -# and use them in subsequent tasks as normal. See :py:func:`flytekit.workflow` +# and use them in subsequent tasks as normal. See {{< py_func_ref flytekit.workflow >}} # You can change the signature of the workflow to take in an argument like this: # def my_wf(name: str) -> str: @workflow