From 3250822db0f1920be728eb929a643b1b642cde3b Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 10 Feb 2026 11:27:46 +0000 Subject: [PATCH 1/3] Kubernetes: rework jump pod provisioning With these changes, run_job() only creates a jump pod, and it is provisioned in update_provisioning_data() as follows: - check if the pod is running, try again later if not - collect all cluster external IPs, prefer the pod's node IP, fall back to a random IP if the pod's node has no external IP - connect to the jump pod, add the user's public SSH key This patch also fixes a bug introduced in https://github.com/dstackai/dstack/pull/3273 where it was not possible to add another user's public key due to `ForceCommand /bin/false`. Fixes: https://github.com/dstackai/dstack/issues/3559 --- .../core/backends/kubernetes/compute.py | 353 +++++++++--------- .../core/backends/kubernetes/resources.py | 9 +- .../core/backends/kubernetes/utils.py | 29 -- 3 files changed, 188 insertions(+), 203 deletions(-) diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index 10b8e5366f..3d9af0007d 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -1,13 +1,13 @@ import shlex import subprocess import tempfile -import threading import time from enum import Enum from typing import List, Optional from gpuhunt import AcceleratorVendor from kubernetes import client +from typing_extensions import Self from dstack._internal.core.backends.base.compute import ( Compute, @@ -34,6 +34,7 @@ NVIDIA_GPU_NODE_TAINT, NVIDIA_GPU_PRODUCT_LABEL, NVIDIA_GPU_RESOURCE, + PodPhase, TaintEffect, format_memory, get_amd_gpu_from_node_labels, @@ -41,6 +42,7 @@ get_instance_offer_from_node, get_instance_offers, get_node_labels, + get_node_name, get_nvidia_gpu_from_node_labels, is_hard_taint, is_taint_tolerated, @@ -48,10 +50,10 @@ from dstack._internal.core.backends.kubernetes.utils import ( call_api_method, get_api_from_config_data, - get_cluster_public_ip, ) from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT from dstack._internal.core.errors import ComputeError +from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.gateways import ( GatewayComputeConfiguration, GatewayProvisioningData, @@ -73,6 +75,7 @@ JUMP_POD_IMAGE = "testcontainers/sshd:1.3.0@sha256:c50c0f59554dcdb2d9e5e705112144428ae9d04ac0af6322b365a18e24213a6a" JUMP_POD_SSH_PORT = 22 +JUMP_POD_USER = "root" class Operator(str, Enum): @@ -80,6 +83,16 @@ class Operator(str, Enum): IN = "In" +class KubernetesBackendData(CoreModel): + jump_pod_name: str + jump_pod_service_name: str + user_ssh_public_key: str + + @classmethod + def load(cls, raw: str) -> Self: + return cls.__response__.parse_raw(raw) + + class KubernetesCompute( ComputeWithFilteredOffersCached, ComputeWithPrivilegedSupport, @@ -116,39 +129,19 @@ def run_job( commands = get_docker_commands( [run.run_spec.ssh_key_pub.strip(), project_ssh_public_key.strip()] ) - # Before running a job, ensure a jump pod service is running. # There is a one jump pod per Kubernetes backend that is used # as an ssh proxy jump to connect to all other services in Kubernetes. - # Setup jump pod in a separate thread to avoid long-running run_job. - # In case the thread fails, the job will be failed and resubmitted. - jump_pod_hostname = self.proxy_jump.hostname - if jump_pod_hostname is None: - jump_pod_hostname = get_cluster_public_ip(self.api) - if jump_pod_hostname is None: - raise ComputeError( - "Failed to acquire an IP for jump pod automatically. " - "Specify ssh_host for Kubernetes backend." - ) - jump_pod_port, created = _create_jump_pod_service_if_not_exists( + # The service is created here and configured later in update_provisioning_data() + jump_pod_name = f"dstack-{run.project_name}-ssh-jump-pod" + jump_pod_service_name = _get_pod_service_name(jump_pod_name) + _create_jump_pod_service_if_not_exists( api=self.api, namespace=self.config.namespace, - project_name=run.project_name, - ssh_public_keys=[project_ssh_public_key.strip(), run.run_spec.ssh_key_pub.strip()], + jump_pod_name=jump_pod_name, + jump_pod_service_name=jump_pod_service_name, jump_pod_port=self.proxy_jump.port, + project_ssh_public_key=project_ssh_public_key.strip(), ) - if not created: - threading.Thread( - target=_continue_setup_jump_pod, - kwargs={ - "api": self.api, - "namespace": self.config.namespace, - "project_name": run.project_name, - "project_ssh_private_key": project_ssh_private_key.strip(), - "user_ssh_public_key": run.run_spec.ssh_key_pub.strip(), - "jump_pod_host": jump_pod_hostname, - "jump_pod_port": jump_pod_port, - }, - ).start() resources_requests: dict[str, str] = {} resources_limits: dict[str, str] = {} @@ -264,27 +257,32 @@ def run_job( ), ), ) + + backend_data = KubernetesBackendData( + jump_pod_name=jump_pod_name, + jump_pod_service_name=jump_pod_service_name, + user_ssh_public_key=run.run_spec.ssh_key_pub.strip(), + ) return JobProvisioningData( backend=instance_offer.backend, - instance_type=instance_offer.instance, instance_id=instance_name, - # Although we can already get Service's ClusterIP from the `V1Service` object returned - # by the `create_namespaced_service` method, we still need 1) updated instance offer - # 2) PodIP for multinode runs. - # We'll update all these fields once the pod is assigned to the node. - hostname=None, - internal_ip=None, region=instance_offer.region, price=instance_offer.price, username="root", ssh_port=DSTACK_RUNNER_SSH_PORT, dockerized=False, - ssh_proxy=SSHConnectionParams( - hostname=jump_pod_hostname, - username="root", - port=jump_pod_port, - ), - backend_data=None, + # Although we can already get Service's ClusterIP from the `V1Service` object returned + # by the `create_namespaced_service` method, we still need: + # - updated instance offer + # - job pod's PodIP for multinode runs + # - jump pod node's ExternalIP and jump pod service's NodePort for ssh_proxy + # We'll update all these fields once both the jump pod and the job pod are assigned + # to the nodes. + hostname=None, + instance_type=instance_offer.instance, + internal_ip=None, + ssh_proxy=None, + backend_data=backend_data.json(), ) def update_provisioning_data( @@ -293,6 +291,26 @@ def update_provisioning_data( project_ssh_public_key: str, project_ssh_private_key: str, ): + if provisioning_data.backend_data is not None: + # Before running a job, ensure the jump pod is running and has user's public SSH key. + backend_data = KubernetesBackendData.load(provisioning_data.backend_data) + ssh_proxy = _check_and_configure_jump_pod_service( + api=self.api, + namespace=self.config.namespace, + jump_pod_name=backend_data.jump_pod_name, + jump_pod_service_name=backend_data.jump_pod_service_name, + jump_pod_hostname=self.proxy_jump.hostname, + project_ssh_private_key=project_ssh_private_key, + user_ssh_public_key=backend_data.user_ssh_public_key, + ) + if ssh_proxy is None: + # Jump pod is not ready yet + return + provisioning_data.ssh_proxy = ssh_proxy + # Remove backend data to save space in DB and skip this step + # in case update_provisioning_data() is called again. + provisioning_data.backend_data = None + pod = self.api.read_namespaced_pod( name=provisioning_data.instance_id, namespace=self.config.namespace, @@ -560,36 +578,14 @@ def _gpu_matches_gpu_spec(gpu: Gpu, gpu_spec: GPUSpec) -> bool: return True -def _continue_setup_jump_pod( - api: client.CoreV1Api, - namespace: str, - project_name: str, - project_ssh_private_key: str, - user_ssh_public_key: str, - jump_pod_host: str, - jump_pod_port: int, -): - _wait_for_pod_ready( - api=api, - namespace=namespace, - pod_name=_get_jump_pod_name(project_name), - ) - _add_authorized_key_to_jump_pod( - jump_pod_host=jump_pod_host, - jump_pod_port=jump_pod_port, - ssh_private_key=project_ssh_private_key, - ssh_authorized_key=user_ssh_public_key, - ) - - def _create_jump_pod_service_if_not_exists( api: client.CoreV1Api, namespace: str, - project_name: str, - ssh_public_keys: list[str], + jump_pod_name: str, + jump_pod_service_name: str, jump_pod_port: Optional[int], -) -> tuple[int, bool]: - created = False + project_ssh_public_key: str, +) -> None: service: Optional[client.V1Service] = None pod: Optional[client.V1Pod] = None _namespace = call_api_method( @@ -609,52 +605,27 @@ def _create_jump_pod_service_if_not_exists( service = call_api_method( api.read_namespaced_service, expected=404, - name=_get_jump_pod_service_name(project_name), + name=jump_pod_service_name, namespace=namespace, ) pod = call_api_method( api.read_namespaced_pod, expected=404, - name=_get_jump_pod_name(project_name), + name=jump_pod_name, namespace=namespace, ) + # The service may exist without the pod if the node on which the jump pod was running # has been deleted. - if service is None or pod is None: - service = _create_jump_pod_service( - api=api, - namespace=namespace, - project_name=project_name, - ssh_public_keys=ssh_public_keys, - jump_pod_port=jump_pod_port, - ) - created = True - port: Optional[int] = None - if service.spec is not None and service.spec.ports: - port = service.spec.ports[0].node_port - if port is None: - raise ComputeError( - f"Failed to get NodePort of jump pod Service for project '{project_name}'" - ) - return port, created + if service is not None and pod is not None: + return - -def _create_jump_pod_service( - api: client.CoreV1Api, - namespace: str, - project_name: str, - ssh_public_keys: list[str], - jump_pod_port: Optional[int], -) -> client.V1Service: - # TODO use restricted ssh-forwarding-only user for jump pod instead of root. - pod_name = _get_jump_pod_name(project_name) call_api_method( api.delete_namespaced_pod, expected=404, namespace=namespace, - name=pod_name, + name=jump_pod_name, ) - # False if we found at least one node without any "hard" taint, that is, if we don't need to # specify the toleration. toleration_required = True @@ -684,17 +655,16 @@ def _create_jump_pod_service( ) if not tolerations: logger.warning("No appropriate node found, the jump pod may never be scheduled") - - commands = _get_jump_pod_commands(authorized_keys=ssh_public_keys) + commands = _get_jump_pod_commands(authorized_keys=[project_ssh_public_key]) pod = client.V1Pod( metadata=client.V1ObjectMeta( - name=pod_name, - labels={"app.kubernetes.io/name": pod_name}, + name=jump_pod_name, + labels={"app.kubernetes.io/name": jump_pod_name}, ), spec=client.V1PodSpec( containers=[ client.V1Container( - name=f"{pod_name}-container", + name=f"{jump_pod_name}-container", image=JUMP_POD_IMAGE, command=["/bin/sh"], args=["-c", " && ".join(commands)], @@ -712,18 +682,17 @@ def _create_jump_pod_service( namespace=namespace, body=pod, ) - service_name = _get_jump_pod_service_name(project_name) call_api_method( api.delete_namespaced_service, expected=404, namespace=namespace, - name=service_name, + name=jump_pod_service_name, ) service = client.V1Service( - metadata=client.V1ObjectMeta(name=service_name), + metadata=client.V1ObjectMeta(name=jump_pod_service_name), spec=client.V1ServiceSpec( type="NodePort", - selector={"app.kubernetes.io/name": pod_name}, + selector={"app.kubernetes.io/name": jump_pod_name}, ports=[ client.V1ServicePort( port=JUMP_POD_SSH_PORT, @@ -733,12 +702,110 @@ def _create_jump_pod_service( ], ), ) - return api.create_namespaced_service( + api.create_namespaced_service( namespace=namespace, body=service, ) +def _check_and_configure_jump_pod_service( + api: client.CoreV1Api, + namespace: str, + jump_pod_name: str, + jump_pod_service_name: str, + jump_pod_hostname: Optional[str], + project_ssh_private_key: str, + user_ssh_public_key: str, +) -> Optional[SSHConnectionParams]: + jump_pod = api.read_namespaced_pod( + namespace=namespace, + name=jump_pod_name, + ) + jump_pod_phase = PodPhase(get_or_error(get_or_error(jump_pod.status).phase)) + if jump_pod_phase.is_finished(): + raise ComputeError(f"Jump pod {jump_pod_name} is unexpectedly finished") + if not jump_pod_phase.is_running(): + logger.debug("Jump pod %s is not running yet", jump_pod_name) + return None + + if jump_pod_hostname is None: + jump_pod_node_name = get_or_error(get_or_error(jump_pod.spec).node_name) + cluster_external_ips: list[str] = [] + for node in api.list_node().items: + node_external_ips = [ + node_address.address + for node_address in get_or_error(get_or_error(node.status).addresses) + if node_address.type == "ExternalIP" + ] + if node_external_ips: + if get_node_name(node) == jump_pod_node_name: + jump_pod_hostname = node_external_ips[0] + break + cluster_external_ips.extend(node_external_ips) + if jump_pod_hostname is None: + if not cluster_external_ips: + raise ComputeError( + "Failed to acquire an IP for jump pod automatically." + " Specify proxy_jump.hostname for Kubernetes backend." + ) + jump_pod_hostname = cluster_external_ips[0] + logger.info( + ( + "Jump pod %s is running on node %s which has no external IP," + " picking a random external IP: %s" + ), + jump_pod_name, + jump_pod_node_name, + jump_pod_hostname, + ) + + jump_pod_service = api.read_namespaced_service( + name=jump_pod_service_name, + namespace=namespace, + ) + jump_pod_service_ports = get_or_error(jump_pod_service.spec).ports + if not jump_pod_service_ports: + raise ComputeError("Jump pod service %s ports are empty", jump_pod_service_name) + if (jump_pod_port := jump_pod_service_ports[0].node_port) is None: + raise ComputeError("Jump pod service %s port is not set", jump_pod_service_name) + + ssh_exit_status, ssh_output = _run_ssh_command( + hostname=jump_pod_hostname, + port=jump_pod_port, + username=JUMP_POD_USER, + ssh_private_key=project_ssh_private_key, + # command= in authorized_keys is equivalent to ForceCommand in sshd_config + # By forcing the /bin/false command we only allow proxy jumping, no shell access + command=f""" + if grep -qvF '{user_ssh_public_key}' ~/.ssh/authorized_keys; then + echo 'command="/bin/false" {user_ssh_public_key}' >> ~/.ssh/authorized_keys + fi + """, + ) + if ssh_exit_status != 0: + logger.debug( + "Jump pod %s @ %s:%d, SSH command failed, exit status: %d, output: %s", + jump_pod_name, + jump_pod_hostname, + jump_pod_port, + ssh_exit_status, + ssh_output, + ) + return None + + logger.debug( + "Jump pod %s is available @ %s:%d", + jump_pod_name, + jump_pod_hostname, + jump_pod_port, + ) + return SSHConnectionParams( + hostname=jump_pod_hostname, + port=jump_pod_port, + username=JUMP_POD_USER, + ) + + def _get_jump_pod_commands(authorized_keys: list[str]) -> list[str]: authorized_keys_content = "\n".join(authorized_keys).strip() commands = [ @@ -755,40 +822,11 @@ def _get_jump_pod_commands(authorized_keys: list[str]) -> list[str]: " -o LogLevel=ERROR" " -o PasswordAuthentication=no" " -o AllowTcpForwarding=local" - # proxy jumping only, no shell access - " -o ForceCommand=/bin/false" ), ] return commands -def _wait_for_pod_ready( - api: client.CoreV1Api, - namespace: str, - pod_name: str, - timeout_seconds: int = 300, -): - start_time = time.time() - while True: - pod = call_api_method( - api.read_namespaced_pod, - expected=404, - name=pod_name, - namespace=namespace, - ) - if pod is not None: - pod_status = get_or_error(pod.status) - phase = get_or_error(pod_status.phase) - container_statuses = get_or_error(pod_status.container_statuses) - if phase == "Running" and all(status.ready for status in container_statuses): - return True - elapsed_time = time.time() - start_time - if elapsed_time >= timeout_seconds: - logger.warning("Timeout waiting for pod %s to be ready", pod_name) - return False - time.sleep(1) - - def _wait_for_load_balancer_address( api: client.CoreV1Api, namespace: str, @@ -824,24 +862,6 @@ def _wait_for_load_balancer_address( time.sleep(1) -def _add_authorized_key_to_jump_pod( - jump_pod_host: str, - jump_pod_port: int, - ssh_private_key: str, - ssh_authorized_key: str, -): - _run_ssh_command( - hostname=jump_pod_host, - port=jump_pod_port, - ssh_private_key=ssh_private_key, - command=( - f'if grep -qvF "{ssh_authorized_key}" ~/.ssh/authorized_keys; then ' - f"echo {ssh_authorized_key} >> ~/.ssh/authorized_keys; " - "fi" - ), - ) - - def _get_gateway_commands( authorized_keys: List[str], router: Optional[AnyRouterConfig] = None ) -> List[str]: @@ -882,11 +902,13 @@ def _get_gateway_commands( return commands -def _run_ssh_command(hostname: str, port: int, ssh_private_key: str, command: str): +def _run_ssh_command( + hostname: str, port: int, username: str, ssh_private_key: str, command: str +) -> tuple[int, bytes]: with tempfile.NamedTemporaryFile("w+", 0o600) as f: f.write(ssh_private_key) f.flush() - subprocess.run( + proc = subprocess.run( [ "ssh", "-F", @@ -897,20 +919,13 @@ def _run_ssh_command(hostname: str, port: int, ssh_private_key: str, command: st f.name, "-p", str(port), - f"root@{hostname}", + f"{username}@{hostname}", command, ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, ) - - -def _get_jump_pod_name(project_name: str) -> str: - return f"dstack-{project_name}-ssh-jump-pod" - - -def _get_jump_pod_service_name(project_name: str) -> str: - return f"dstack-{project_name}-ssh-jump-pod-service" + return proc.returncode, proc.stdout def _get_pod_service_name(pod_name: str) -> str: diff --git a/src/dstack/_internal/core/backends/kubernetes/resources.py b/src/dstack/_internal/core/backends/kubernetes/resources.py index 018ff5fb62..d5cb1739f4 100644 --- a/src/dstack/_internal/core/backends/kubernetes/resources.py +++ b/src/dstack/_internal/core/backends/kubernetes/resources.py @@ -61,12 +61,11 @@ class PodPhase(str, Enum): FAILED = "Failed" UNKNOWN = "Unknown" # Deprecated: It isn't being set since 2015 - @classmethod - def finished_statuses(cls) -> list["PodPhase"]: - return [cls.SUCCEEDED, cls.FAILED] - def is_finished(self): - return self in self.finished_statuses() + return self in [self.SUCCEEDED, self.FAILED] + + def is_running(self): + return self == self.RUNNING class TaintEffect(str, Enum): diff --git a/src/dstack/_internal/core/backends/kubernetes/utils.py b/src/dstack/_internal/core/backends/kubernetes/utils.py index 78213b6178..fb7816b572 100644 --- a/src/dstack/_internal/core/backends/kubernetes/utils.py +++ b/src/dstack/_internal/core/backends/kubernetes/utils.py @@ -9,8 +9,6 @@ ) from typing_extensions import ParamSpec -from dstack._internal.utils.common import get_or_error - T = TypeVar("T") P = ParamSpec("P") @@ -52,30 +50,3 @@ def call_api_method( if e.status not in expected: raise return None - - -def get_cluster_public_ip(api: CoreV1Api) -> Optional[str]: - """ - Returns public IP of any cluster node. - """ - public_ips = get_cluster_public_ips(api) - if len(public_ips) == 0: - return None - return public_ips[0] - - -def get_cluster_public_ips(api: CoreV1Api) -> list[str]: - """ - Returns public IPs of all cluster nodes. - """ - public_ips = [] - for node in api.list_node().items: - node_status = get_or_error(node.status) - addresses = get_or_error(node_status.addresses) - - # Look for an external IP address - for address in addresses: - if address.type == "ExternalIP": - public_ips.append(address.address) - - return public_ips From 4f3e8ff4ef723742543de8c1312686a8e925d513 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 10 Feb 2026 12:11:02 +0000 Subject: [PATCH 2/3] ComputeError -> ProvisioningError --- .../_internal/core/backends/kubernetes/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index 3d9af0007d..6e1b4deb15 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -52,7 +52,7 @@ get_api_from_config_data, ) from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT -from dstack._internal.core.errors import ComputeError +from dstack._internal.core.errors import ComputeError, ProvisioningError from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.gateways import ( GatewayComputeConfiguration, @@ -723,7 +723,7 @@ def _check_and_configure_jump_pod_service( ) jump_pod_phase = PodPhase(get_or_error(get_or_error(jump_pod.status).phase)) if jump_pod_phase.is_finished(): - raise ComputeError(f"Jump pod {jump_pod_name} is unexpectedly finished") + raise ProvisioningError(f"Jump pod {jump_pod_name} is unexpectedly finished") if not jump_pod_phase.is_running(): logger.debug("Jump pod %s is not running yet", jump_pod_name) return None @@ -744,7 +744,7 @@ def _check_and_configure_jump_pod_service( cluster_external_ips.extend(node_external_ips) if jump_pod_hostname is None: if not cluster_external_ips: - raise ComputeError( + raise ProvisioningError( "Failed to acquire an IP for jump pod automatically." " Specify proxy_jump.hostname for Kubernetes backend." ) @@ -765,9 +765,9 @@ def _check_and_configure_jump_pod_service( ) jump_pod_service_ports = get_or_error(jump_pod_service.spec).ports if not jump_pod_service_ports: - raise ComputeError("Jump pod service %s ports are empty", jump_pod_service_name) + raise ProvisioningError("Jump pod service %s ports are empty", jump_pod_service_name) if (jump_pod_port := jump_pod_service_ports[0].node_port) is None: - raise ComputeError("Jump pod service %s port is not set", jump_pod_service_name) + raise ProvisioningError("Jump pod service %s port is not set", jump_pod_service_name) ssh_exit_status, ssh_output = _run_ssh_command( hostname=jump_pod_hostname, From 244b18837daf1c90d1fa654df5c49ad94c72e1d4 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 10 Feb 2026 12:56:21 +0000 Subject: [PATCH 3/3] Use ConnectTimeout, pick IP randomly --- src/dstack/_internal/core/backends/kubernetes/compute.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index 6e1b4deb15..51abddc70c 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -1,3 +1,4 @@ +import random import shlex import subprocess import tempfile @@ -748,7 +749,7 @@ def _check_and_configure_jump_pod_service( "Failed to acquire an IP for jump pod automatically." " Specify proxy_jump.hostname for Kubernetes backend." ) - jump_pod_hostname = cluster_external_ips[0] + jump_pod_hostname = random.choice(cluster_external_ips) logger.info( ( "Jump pod %s is running on node %s which has no external IP," @@ -915,6 +916,10 @@ def _run_ssh_command( "none", "-o", "StrictHostKeyChecking=no", + "-o", + # The same timeout as in core.services.ssh.tunnel.SSH_DEFAULT_OPTIONS, + # which is used, for example, by server.services.runner.ssh.runner_ssh_tunnel() + "ConnectTimeout=3", "-i", f.name, "-p",