From d7190a2521cfa92c7d29b2b78dab200b6768c603 Mon Sep 17 00:00:00 2001 From: tomasz t Date: Tue, 4 Aug 2026 15:14:05 -0700 Subject: [PATCH 1/4] allow passing kwargs to sftp client Add sftp_client_kwargs parameter to SSHFileSystem and the channel pools, forwarded to asyncssh.SSHClientConnection.start_sftp_client (e.g. env, send_env, path_encoding, path_errors, sftp_version). Squashed and rebased onto main from #41. Fixes #39 --- sshfs/pools/base.py | 12 ++++++++---- sshfs/spec.py | 29 ++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/sshfs/pools/base.py b/sshfs/pools/base.py index 9363800..5b1908e 100644 --- a/sshfs/pools/base.py +++ b/sshfs/pools/base.py @@ -1,5 +1,6 @@ import asyncio from contextlib import AsyncExitStack, suppress +from typing import Optional from asyncssh.misc import ChannelOpenError @@ -15,9 +16,10 @@ def __init__( self, client, *, - max_channels=None, - timeout=MAX_TIMEOUT, - unsafe_terminate=True, + max_channels: Optional[int] = None, + timeout: int = MAX_TIMEOUT, + unsafe_terminate: bool = True, + sftp_client_kwargs: Optional[dict] = None, **kwargs, ): self.client = client @@ -38,6 +40,8 @@ def __init__( self.unsafe_terminate = unsafe_terminate self._stack = AsyncExitStack() + self.sftp_client_kwargs = sftp_client_kwargs or {} + async def _maybe_new_channel(self): # If there is no hard limit or the limit is not hit yet # try to create a new channel @@ -47,7 +51,7 @@ async def _maybe_new_channel(self): ): try: return await self._stack.enter_async_context( - self.client.start_sftp_client() + self.client.start_sftp_client(**self.sftp_client_kwargs) ) except ChannelOpenError: # If we can't create any more channels, then change diff --git a/sshfs/spec.py b/sshfs/spec.py index 77a1e11..9d20cd8 100644 --- a/sshfs/spec.py +++ b/sshfs/spec.py @@ -5,6 +5,7 @@ import weakref from contextlib import AsyncExitStack, suppress from datetime import datetime, timezone +from typing import Optional import asyncssh from asyncssh.sftp import SFTPOpUnsupported @@ -40,6 +41,7 @@ def __init__( host, *, pool_type=SFTPSoftChannelPool, + sftp_client_kwargs: Optional[dict] = None, **kwargs, ): """ @@ -51,15 +53,20 @@ def __init__( SSH host to connect. **kwargs: Any Any option that will be passed to either the top level - `AsyncFileSystem` or the `asyncssh.connect`. + `AsyncFileSystem` (e.g. timeout) + or the `asyncssh.connect`. pool_type: sshfs.pools.base.BaseSFTPChannelPool Pool manager to use (when doing concurrent operations together, pool managers offer the flexibility of prioritizing channels and deciding which to use). + sftp_client_kwargs: Optional[dict] + Parameters to pass to asyncssh.SSHClientConnection.start_sftp_client method + (e.g. env, send_env, path_encoding, path_errors, sftp_version). """ super().__init__(self, **kwargs) + _timeout = kwargs.pop("timeout", None) max_sessions = kwargs.pop("max_sessions", _DEFAULT_MAX_SESSIONS) if max_sessions <= _SHELL_CHANNELS: raise ValueError( @@ -67,6 +74,7 @@ def __init__( ) _client_args = kwargs.copy() _client_args.setdefault("known_hosts", None) + sftp_client_kwargs = sftp_client_kwargs or {} self._stack = AsyncExitStack() self.active_executors = 0 @@ -74,7 +82,9 @@ def __init__( host, pool_type, max_sftp_channels=max_sessions - _SHELL_CHANNELS, - **_client_args, + timeout=_timeout, # goes to sync_wrapper + connect_args=_client_args, # for asyncssh.connect + sftp_client_kwargs=sftp_client_kwargs, # for asyncssh.SSHClientConnection.start_sftp_client ) weakref.finalize( self, self._finalize, self.loop, self._pool, self._stack @@ -95,13 +105,22 @@ def _get_kwargs_from_urls(urlpath): @wrap_exceptions async def _connect( - self, host, pool_type, max_sftp_channels, **client_args + self, + host, + pool_type, + max_sftp_channels, + connect_args, + sftp_client_kwargs, ): self._client_lock = asyncio.Semaphore(_SHELL_CHANNELS) - _raw_client = asyncssh.connect(host, **client_args) + _raw_client = asyncssh.connect(host, **connect_args) client = await self._stack.enter_async_context(_raw_client) - pool = pool_type(client, max_channels=max_sftp_channels) + pool = pool_type( + client, + max_channels=max_sftp_channels, + sftp_client_kwargs=sftp_client_kwargs, + ) return client, pool connect = sync_wrapper(_connect) From d02015d0a193927abc103ef0fcbedfd7ca8bdc22 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Tue, 4 Aug 2026 15:15:59 -0700 Subject: [PATCH 2/4] tests: cover sftp_client_kwargs; polish docs Assert the options reach the channel pool and that a filesystem constructed with sftp_version=3 works end to end. Trim the call-site comments and wrap the docstring within the line limit. Co-Authored-By: Claude Fable 5 --- sshfs/spec.py | 17 ++++++++++------- tests/test_sshfs.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/sshfs/spec.py b/sshfs/spec.py index 9d20cd8..b47dcf0 100644 --- a/sshfs/spec.py +++ b/sshfs/spec.py @@ -53,15 +53,16 @@ def __init__( SSH host to connect. **kwargs: Any Any option that will be passed to either the top level - `AsyncFileSystem` (e.g. timeout) - or the `asyncssh.connect`. + `AsyncFileSystem` or the `asyncssh.connect`. `timeout` + limits the initial connection setup. pool_type: sshfs.pools.base.BaseSFTPChannelPool Pool manager to use (when doing concurrent operations together, pool managers offer the flexibility of prioritizing channels and deciding which to use). sftp_client_kwargs: Optional[dict] - Parameters to pass to asyncssh.SSHClientConnection.start_sftp_client method - (e.g. env, send_env, path_encoding, path_errors, sftp_version). + Options passed to `SSHClientConnection.start_sftp_client` + (e.g. env, send_env, path_encoding, path_errors, + sftp_version). """ super().__init__(self, **kwargs) @@ -82,9 +83,11 @@ def __init__( host, pool_type, max_sftp_channels=max_sessions - _SHELL_CHANNELS, - timeout=_timeout, # goes to sync_wrapper - connect_args=_client_args, # for asyncssh.connect - sftp_client_kwargs=sftp_client_kwargs, # for asyncssh.SSHClientConnection.start_sftp_client + # timeout is consumed by the sync() machinery underneath + # sync_wrapper, it never reaches _connect + timeout=_timeout, + connect_args=_client_args, + sftp_client_kwargs=sftp_client_kwargs, ) weakref.finalize( self, self._finalize, self.loop, self._pool, self._stack diff --git a/tests/test_sshfs.py b/tests/test_sshfs.py index ebe351b..5635b88 100644 --- a/tests/test_sshfs.py +++ b/tests/test_sshfs.py @@ -100,6 +100,21 @@ def test_fsspec_url_parsing(ssh_server, remote_dir, user="user"): } +def test_sftp_client_kwargs(ssh_server, base_remote_dir, user="user"): + fs = SSHFileSystem( + host=ssh_server.host, + port=ssh_server.port, + username=user, + client_keys=[USERS[user]], + sftp_client_kwargs={"sftp_version": 3}, + ) + assert fs._pool.sftp_client_kwargs == {"sftp_version": 3} + + file = posixpath.join(base_remote_dir, "sftp_client_kwargs_probe") + fs.touch(file) + assert fs.exists(file) + + def test_info(fs, remote_dir): fs.touch(remote_dir + "/a.txt") details = fs.info(remote_dir + "/a.txt") From cb498fb5f9259d257cf4f8dd3dfb18c090ce929c Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Tue, 4 Aug 2026 15:21:10 -0700 Subject: [PATCH 3/4] tests: assert sftp_client_kwargs reach start_sftp_client; fix timeout annotation Both raised by Copilot review: the pool-level test captures the kwargs actually passed to start_sftp_client (the fs-level test only proves they are stored), and the timeout annotation now allows None as the docstring describes. Co-Authored-By: Claude Fable 5 --- sshfs/pools/base.py | 2 +- tests/test_sftp_pools.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/sshfs/pools/base.py b/sshfs/pools/base.py index 5b1908e..1510a29 100644 --- a/sshfs/pools/base.py +++ b/sshfs/pools/base.py @@ -17,7 +17,7 @@ def __init__( client, *, max_channels: Optional[int] = None, - timeout: int = MAX_TIMEOUT, + timeout: Optional[float] = MAX_TIMEOUT, unsafe_terminate: bool = True, sftp_client_kwargs: Optional[dict] = None, **kwargs, diff --git a/tests/test_sftp_pools.py b/tests/test_sftp_pools.py index 39375b9..8e7af33 100644 --- a/tests/test_sftp_pools.py +++ b/tests/test_sftp_pools.py @@ -38,11 +38,13 @@ class FakeSSHClient: def __init__(self, max_channels=None): self.counter = 0 self.max_channels = max_channels + self.received_sftp_client_kwargs = None @asynccontextmanager - async def start_sftp_client(self): + async def start_sftp_client(self, **kwargs): from asyncssh.misc import ChannelOpenError + self.received_sftp_client_kwargs = kwargs if self.max_channels is not None and self.counter >= self.max_channels: raise ChannelOpenError(None, None) @@ -55,6 +57,19 @@ def fake_client(): yield FakeSSHClient() +@all_queues +@pytest.mark.asyncio +async def test_pool_forwards_sftp_client_kwargs(queue_type, fake_client): + pool = queue_type( + fake_client, + sftp_client_kwargs={"sftp_version": 3}, + poll=False, + ) + async with pool.get() as channel: + assert channel.no == 1 + assert fake_client.received_sftp_client_kwargs == {"sftp_version": 3} + + @pytest.mark.asyncio async def test_pool_soft_queue_caching(fake_client): pool = SFTPSoftChannelPool(fake_client, poll=False) From 032c69ec0b597320a728b4f0f672d30e8ae05f1f Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Tue, 4 Aug 2026 15:52:56 -0700 Subject: [PATCH 4/4] tests: cover the path_encoding scenario from #39 path_encoding=None flows through sftp_client_kwargs to a real channel and asyncssh delivers paths as raw bytes, which is the workaround for servers with non-UTF-8 file names that motivated the issue. Co-Authored-By: Claude Fable 5 --- tests/test_sshfs.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_sshfs.py b/tests/test_sshfs.py index 5635b88..9bd3242 100644 --- a/tests/test_sshfs.py +++ b/tests/test_sshfs.py @@ -115,6 +115,28 @@ def test_sftp_client_kwargs(ssh_server, base_remote_dir, user="user"): assert fs.exists(file) +def test_sftp_client_kwargs_path_encoding( + ssh_server, base_remote_dir, user="user" +): + # path_encoding=None makes asyncssh deliver remote paths as raw + # bytes instead of str, for servers with non-UTF-8 file names (#39). + fs = SSHFileSystem( + host=ssh_server.host, + port=ssh_server.port, + username=user, + client_keys=[USERS[user]], + sftp_client_kwargs={"path_encoding": None}, + ) + + directory = Path(base_remote_dir) / "path_encoding_probe" + directory.mkdir() + (directory / "data.txt").write_bytes(b"data") + + encoded = str(directory).encode() + assert fs.ls(encoded) == [encoded + b"/data.txt"] + assert fs.cat_file(encoded + b"/data.txt") == b"data" + + def test_info(fs, remote_dir): fs.touch(remote_dir + "/a.txt") details = fs.info(remote_dir + "/a.txt")