Directory transfers with Paramiko
How do you use Paramiko to transfer complete directories? I'm trying to use:
sftp.put("/Folder1","/Folder2")
which is giving me this error -
Error : [Errno 21] Is a 开发者_运维技巧directory
You can subclass paramiko.SFTPClient and add the following method to it:
import paramiko
import os
class MySFTPClient(paramiko.SFTPClient):
def put_dir(self, source, target):
''' Uploads the contents of the source directory to the target path. The
target directory needs to exists. All subdirectories in source are
created under target.
'''
for item in os.listdir(source):
if os.path.isfile(os.path.join(source, item)):
self.put(os.path.join(source, item), '%s/%s' % (target, item))
else:
self.mkdir('%s/%s' % (target, item), ignore_existing=True)
self.put_dir(os.path.join(source, item), '%s/%s' % (target, item))
def mkdir(self, path, mode=511, ignore_existing=False):
''' Augments mkdir by adding an option to not fail if the folder exists '''
try:
super(MySFTPClient, self).mkdir(path, mode)
except IOError:
if ignore_existing:
pass
else:
raise
To use it:
transport = paramiko.Transport((HOST, PORT))
transport.connect(username=USERNAME, password=PASSWORD)
sftp = MySFTPClient.from_transport(transport)
sftp.mkdir(target_path, ignore_existing=True)
sftp.put_dir(source_path, target_path)
sftp.close()
You'll need to do this just like you would locally with python (if you weren't using shutils).
Combine os.walk()
, with sftp.mkdir()
and sftp.put()
. You may also want to check each file and directory with os.path.islink()
depending on whether you want to resolve symlinks or not.
Here's my piece of code:
import errno
import os
import stat
def download_files(sftp_client, remote_dir, local_dir):
if not exists_remote(sftp_client, remote_dir):
return
if not os.path.exists(local_dir):
os.mkdir(local_dir)
for filename in sftp_client.listdir(remote_dir):
if stat.S_ISDIR(sftp_client.stat(remote_dir + filename).st_mode):
# uses '/' path delimiter for remote server
download_files(sftp_client, remote_dir + filename + '/', os.path.join(local_dir, filename))
else:
if not os.path.isfile(os.path.join(local_dir, filename)):
sftp_client.get(remote_dir + filename, os.path.join(local_dir, filename))
def exists_remote(sftp_client, path):
try:
sftp_client.stat(path)
except IOError, e:
if e.errno == errno.ENOENT:
return False
raise
else:
return True
This can all be done quite easily using just paramiko.
A high level summary of the code below is:
- connect to the SFTP (steps 1 to 3)
- specify your source and target folders. (step 4)
- copy them over one by one to wherever you like (I've sent them to /tmp/). (step 5)
import paramiko
# 1 - Open a transport
host="your-host-name"
port = port_number
transport = paramiko.Transport((host, port))
# 2 - Auth
password="sftp_password"
username="sftp_username"
transport.connect(username = username, password = password)
# 3 - Go!
sftp = paramiko.SFTPClient.from_transport(transport)
# 4 - Specify your source and target folders.
source_folder="some/folder/path/on/sftp"
inbound_files=sftp.listdir(source_folder)
# 5 - Download all files from that path
for file in inbound_files :
filepath = source_folder+file
localpath = "/tmp/"+file
sftp.get(filepath, localpath)
Works for me doing something like this, all folder and files are copied to the remote server.
parent = os.path.expanduser("~")
for dirpath, dirnames, filenames in os.walk(parent):
remote_path = os.path.join(remote_location, dirpath[len(parent)+1:])
try:
ftp.listdir(remote_path)
except IOError:
ftp.mkdir(remote_path)
for filename in filenames:
ftp.put(os.path.join(dirpath, filename), os.path.join(remote_path, filename))
You might replace sftp = self.client.open_sftp()
with paramiko's one and get rid of libcloud
here.
import os.path
from stat import S_ISDIR
from libcloud.compute.ssh import SSHClient
from paramiko.sftp import SFTPError
class CloudSSHClient(SSHClient):
@staticmethod
def normalize_dirpath(dirpath):
while dirpath.endswith("/"):
dirpath = dirpath[:-1]
return dirpath
def mkdir(self, sftp, remotepath, mode=0777, intermediate=False):
remotepath = self.normalize_dirpath(remotepath)
if intermediate:
try:
sftp.mkdir(remotepath, mode=mode)
except IOError, e:
self.mkdir(sftp, remotepath.rsplit("/", 1)[0], mode=mode,
intermediate=True)
return sftp.mkdir(remotepath, mode=mode)
else:
sftp.mkdir(remotepath, mode=mode)
def put_dir_recursively(self, localpath, remotepath, preserve_perm=True):
"upload local directory to remote recursively"
assert remotepath.startswith("/"), "%s must be absolute path" % remotepath
# normalize
localpath = self.normalize_dirpath(localpath)
remotepath = self.normalize_dirpath(remotepath)
sftp = self.client.open_sftp()
try:
sftp.chdir(remotepath)
localsuffix = localpath.rsplit("/", 1)[1]
remotesuffix = remotepath.rsplit("/", 1)[1]
if localsuffix != remotesuffix:
remotepath = os.path.join(remotepath, localsuffix)
except IOError, e:
pass
for root, dirs, fls in os.walk(localpath):
prefix = os.path.commonprefix([localpath, root])
suffix = root.split(prefix, 1)[1]
if suffix.startswith("/"):
suffix = suffix[1:]
remroot = os.path.join(remotepath, suffix)
try:
sftp.chdir(remroot)
except IOError, e:
if preserve_perm:
mode = os.stat(root).st_mode & 0777
else:
mode = 0777
self.mkdir(sftp, remroot, mode=mode, intermediate=True)
sftp.chdir(remroot)
for f in fls:
remfile = os.path.join(remroot, f)
localfile = os.path.join(root, f)
sftp.put(localfile, remfile)
if preserve_perm:
sftp.chmod(remfile, os.stat(localfile).st_mode & 0777)
I don't think you can do that. Look up the documentation for os.walk
and copy each file "manually".
This is my first StackOverflow answer. I had a task today which is similar to this. So, I tried to find a direct way to copy entire folder from windows to linux using python and paramiko. After a little research, I came up with this solution which works for smaller size folders with subfolders and files in it.
This solution first makes the zip file for the current folder (os.walk() is very much helpful here), then copies to destination server and unzip there.
zipHere = zipfile.ZipFile("file_to_copy.zip", "w")
for root, folders, files in os.walk(FILE_TO_COPY_PATH):
for file in files:
zipHere.write(os.path.join(root, file), arcname=os.path.join(os.path.relpath(root, os.path.dirname(FILE_TO_COPY_PATH)), file))
for folder in folders:
zipHere.write(os.path.join(root, folder), arcname=os.path.join(os.path.relpath(root, os.path.dirname(FILE_TO_COPY_PATH)), folder))
zipHere.close()
# sftp is the paramiko.SFTPClient connection
sftp.put('local_zip_file_location','remote_zip_file_location')
# telnet_conn is the telnetlib.Telnet connection
telnet_conn.write('cd cd_to_zip_file_location')
telnet_conn.write('unzip -o file_to_copy.zip')
Paramiko does not support directory transfers on its own. You have to implement it, as many existing answers here show.
Or you can use pysftp. It's a wrapper around Paramiko that has more Python-ish look and feel and supports recursive operations. See
pysftp.Connection.put_r()
pysftp.Connection.get_r()
Or you can just base your code on pysftp source code. Full a standalone portable Paramiko-only code see my answers to:
- Python pysftp get_r from Linux works fine on Linux but not on Windows
- Python pysftp put_r does not work on Windows
And as my answers above show, you actually have to use your own code, if you are on Windows, as pysftp does not work there.
my answer is similar with above just make a list, and then transfer one by one.
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname='11.11.11.1111', username='root', password='********', port=22)
sftp_client = ssh.open_sftp()
source_folder = '/var/ftp/file_pass'
local_folder = 'C:/temp/file_pass'
inbound_files = sftp_client.listdir(source_folder)
print(inbound_files)
for ele in inbound_files:
try:
path_from = source_folder + '/' + ele
path_to = local_folder + '/'+ ele
sftp_client.get(path_from, path_to)
except:
print(ele)
sftp_client.close()
ssh.close()
This is my approach but and the code handle hidden files also
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect("localhost", port=19000, username="test", password="test")
sftp = ssh.open_sftp()
source_folder="/etc/"
inbound_files=sftp.listdir(source_folder)
for file in inbound_files :
filepath = source_folder+file
localpath = "/home/"+file
# check hidden files
if file.startswith('.'):
pass
else:
sftp.get(filepath, localpath)
As far as I know, Paramiko does not support recursive file upload. However, I have found a solution for recursive upload using Paramiko here. Follows an excerpt of their recursive upload function:
def _send_recursive(self, files):
for base in files:
lastdir = base
for root, dirs, fls in os.walk(base):
# pop back out to the next dir in the walk
while lastdir != os.path.commonprefix([lastdir, root]):
self._send_popd()
lastdir = os.path.split(lastdir)[0]
self._send_pushd(root)
lastdir = root
self._send_files([os.path.join(root, f) for f in fls])
You may try to either use their function SCPClient.put
invoking the above function for recursive upload or implement it on your own.
If you would like to have parallel copy per folder you can use (keep in mind that it will ignore files that already exists localy):
def get_folders(sftp_client, remote_dir, local_dir):
if not exists_remote(sftp_client, remote_dir):
return
if not os.path.exists(local_dir):
os.mkdir(local_dir)
for filename in sftp_client.listdir(remote_dir):
remote_dir_path = f"{remote_dir}/{filename}"
print(f"downloading {remote_dir_path}")
current_stat = sftp_client.stat(remote_dir_path)
if stat.S_ISDIR(current_stat.st_mode):
get_folders(sftp_client, remote_dir_path, os.path.join(local_dir, filename))
else:
if not os.path.isfile(os.path.join(local_dir, filename)):
sftp_client.get(remote_dir_path, os.path.join(local_dir, filename))
def exists_remote(sftp_client, path):
try:
sftp_client.stat(path)
except IOError as e:
if e.errno == errno.ENOENT:
return False
raise
else:
return True
def copy_from_server(dir_path):
import paramiko
server = "A"
username = "B"
password = "C"
remote_path = ""
local_path = ""
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server, username=username, password=password)
sftp = ssh.open_sftp()
print("connected")
get_folders(sftp, f"{remote_path}/{dir_path}",
rf"{local_path}\{dir_path}")
print("downloaded")
sftp.close()
ssh.close()
def parallel_copy_from_server():
dir_names = ['A', 'B']
NUM_OF_CPUS = multiprocessing.cpu_count()
with Pool(NUM_OF_CPUS) as p:
results_list = p.map(copy_from_server, dir_names)
精彩评论