import os from vector_utils.connections.sftp import SftpConnection from vector_utils.connections.connection_info import ConnectionInfo # This is a test sftp server (gets spun up with lambda-fb-sr-delivery local container) # and the private key already exists in version control priv_key = """ -----BEGIN OPENSSH PRIVATE KEY----- b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn NhAAAAAwEAAQAAAgEAnOytgjEWsSNwFhhjPnrTBjANk8GTxCLUn2CK66lbefu+R/dR1ruM Psz5mOhFtzA3cib4sTn+ZTkYHaUu+KDRUg23Qrazttwq2MyVFO4TlKtPvV8IycJRInQF7n 49u9KzIri1D2S0pVX+a74CaRr7AzIbyJkJHeq+dV6pMgSqse8wgGxr0jjGW/ejbQVoNi6E CpEbhuDvLr/W54pASNZnC6hZFF4NMUBVHtMoF7Ai73ouIPJrX7ALwNeQzzgCy1wl9Ks6jS yc+dphRzaVYU0nWNiRfx768GhrwsY9OIXFwr4lMxyIgYpYD7TTrYsS0C8f0xoQKehpN1JM QsbXd4QnYa+oc0LuR0N6u55ktP0tKULAqiN91G0q96nFWblGBaEX/yXWaKzXCLYr8qdsDb uHIl71xNmvoqW4i7TCpLLRajI11miJBTjb/AvpmJrzat3EbPXd/teb0Eb6uNWxIXyuLc/H NbUCXApEhpjQTudp461m2yZfWK0ENW0xzK9qMdqiy7chYchA0bANUQo5YZRYGG+wNRtPyf 59+F5H5XteYkMBVUyJTPs1clEt3PVaQ3i4dJPHlC121JLIS+z2ZOEVfHAEgm76FFG8Q530 VEPEgDWl9cGXi1ZFPhiSNSDmFvxDWDuH2K/JeWRcqOAhiUuhTQhH+TtsieEVBZffVNTWlM EAAAdYPZCLXz2Qi18AAAAHc3NoLXJzYQAAAgEAnOytgjEWsSNwFhhjPnrTBjANk8GTxCLU n2CK66lbefu+R/dR1ruMPsz5mOhFtzA3cib4sTn+ZTkYHaUu+KDRUg23Qrazttwq2MyVFO 4TlKtPvV8IycJRInQF7n49u9KzIri1D2S0pVX+a74CaRr7AzIbyJkJHeq+dV6pMgSqse8w gGxr0jjGW/ejbQVoNi6ECpEbhuDvLr/W54pASNZnC6hZFF4NMUBVHtMoF7Ai73ouIPJrX7 ALwNeQzzgCy1wl9Ks6jSyc+dphRzaVYU0nWNiRfx768GhrwsY9OIXFwr4lMxyIgYpYD7TT rYsS0C8f0xoQKehpN1JMQsbXd4QnYa+oc0LuR0N6u55ktP0tKULAqiN91G0q96nFWblGBa EX/yXWaKzXCLYr8qdsDbuHIl71xNmvoqW4i7TCpLLRajI11miJBTjb/AvpmJrzat3EbPXd /teb0Eb6uNWxIXyuLc/HNbUCXApEhpjQTudp461m2yZfWK0ENW0xzK9qMdqiy7chYchA0b ANUQo5YZRYGG+wNRtPyf59+F5H5XteYkMBVUyJTPs1clEt3PVaQ3i4dJPHlC121JLIS+z2 ZOEVfHAEgm76FFG8Q530VEPEgDWl9cGXi1ZFPhiSNSDmFvxDWDuH2K/JeWRcqOAhiUuhTQ hH+TtsieEVBZffVNTWlMEAAAADAQABAAACAQCASNvvOsIaPaY76eP4B+DLhi0TRYTPKso3 IG1xChpGNo7vtjepG9ki3d6OKhQUFc69+y4DdC2MAoi8c3Vau91i/uN6HnH8YzCD23tkiN xpBUNCxJYimkBJsh/HQFynVzAqimU6GFG1d/9thP/jfLMHgcdEzhat5mGdAa2/viDtJWS5 F+ltCgze0is/zNPMt2oqWUXvlaLyzNGap2K2KNBEr3NjOx/9y2HKuURlWRC2RWPAe21b5E c3dwJi5J8kMEi3I+PR1U7e0rADz5N+39vyvdjz2xe5wSOXLwiNVI6/DULL7d+KRtaGwITr sm3G0z9pDD16hQy+tQCy9mrewaKO6Zn3yW8apv+kObAAaD50YDguVTYgux6dSvHNf4Qhc1 UyVMDSKVzIhOaVQfkWBvfse9viGcE3a+dpvhx9tiItZnX/mrOcu0Y5o6vpolC3CGLENof7 2PyI1/t+E7IBoom4dC8p1yGhMPSTONClkosi/lqV3O9SwkzThshp9xuczFofVSSj/GnbBV A7fPWNTj/icoU/fvbc9FsMsGOCjbrx75jC5YkTibcA7vf45APLj2Wu/NNs9S1bfDM4enpK 6VGcJFt9HHsV6MyUwtstmENqApG5Rk8OnykvPaBeL72oC2ZTMZa/BRWe3yq3WE7XUIzHfd +IrLHuh/+2LTj0YbGZIQAAAQBnag2ZTzJB+I+amh10uPRsMfgWdomOY8Cmzuf1KD6YiCmC Gg2Gh+MDPFJiD2U1rEzUqJ4x9uuB43va7M85CImjzpjCLZHnQpVSo9J0bim1b1wtnylcDB ePPhnXJCbz8rOLUuefTJfOlhJkYLMb2SWagPENW+6XwE1D99McJbWPhqBPkI3GXqK/HpEv ybBsX1cUn0T5DyGRblgZhict1JVZEHHYuXqEeggNZF5RINQzBnayySev+PKCpj82dbRL/p lUFOAgnKthNH00/JSX19mv73CsQkrcGFLXTO3or4JjrRehpG1hhjQGunoBMlCYl1QvR0YJ Gek4qw7ddQ9P1BO6AAABAQDKaRogmiAcUSJesEE3cIGUYa058npYr4uFOMy2O3s7Q7NQUk SxyTlH0L+IE43y1kR9T5p43qI4hgISkppTJQPFIq189D9NnPEHJxJkBEFVU+JvszHgfcF9 35MUKiWOhzkf5JBv+nExKT7TKLW0OY2S50m5ryUXYaEr8LuRpzH7dQiuh9JU+AYHvcevWi QkBBvhUG2xKTmr7vOtLCwX8i+JEjCKfU5iANj412l22Vh9+3/BlTxsrTfR7lPwFi+P63Mi HT8fBdaI/BMfsF5CrEHJXKb22xdCLN6z/x4VW97PTz/xGGYg7adQRzXjCYvFU7q+b4jsgo fV7aU6o7m10SMPAAABAQDGeKR2ICVdjBeA0mlV5JRExNRAkFTCuDz234mafC6emflyknE5 opPAi+14zXiPGA+E32ZT9OPsdSOBXCKYtQ9/N0rwJyS5k5Ncpz5lcbKlBiyYr9Dl4Ck3Z+ BhKLFxtdSgGYwzlVBnAy2PvMiL/A4KTVGgpqV9jfxTxmVUoLEqG2VOiNmkYH+Xb/yU7rXy ihyAbE62/i2lR6Uvyd47fQTrWPrLPYfHS0EhBS7lxJ5IXouv+QiLt/yntHJSfS1Pf7aN+N cEnArZNtrUz0EtsyABMoG1exn8+CfbJ1nJixmDY/N6vmtuABQIpYfE6PlcYz7UCZDu4LQh LjB7SF2kGIsvAAAAH3BkdW1vdWxpbkBVU09DMDJaVzBHRkxWRE4ubG9jYWwBAgM= -----END OPENSSH PRIVATE KEY----- """ def _sftp_conn_info(): return ConnectionInfo( { 'authenticate_type': 'priv_key_in_mem', 'connection_type': 'sftp', 'domain_name':'localhost', 'user_name': 'orchard', 'priv_key': priv_key, 'port': 2223 }) sftp_conn = SftpConnection(_sftp_conn_info()) print('successfully connected!') def download_files(): top_level_remote_path = ('/test-fingerprinting/202109142023166502') top_level_remote_dir = sftp_conn.scan_dir(top_level_remote_path) top_level_local_path = ('./mags-local-test-fingerprinting/202109142023166502') for item in top_level_remote_dir: if item[-3:] == 'xml' or item[-3:] == 'wav': sftp_conn.transfer_files([ {'remote': f'{top_level_remote_path}/{item}', 'local':f'{top_level_local_path}/{item}' }], transfer_mode='download') else: next_level_local_path = f'{top_level_local_path}/{item}' os.mkdir(next_level_local_path) next_level_remote_path = f'{top_level_remote_path}/{item}' next_level_remote_dir = sftp_conn.scan_dir(next_level_remote_path) for thing in next_level_remote_dir: if thing[-3:] == 'xml' or thing[-3:] == 'wav': sftp_conn.transfer_files([{ 'remote': f'{next_level_remote_path}/{thing}', 'local':f'{next_level_local_path}/{thing}' }], transfer_mode='download') else: last_level_local_path = f'{next_level_local_path}/{thing}' os.mkdir(last_level_local_path) last_level_remote_path = f'{next_level_remote_path}/{thing}' last_level_remote_dir = sftp_conn.scan_dir(last_level_remote_path) for file in last_level_remote_dir: if file[-3:] == 'xml' or file[-3:] == 'wav': sftp_conn.transfer_files([{ 'remote': f'{last_level_remote_path}/{file}', 'local':f'{last_level_local_path}/{file}' }], transfer_mode='download') # download_files() def upload_files(): sftp_conn.mkdir('/fingerprinting/202110051522166502') top_level_remote_path = ('/fingerprinting/202110051522166502') top_level_local_path = ('./mags-local-test-fingerprinting/202110051522166502') top_level_local_dir = os.listdir(top_level_local_path) print('top_level_local_dir', top_level_local_dir) for item in top_level_local_dir: if item[-3:] == 'xml' or item[-3:] == 'wav': sftp_conn.transfer_files([ {'remote': f'{top_level_remote_path}/{item}', 'local':f'{top_level_local_path}/{item}' }], transfer_mode='upload') else: next_level_remote_path = f'{top_level_remote_path}/{item}' sftp_conn.mkdir(next_level_remote_path) next_level_local_path = f'{top_level_local_path}/{item}' next_level_local_dir = os.listdir(next_level_local_path) for thing in next_level_local_dir: if thing[-3:] == 'xml' or thing[-3:] == 'wav': sftp_conn.transfer_files([{ 'remote': f'{next_level_remote_path}/{thing}', 'local':f'{next_level_local_path}/{thing}' }], transfer_mode='upload') else: last_level_remote_path = f'{next_level_remote_path}/{thing}' sftp_conn.mkdir(last_level_remote_path) last_level_local_path = f'{next_level_local_path}/{thing}' last_level_local_dir = os.listdir(last_level_local_path) for file in last_level_local_dir: if file[-3:] == 'xml' or file[-3:] == 'wav': sftp_conn.transfer_files([{ 'remote': f'{last_level_remote_path}/{file}', 'local':f'{last_level_local_path}/{file}' }], transfer_mode='upload') # upload_files()