Hi:
It is a practical matter for me, that when I use FTP programmatically, that
I frequently to multiple uploads.
To that end, I have made the following changes to the ftp module,
(define (transfer user-id password host subdir file-names mode)
(if (string? file-names)(set 'file-names(list file-names)))
as the first two lines of function 'transfer
and the following change to the 'PUT option:
(if (= mode PUT)
(and
(dolist (file-name file-names)
(println "<BR>file-name = " file-name) ;; DEBUG file-name
(check-file file-name)
(net-send socket (append "STOR " file-name "rn"))
(send-get-result "STATrn" "1")
(println "opening file")
(set 'fle (open file-name "r"))
(while (> (read-buffer fle 'buffer 512) 0)
(if debug-mode (print "."))
(net-send socket2 buffer 512))
(close fle)
(println "file closed")
)) true)
I then make the following function call:
(set 'FTP:debug-mode true)
(set 'res(FTP:put "*user*" "*pwd*" "www.johnsons-web.com" "/demo/newlisp"
'("usr.lsp" "testcgi.dbg"))
The results follow:
file-names = ("usr.lsp" "testcgi.dbg")
sent:USER *user*
331 Please specify the password.
sent:PASS *pwd*
230 Login successful. Have fun.
sent:CWD /demo/newlisp
250 Directory successfully changed.
sent:TYPE I
200 Switching to Binary mode.
sent:PASV
227 Entering Passive Mode (12,24,138,43,204,222)
file-name = usr.lsp
sent:STAT
150 Ok to send data.
opening file
........file closed
file-name = testcgi.dbg
sent:STAT
The result is that the transmission "hangs" after the first
send and prior to the second call to 'open
:-) If we get this one working, I promise not to hack any more
of Lutz's code for at least a week or until I learn a bit more about
TCP/IP
Thanks
Tim
My guess is that this isn't a newLISP problem but something to do with the ftp protocol/handshaking stuff.
To send multiple files in ftp you use
By the way - is that 'Rudy Rucker' in the credits for the ftp module the same as the famous Rudy Rucker (I have a book or two of his...).
Oops, its Eddie Rucker, a professor at http://www.bmc.edu/ , not Rudy, that was a typo :-( , and has now been corrected:
http://newlisp.org/code/modules/ftp.lsp.html
I've been using my own python object to do multiple file uploads for many years
now.
First: Code for the object initializer:
class FtpPro(FTP):
""" FTP transfer handler """
def __init__(self,**kw):
""" Takes a dictionary with host, login, password and
optional account, working directory and transfer
settings. """
self.CRLF = 'rn'
self.__class_name__ = "FtpPro"
self.host = ''
self.pwd = ''
self.user = ''
self.dir = ''
self.pasv = None
self.fp = None
self.src = None ## NOT USED YET
self.report = None # if None report bytes transfered, else print
for k in kw.keys():
if self.__dict__.has_key(k):
self.__dict__[k] = kw[k]
else:
raise KeyError('%s is not an acceptable keyword for the FtpPro class' % `k`)
self.connect(self.host)
self.login(self.user,self.pwd)
if self.pasv != None:
self.passiveserver = self.pasv
if self.dir:
self.cwd(self.dir)
#---------------------------------------------------------------------------------------------------
def Upload(self,file_names,type=None):
""" Upload with list of filenames. Default type is text, if is true upload as binary """
if type:
self.send_binary_files(file_names)
else:
self.send_text_files(file_names)
#---------------------------------------------------------------------------------------------------
def send_binary_files(self,file_names):
''' Send list of files in binary mode '''
if type(file_names) == type(''):
self.send_binary_file(file_names)
else:
for file_name in file_names:
self.send_binary_file(file_name)
#---------------------------------------------------------------------------------------------------
def send_binary_file(self,file_name,blocksize=892):
'''Store a file in binary mode. Report transfer progress'''
total_sent = tmp_sent = 0
if not os.path.exists(file_name):
self.prin(file_name + " does not exist")
return
increment,increments,blocksize,file_size = self.get_transfer_increment(file_name)
fp = open(file_name,'rb')
self.voidcmd('TYPE I')
conn = self.transfercmd('STOR ' + file_name)
if self.report:
sys.stdout.write('n %s ' % file_name)
else:
self.prin(' Sending %s As binary file:TOTAL[%d]' % (file_name,file_size))
while 1:
buf = fp.read(blocksize)
if not buf: break
tmp_sent += blocksize # keep track of current increment
total_sent += blocksize # keep track of total sent
conn.send(buf)
if tmp_sent >= increment: # report progress
self.prn(total_sent) # number and a comma
tmp_sent = 0
conn.close()
self.prn(' DONEn')
return self.voidresp()
And lastly the calling code:
uploader = FtpPro.FtpPro(**D["ftp"])
uploader.Upload(files,"b")
And of course mput can do multiple uploads.
See
Might be a clue there.
:-) I'm sure it can be done in newlisp
Tim
Oh, here's some more - sorry, but python can be a bit of a black box with
all of the inherited classes. The following is from the ftplib module in
the standard distribution, which is inherited by my code above.
def voidcmd(self, cmd):
"""Send a command and expect a response beginning with '2'."""
self.putcmd(cmd)
return self.voidresp()
##================================
def voidresp(self):
"""Expect a response beginning with '2'."""
resp = self.getresp()
if resp[0] != '2':
raise error_reply, resp
return resp
Perhaps this sheds some light where the protocols are addressed.
I'm not kidding when I say I'm pretty much a bonehead when it comes
to socket programming. What I do is so far and few between that I
forget from one time to the other.
regards
Tim
Quote from: "cormullion"
To send multiple files in ftp you use
Sorry about that.... just realized I forgot to send an answer to your question.
Yes mput works fine - with and without globbing - same site, same subdir,
same files.
Tim
I've started on an FTP module... But at the rate of two functions a day, and given that I've never programmed at this level before, it's going to take a while. :)
Quote from: "cormullion"
Ha! Join the club! I'm guessing it will be a couple of months before I've
got the newlisp resources put together to meet my own standards for
large web projects - but I'm looking forward to that stage.
tj
My attempt to write another version of FTP has hit the same problem with sending multiple files as Lutz's original one does.
I think that it has something to do with passive mode. Passive mode is the separate data channel that gets set up by the server. If after sending a file I close the connection to the second socket I can get a STAT response on the first connection, but I can't open the second one again, and I can't enter passive mode again either, because it doesn't work. I wonder whether it's some restriction I don't know about, like a lock or something.
Although I don't believe that you can only upload one file at a time, at present I can't see how to do more than one. There's no MPUT command in the FTP protocol.
Here's the output from my code's attempt to upload two files one after the other. The first one is OK, the second one can't start. The 425 error is misleading, I think, because it's the absence of an open connection that it's complaining about.
331 Password required for xxx
230 User xxx logged in
250 CWD command successful
200 Type set to I
227 Entering Passive Mode (1,2,3,4,5,6).
150 Opening BINARY mode data connection for file1.lsp
226 Transfer complete
226
211-Status of 'ProFTPD'
Connected from 1.2.3.4 (1.2.3.4)
Logged in as xxx
TYPE: BINARY, STRUcture: File, Mode: Stream
Total bytes transferred for session: 1815
No data connection
211 End of status
425 Unable to build data connection: File name too long
221 Goodbye.
I could be completely wrong - this is not my area of expertise! :)
Since this can be done in python, and some of the python connection
code is native C code, it might be worth investigating whether the
native C code for newlisp needs to be changed.
Lutz would know.
I added the following modifications to ftp.lsp
(set 'progress-mode nil)
(set 'progress-marker ">")
;; .......
;; @syntax (FTP:mput <str-user-id> <str-password> <str-host> <str-dir> <str-file-names>)
;; @param <(ID password host dir vargs...)> as FTP:put with any number of filenames as varg list
;; @return 'true' on success, 'nil' on failure.
;; @example
;; (FTP:put "somebody" "secret" "host.com" "subdir" "file1" "file2" ....) ;; upload
(define (mput user-id password host subdir)
(let ((profile (list user-id password host subdir "")))
(doargs (file-name)
(if progress-mode (print file-name))
(set-nth (+ -1 (length profile)) profile file-name)
(apply put profile)
(if progress-mode(print "n")))))
;; ......... In the PUT sexp
(while (> (read-buffer fle 'buffer 512) 0)
(if (or debug-mode progress-mode)
(print progress-marker))
Works for me, for my purposes. I'm going to research the
pros and cons of dynamically managing the packets size also.
Tim
In your previous Python posts, I thought I noticed that the connection was opened and closed for each file. Since I don't speak Python, it's hard to say... Perhaps that's how to do it...?
Quote from: "cormullion"
The connection is in the object initializer - which is only called once.
The transfer is in the send_binary_file() function which is called once for
each file. See my code above.
Below is dump for ftp/mput
230 Login successful. Have fun.
Remote system type is UNIX.
Using binary mode to transfer files.
ftp> mput usr.lsp testcgi.dbg
local: usr.lsp remote: usr.lsp
227 Entering Passive Mode (12,24,138,43,232,118)
150 Ok to send data.
226 File receive OK.
3694 bytes sent in 0.00 secs (34032.3 kB/s)
local: testcgi.dbg remote: testcgi.dbg
227 Entering Passive Mode (12,24,138,43,126,152)
150 Ok to send data.
226 File receive OK.
1646 bytes sent in 0.00 secs (19136.0 kB/s)
ftp> quit
Ah yes - I've got it now. There needs to be a switch to passive mode before each upload, and a confirmation after closing the data connection.
If you fancy taking over and doing some coding, Tim, I'll post what I've done so far - tomorrow probably because it's time to sleep now... !
Quote from: "cormullion"
If you fancy taking over and doing some coding, Tim, I'll post what I've done so far - tomorrow probably because it's time to sleep now... !
:-) Beam it up to me Scotty.
tj
Here's how far I've got:
//http://unbalanced-parentheses.nfshost.com/downloads/ftp.lsp
Needs lots of work, of course, but the dilithium crystals are transferring small amounts of power to the internet, so there's some glimmers of hope.
The elegant bits are pinched from Lutz and Eddie's original - the rest is assembled from pieces of code from anyone and everyone. In that sense alone it's written in assembly language... :)
I've saved it. Just got some more work orders in so I may not have
time until the weekend. I hate it when coding work gets in the
way of coding fun....
Thanks
Tim
Tried it with two files. Works! I made one change:
Removed the space after 'I' - was getting an 'unrecognized type command'
message - probably a difference in the way that the FTP server provided
responses.
Still too busy here to test further 'til weekend, but I think you
solved it.
Thanks
Good Work!
That's encouraging. At least it works - which is the first step. Look forward to seeing your improvements next week...! :)
I noticed one piece of ugly code that can be simplified:
(integer? (int (response 1)))
(integer? (int (response 2)))
(= " " (response 3)))
better as:
(= " " (response 3)))
My next step in building my programming platform is a dependency manager,
which will not only handle uploads to remote servers but resolve (and upload) any dependencies, so I will be giving this a run for the money.
One thought occurs to me:
I'd like to see this tested by others to verify that your code is not "server-dependent" so I would urge others to try your code.
And regardless that this is on a *n?x forum, it would be great if
it turns out to be OS - independent.
Thanks Again
Tim
The
; can download f to fname to avoid overwriting local copy of f
(local (local-file)
(and
(if (empty? fname)
(set 'fname f)
true)
(passive)
(unless (positive-preliminary-response? (execute-command-return-code (string "retr " f)))
(begin
(pp "file not found")
(net-close connection-socket2)
(execute-command-return-code "stat"))
(begin
(set 'local-file (open fname "w"))
(while (net-receive connection-socket2 'buffer 512)
(begin
(write-buffer local-file buffer)
(print ".")))
(close local-file) ; close file
(net-close connection-socket2)
; get status of transfer
(positive-complete-response? (get-full-server-reply)))))))
starting to get lots of 'begins' though, which is usually the first sign that my code is getting out of hand ("Begin the Beguine")...
There can also be a
(and
(connect host)
(login user-name password working-dir)
(map get-files (args))
(logout)
(disconnect)))
but I'm not sure whether you can use
Hi cormullion:
This code
(until (and (int (0 2 response))
(= " " (response 3)))
I've done documentation, added reporting for the data transfer,
seperated from the command reporting and newlisp doc syntax.
I'll shoot you a copy of the changed file "end of business" tomorrow.
I'm going to implement it as part of a remote server updating scheme,
that should provide some more testing.
Here's a piece of what I added. Sort of pythonish:
(set 'connection-socket1 nil
'logged-in? nil
'quiet true
'report true
'history nil
'intro "Connected to host. Upload follows:n"
'report-file "Sending: "
'packet-marker "."
'control-vars '("quiet" "report" "intro" "report-file" "packet-marker")
)
;; @syntax (FTP:config ...<quoted-var1> <val1> <quoted-var1> <val2> ...)
;; @description Sets control variables. Throws error if var isn't "registered"
;; @example
;; (FTP:config 'quiet 1 'report-file "Uploading: ") => changes FTP:quiet and FTP:report-file
(define (config)
(dolist (keyval (explode (args) 2))
(_config (keyval 0) (keyval 1))))
## Set a control variable, if allowable
(define (_config k val)
(let ((key (last(parse(string k)":")))) ;; remove context prefix
(if (not(find key control-vars)) ;; set only allowed vars
(raise (append "key: ['" key "] not in 'control-vars")))
(set (sym key) val)))
## Raise an error
(define (raise msg)
(throw-error (append "(FTP context) " msg)))
Cheers
Tim