diff --git a/autoload/vital/__vital__/Web/AsyncHTTP.vim b/autoload/vital/__vital__/Web/AsyncHTTP.vim new file mode 100644 index 000000000..77c9004c4 --- /dev/null +++ b/autoload/vital/__vital__/Web/AsyncHTTP.vim @@ -0,0 +1,376 @@ +let s:save_cpo = &cpo +set cpo&vim + +function! s:_vital_loaded(V) abort + let s:V = a:V + let s:Prelude = s:V.import('Prelude') + let s:HTTP = s:V.import('Web.HTTP') + let s:Core = s:V.import('Web.HTTP.Core') + let s:AsyncProcess = s:V.import('System.AsyncProcess') + let s:String = s:V.import('Data.String') +endfunction + +function! s:_vital_depends() abort + return { + \ 'modules':['Prelude', 'Web.HTTP', 'Web.HTTP.Core', 'Data.String', 'System.AsyncProcess'] , + \} +endfunction + +function! s:_request_cb(settings, responses, exit_code) abort + for file in values(a:settings._file) + if filereadable(file) + call delete(file) + endif + endfor + + call map(a:responses, 's:Core.build_response(v:val[0], v:val[1])') + let last_response = s:Core.build_last_response(a:responses) + if has_key(a:settings, 'user_cb') + call a:settings.user_cb(last_response) + endif +endfunction + +function! s:request(...) abort + let settings = s:Core.build_settings(a:000) + let settings.method = toupper(settings.method) + if !has_key(settings, 'url') + throw 'vital: Web.AsyncHTTP: "url" parameter is required.' + endif + if !s:Prelude.is_list(settings.client) + let settings.client = [settings.client] + endif + let client = s:_get_client(settings) + if empty(client) + throw 'vital: Web.AsyncHTTP: Available client not found: ' + \ . string(settings.client) + endif + if has_key(settings, 'contentType') + let settings.headers['Content-Type'] = settings.contentType + endif + if has_key(settings, 'param') + if s:Prelude.is_dict(settings.param) + let getdatastr = s:HTTP.encodeURI(settings.param) + else + let getdatastr = settings.param + endif + if strlen(getdatastr) + let settings.url .= '?' . getdatastr + endif + endif + if has_key(settings, 'data') + let settings.data = s:Core.postdata(settings.data) + let settings.headers['Content-Length'] = len(join(settings.data, "\n")) + endif + let settings._file = {} + + let responses = client.request(settings) +endfunction + +function! s:get(url, ...) abort + let settings = { + \ 'url': a:url, + \ 'param': a:0 > 0 ? a:1 : {}, + \ 'headers': a:0 > 1 ? a:2 : {}, + \ } + return s:request(settings) +endfunction + +function! s:post(url, ...) abort + let settings = { + \ 'url': a:url, + \ 'data': a:0 > 0 ? a:1 : {}, + \ 'headers': a:0 > 1 ? a:2 : {}, + \ 'method': a:0 > 2 ? a:3 : 'POST', + \ } + return s:request(settings) +endfunction + +" Clients +function! s:_get_client(settings) abort + for name in a:settings.client + if has_key(s:clients, name) && s:clients[name].available(a:settings) + return s:clients[name] + endif + endfor + return {} +endfunction + +" implements clients +let s:clients = {} + +let s:clients.curl = {} + +let s:clients.curl.errcode = {} +let s:clients.curl.errcode[1] = 'Unsupported protocol. This build of curl has no support for this protocol.' +let s:clients.curl.errcode[2] = 'Failed to initialize.' +let s:clients.curl.errcode[3] = 'URL malformed. The syntax was not correct.' +let s:clients.curl.errcode[4] = 'A feature or option that was needed to perform the desired request was not enabled or was explicitly disabled at buildtime. To make curl able to do this, you probably need another build of libcurl!' +let s:clients.curl.errcode[5] = 'Couldn''t resolve proxy. The given proxy host could not be resolved.' +let s:clients.curl.errcode[6] = 'Couldn''t resolve host. The given remote host was not resolved.' +let s:clients.curl.errcode[7] = 'Failed to connect to host.' +let s:clients.curl.errcode[8] = 'FTP weird server reply. The server sent data curl couldn''t parse.' +let s:clients.curl.errcode[9] = 'FTP access denied. The server denied login or denied access to the particular resource or directory you wanted to reach. Most often you tried to change to a directory that doesn''t exist on the server.' +let s:clients.curl.errcode[11] = 'FTP weird PASS reply. Curl couldn''t parse the reply sent to the PASS request.' +let s:clients.curl.errcode[13] = 'FTP weird PASV reply, Curl couldn''t parse the reply sent to the PASV request.' +let s:clients.curl.errcode[14] = 'FTP weird 227 format. Curl couldn''t parse the 227-line the server sent.' +let s:clients.curl.errcode[15] = 'FTP can''t get host. Couldn''t resolve the host IP we got in the 227-line.' +let s:clients.curl.errcode[17] = 'FTP couldn''t set binary. Couldn''t change transfer method to binary.' +let s:clients.curl.errcode[18] = 'Partial file. Only a part of the file was transferred.' +let s:clients.curl.errcode[19] = 'FTP couldn''t download/access the given file, the RETR (or similar) command failed.' +let s:clients.curl.errcode[21] = 'FTP quote error. A quote command returned error from the server.' +let s:clients.curl.errcode[22] = 'HTTP page not retrieved. The requested url was not found or returned another error with the HTTP error code being 400 or above. This return code only appears if -f, --fail is used.' +let s:clients.curl.errcode[23] = 'Write error. Curl couldn''t write data to a local filesystem or similar.' +let s:clients.curl.errcode[25] = 'FTP couldn''t STOR file. The server denied the STOR operation, used for FTP uploading.' +let s:clients.curl.errcode[26] = 'Read error. Various reading problems.' +let s:clients.curl.errcode[27] = 'Out of memory. A memory allocation request failed.' +let s:clients.curl.errcode[28] = 'Operation timeout. The specified time-out period was reached according to the conditions.' +let s:clients.curl.errcode[30] = 'FTP PORT failed. The PORT command failed. Not all FTP servers support the PORT command, try doing a transfer using PASV instead!' +let s:clients.curl.errcode[31] = 'FTP couldn''t use REST. The REST command failed. This command is used for resumed FTP transfers.' +let s:clients.curl.errcode[33] = 'HTTP range error. The range "command" didn''t work.' +let s:clients.curl.errcode[34] = 'HTTP post error. Internal post-request generation error.' +let s:clients.curl.errcode[35] = 'SSL connect error. The SSL handshaking failed.' +let s:clients.curl.errcode[36] = 'FTP bad download resume. Couldn''t continue an earlier aborted download.' +let s:clients.curl.errcode[37] = 'FILE couldn''t read file. Failed to open the file. Permissions?' +let s:clients.curl.errcode[38] = 'LDAP cannot bind. LDAP bind operation failed.' +let s:clients.curl.errcode[39] = 'LDAP search failed.' +let s:clients.curl.errcode[41] = 'Function not found. A required LDAP function was not found.' +let s:clients.curl.errcode[42] = 'Aborted by callback. An application told curl to abort the operation.' +let s:clients.curl.errcode[43] = 'Internal error. A function was called with a bad parameter.' +let s:clients.curl.errcode[45] = 'Interface error. A specified outgoing interface could not be used.' +let s:clients.curl.errcode[47] = 'Too many redirects. When following redirects, curl hit the maximum amount.' +let s:clients.curl.errcode[48] = 'Unknown option specified to libcurl. This indicates that you passed a weird option to curl that was passed on to libcurl and rejected. Read up in the manual!' +let s:clients.curl.errcode[49] = 'Malformed telnet option.' +let s:clients.curl.errcode[51] = 'The peer''s SSL certificate or SSH MD5 fingerprint was not OK.' +let s:clients.curl.errcode[52] = 'The server didn''t reply anything, which here is considered an error.' +let s:clients.curl.errcode[53] = 'SSL crypto engine not found.' +let s:clients.curl.errcode[54] = 'Cannot set SSL crypto engine as default.' +let s:clients.curl.errcode[55] = 'Failed sending network data.' +let s:clients.curl.errcode[56] = 'Failure in receiving network data.' +let s:clients.curl.errcode[58] = 'Problem with the local certificate.' +let s:clients.curl.errcode[59] = 'Couldn''t use specified SSL cipher.' +let s:clients.curl.errcode[60] = 'Peer certificate cannot be authenticated with known CA certificates.' +let s:clients.curl.errcode[61] = 'Unrecognized transfer encoding.' +let s:clients.curl.errcode[62] = 'Invalid LDAP URL.' +let s:clients.curl.errcode[63] = 'Maximum file size exceeded.' +let s:clients.curl.errcode[64] = 'Requested FTP SSL level failed.' +let s:clients.curl.errcode[65] = 'Sending the data requires a rewind that failed.' +let s:clients.curl.errcode[66] = 'Failed to initialise SSL Engine.' +let s:clients.curl.errcode[67] = 'The user name, password, or similar was not accepted and curl failed to log in.' +let s:clients.curl.errcode[68] = 'File not found on TFTP server.' +let s:clients.curl.errcode[69] = 'Permission problem on TFTP server.' +let s:clients.curl.errcode[70] = 'Out of disk space on TFTP server.' +let s:clients.curl.errcode[71] = 'Illegal TFTP operation.' +let s:clients.curl.errcode[72] = 'Unknown TFTP transfer ID.' +let s:clients.curl.errcode[73] = 'File already exists (TFTP).' +let s:clients.curl.errcode[74] = 'No such user (TFTP).' +let s:clients.curl.errcode[75] = 'Character conversion failed.' +let s:clients.curl.errcode[76] = 'Character conversion functions required.' +let s:clients.curl.errcode[77] = 'Problem with reading the SSL CA cert (path? access rights?).' +let s:clients.curl.errcode[78] = 'The resource referenced in the URL does not exist.' +let s:clients.curl.errcode[79] = 'An unspecified error occurred during the SSH session.' +let s:clients.curl.errcode[80] = 'Failed to shut down the SSL connection.' +let s:clients.curl.errcode[82] = 'Could not load CRL file, missing or wrong format (added in 7.19.0).' +let s:clients.curl.errcode[83] = 'Issuer check failed (added in 7.19.0).' +let s:clients.curl.errcode[84] = 'The FTP PRET command failed' +let s:clients.curl.errcode[85] = 'RTSP: mismatch of CSeq numbers' +let s:clients.curl.errcode[86] = 'RTSP: mismatch of Session Identifiers' +let s:clients.curl.errcode[87] = 'unable to parse FTP file list' +let s:clients.curl.errcode[88] = 'FTP chunk callback reported error' +let s:clients.curl.errcode[89] = 'No connection available, the session will be queued' +let s:clients.curl.errcode[90] = 'SSL public key does not matched pinned public key' + + +function! s:clients.curl.available(settings) abort + return executable(self._command(a:settings)) +endfunction + +function! s:clients.curl._command(settings) abort + return get(get(a:settings, 'command', {}), 'curl', 'curl') +endfunction + +function! s:_curl_cb(has_output_file, output_file, settings, exit_code) abort + let headerstr = s:Core.readfile(a:settings._file.header) + let header_chunks = split(headerstr, "\r\n\r\n") + let headers = map(header_chunks, 'split(v:val, "\r\n")') + if a:exit_code != 0 && empty(headers) + if has_key(s:clients.curl.errcode, a:exit_code) + throw 'vital: Web.AsyncHTTP: ' . s:clients.curl.errcode[a:exit_code] + else + throw 'vital: Web.AsyncHTTP: Unknown error code has occurred in curl: code=' . a:exit_code + endif + endif + if !empty(headers) + let responses = map(headers, '[v:val, ""]') + else + let responses = [[[], '']] + endif + if a:has_output_file || a:settings.method ==? 'HEAD' + let content = '' + else + let content = s:Core.readfile(a:output_file) + endif + let responses[-1][1] = content + + return s:_request_cb(a:settings, responses, a:exit_code) +endfunction + +function! s:clients.curl.request(settings) abort + let quote = s:Core.quote() + let command = self._command(a:settings) + if has_key(a:settings, 'unixSocket') + let command .= ' --unix-socket ' . quote . a:settings.unixSocket . quote + endif + let a:settings._file.header = s:Core.tempname() + let command .= ' --dump-header ' . quote . a:settings._file.header . quote + let has_output_file = has_key(a:settings, 'outputFile') + if has_output_file + let output_file = s:Core.file_resolve(a:settings.outputFile) + else + let output_file = s:Core.tempname() + let a:settings._file.content = output_file + endif + let command .= ' --output ' . quote . output_file . quote + if has_key(a:settings, 'gzipDecompress') && a:settings.gzipDecompress + let command .= ' --compressed ' + endif + let command .= ' -L -s -k ' + if a:settings.method ==? 'HEAD' + let command .= '--head' + else + let command .= '-X ' . a:settings.method + endif + let command .= ' --max-redirs ' . a:settings.maxRedirect + let command .= s:Core.make_header_args(a:settings.headers, '-H ', quote) + let timeout = get(a:settings, 'timeout', '') + let command .= ' --retry ' . a:settings.retry + if timeout =~# '^\d\+$' + let command .= ' --max-time ' . timeout + endif + if has_key(a:settings, 'username') + let auth = a:settings.username . ':' . get(a:settings, 'password', '') + let auth = escape(auth, quote) + if has_key(a:settings, 'authMethod') + if index(['basic', 'digest', 'ntlm', 'negotiate'], a:settings.authMethod) == -1 + throw 'vital: Web.AsyncHTTP: Invalid authorization method: ' . a:settings.authMethod + endif + let method = a:settings.authMethod + else + let method = 'anyauth' + endif + let command .= ' --' . method . ' --user ' . quote . auth . quote + endif + if has_key(a:settings, 'bearerToken') + \ && has_key(a:settings, 'authMethod') && (a:settings.authMethod ==? 'oauth2') + let command .= ' --oauth2-bearer ' . quote . a:settings.bearerToken . quote + endif + if has_key(a:settings, 'data') + let a:settings._file.post = s:Core.make_postfile(a:settings.data) + let command .= ' --data-binary @' . quote . a:settings._file.post . quote + endif + let command .= ' ' . quote . a:settings.url . quote + + call s:AsyncProcess.execute(command, {'exit_cb': function('s:_curl_cb', [has_output_file, output_file, a:settings])}) +endfunction + +let s:clients.wget = {} +let s:clients.wget.errcode = {} +let s:clients.wget.errcode[1] = 'Generic error code.' +let s:clients.wget.errcode[2] = 'Parse error---for instance, when parsing command-line options, the .wgetrc or .netrc...' +let s:clients.wget.errcode[3] = 'File I/O error.' +let s:clients.wget.errcode[4] = 'Network failure.' +let s:clients.wget.errcode[5] = 'SSL verification failure.' +let s:clients.wget.errcode[6] = 'Username/password authentication failure.' +let s:clients.wget.errcode[7] = 'Protocol errors.' +let s:clients.wget.errcode[8] = 'Server issued an error response.' + + +function! s:clients.wget.available(settings) abort + if has_key(a:settings, 'authMethod') + return 0 + endif + return executable(self._command(a:settings)) +endfunction + +function! s:clients.wget._command(settings) abort + return get(get(a:settings, 'command', {}), 'wget', 'wget') +endfunction + +function! s:_wget_cb(has_output_file, output_file, settings, exit_code) abort + if filereadable(a:settings._file.header) + let header_lines = readfile(a:settings._file.header, 'b') + call map(header_lines, 'matchstr(v:val, "^\\s*\\zs.*")') + let headerstr = join(header_lines, "\r\n") + let header_chunks = split(headerstr, '\r\n\zeHTTP/\%(1\.\d\|2\)') + let headers = map(header_chunks, 'split(v:val, "\r\n")') + let responses = map(headers, '[v:val, ""]') + else + let headers = [] + let responses = [[[], '']] + endif + if has_key(s:clients.wget.errcode, a:exit_code) && empty(headers) + throw 'vital: Web.AsyncHTTP: ' . s:clients.wget.errcode[a:exit_code] + endif + if a:has_output_file + let content = '' + else + let content = s:Core.readfile(a:output_file) + endif + let responses[-1][1] = content + + return s:_request_cb(a:settings, responses, a:exit_code) +endfunction + +function! s:clients.wget.request(settings) abort + if has_key(a:settings, 'unixSocket') + throw 'vital: Web.AsyncHTTP: unixSocket only can be used with the curl.' + endif + let quote = s:Core.quote() + let command = self._command(a:settings) + let method = a:settings.method + if method ==# 'HEAD' + let command .= ' --spider' + elseif method !=# 'GET' && method !=# 'POST' + let a:settings.headers['X-HTTP-Method-Override'] = a:settings.method + endif + let a:settings._file.header = s:Core.tempname() + let command .= ' -o ' . quote . a:settings._file.header . quote + let has_output_file = has_key(a:settings, 'outputFile') + if has_output_file + let output_file = s:Core.file_resolve(a:settings.outputFile) + else + let output_file = s:Core.tempname() + let a:settings._file.content = output_file + endif + let command .= ' -O ' . quote . output_file . quote + let command .= ' --server-response -q -L ' + let command .= ' --max-redirect=' . a:settings.maxRedirect + let command .= s:Core.make_header_args(a:settings.headers, '--header=', quote) + let timeout = get(a:settings, 'timeout', '') + let command .= ' --tries=' . a:settings.retry + if timeout =~# '^\d\+$' + let command .= ' --timeout=' . timeout + endif + if has_key(a:settings, 'username') + let command .= ' --http-user=' . quote . escape(a:settings.username, quote) . quote + endif + if has_key(a:settings, 'password') + let command .= ' --http-password=' . quote . escape(a:settings.password, quote) . quote + endif + if has_key(a:settings, 'bearerToken') + let command .= ' --header=' . quote . 'Authorization: Bearer ' . a:settings.bearerToken . quote + endif + let command .= ' ' . quote . a:settings.url . quote + if has_key(a:settings, 'data') + let a:settings._file.post = s:Core.make_postfile(a:settings.data) + let command .= ' --post-file=' . quote . a:settings._file.post . quote + endif + + call s:AsyncProcess.execute(command, {'exit_cb': function('s:_wget_cb', [a:settings])}) +endfunction + + +let &cpo = s:save_cpo +unlet s:save_cpo + +" vim:set et ts=2 sts=2 sw=2 tw=0: + diff --git a/autoload/vital/__vital__/Web/HTTP.vim b/autoload/vital/__vital__/Web/HTTP.vim index 0bd34fc4f..a3ed5e004 100644 --- a/autoload/vital/__vital__/Web/HTTP.vim +++ b/autoload/vital/__vital__/Web/HTTP.vim @@ -9,19 +9,16 @@ function! s:_vital_loaded(V) abort let s:Prelude = s:V.import('Prelude') let s:Process = s:V.import('Process') let s:String = s:V.import('Data.String') + let s:Core = s:V.import('Web.HTTP.Core') endfunction function! s:_vital_depends() abort return { - \ 'modules':['Prelude', 'Data.String', 'Process'] , + \ 'modules':['Prelude', 'Data.String', 'Process', 'Web.HTTP.Core'] , \ 'files': ['HTTP_python2.py', 'HTTP_python3.py'], \} endfunction -function! s:__urlencode_char(c) abort - return printf('%%%02X', char2nr(a:c)) -endfunction - function! s:decodeURI(str) abort let ret = a:str let ret = substitute(ret, '+', ' ', 'g') @@ -29,18 +26,6 @@ function! s:decodeURI(str) abort return ret endfunction -function! s:escape(str) abort - let result = '' - for i in range(len(a:str)) - if a:str[i] =~# '^[a-zA-Z0-9_.~-]$' - let result .= a:str[i] - else - let result .= s:__urlencode_char(a:str[i]) - endif - endfor - return result -endfunction - function! s:encodeURI(items) abort let ret = '' if s:Prelude.is_dict(a:items) @@ -58,7 +43,7 @@ function! s:encodeURI(items) abort let ret .= item endfor else - let ret = s:escape(a:items) + let ret = s:Core.escape(a:items) endif return ret endfunction @@ -95,7 +80,7 @@ function! s:encodeURIComponent(items) abort endfunction function! s:request(...) abort - let settings = s:_build_settings(a:000) + let settings = s:Core.build_settings(a:000) let settings.method = toupper(settings.method) if !has_key(settings, 'url') throw 'vital: Web.HTTP: "url" parameter is required.' @@ -122,7 +107,7 @@ function! s:request(...) abort endif endif if has_key(settings, 'data') - let settings.data = s:_postdata(settings.data) + let settings.data = s:Core.postdata(settings.data) let settings.headers['Content-Length'] = len(join(settings.data, "\n")) endif let settings._file = {} @@ -135,8 +120,8 @@ function! s:request(...) abort endif endfor - call map(responses, 's:_build_response(v:val[0], v:val[1])') - return s:_build_last_response(responses) + call map(responses, 's:Core.build_response(v:val[0], v:val[1])') + return s:Core.build_last_response(responses) endfunction function! s:get(url, ...) abort @@ -158,107 +143,6 @@ function! s:post(url, ...) abort return s:request(settings) endfunction -function! s:_readfile(file) abort - if filereadable(a:file) - return join(readfile(a:file, 'b'), "\n") - endif - return '' -endfunction - -function! s:_make_postfile(data) abort - let fname = s:_tempname() - call writefile(a:data, fname, 'b') - return fname -endfunction - -function! s:_tempname() abort - return s:_file_resolve(tempname()) -endfunction - -function! s:_file_resolve(file) abort - return fnamemodify(a:file, ':p:gs?\\?/?') -endfunction - -function! s:_postdata(data) abort - if s:Prelude.is_dict(a:data) - return [s:encodeURI(a:data)] - elseif s:Prelude.is_list(a:data) - return a:data - else - return split(a:data, "\n") - endif -endfunction - -function! s:_build_response(header, content) abort - let response = { - \ 'header' : a:header, - \ 'content': a:content, - \ 'status': 0, - \ 'statusText': '', - \ 'success': 0, - \ } - - if !empty(a:header) - let status_line = get(a:header, 0) - let matched = matchlist(status_line, '^HTTP/\%(1\.\d\|2\)\s\+\(\d\+\)\s\+\(.*\)') - if !empty(matched) - let [status, status_text] = matched[1 : 2] - let response.status = status - 0 - let response.statusText = status_text - let response.success = status =~# '^2' - call remove(a:header, 0) - endif - endif - return response -endfunction - -function! s:_build_last_response(responses) abort - let all_headers = [] - for response in a:responses - call extend(all_headers, response.header) - endfor - let last_response = remove(a:responses, -1) - let last_response.redirectInfo = a:responses - let last_response.allHeaders = all_headers - return last_response -endfunction - -function! s:_build_settings(args) abort - let settings = { - \ 'method': 'GET', - \ 'headers': {}, - \ 'client': ['python', 'curl', 'wget', 'python3', 'python2'], - \ 'maxRedirect': 20, - \ 'retry': 1, - \ } - let args = copy(a:args) - if len(args) == 0 - throw 'vital: Web.HTTP: request() needs one or more arguments.' - endif - if s:Prelude.is_dict(args[-1]) - call extend(settings, remove(args, -1)) - endif - if len(args) == 2 - let settings.method = remove(args, 0) - endif - if !empty(args) - let settings.url = args[0] - endif - - return settings -endfunction - -function! s:_make_header_args(headdata, option, quote) abort - let args = '' - for [key, value] in items(a:headdata) - if s:Prelude.is_windows() - let value = substitute(value, '"', '"""', 'g') - endif - let args .= ' ' . a:option . a:quote . key . ': ' . value . a:quote - endfor - return args -endfunction - function! s:parseHeader(headers) abort " FIXME: User should be able to specify the treatment method of the duplicate item. let header = {} @@ -383,18 +267,18 @@ function! s:clients.curl._command(settings) abort endfunction function! s:clients.curl.request(settings) abort - let quote = s:_quote() + let quote = s:Core.quote() let command = self._command(a:settings) if has_key(a:settings, 'unixSocket') let command .= ' --unix-socket ' . quote . a:settings.unixSocket . quote endif - let a:settings._file.header = s:_tempname() + let a:settings._file.header = s:Core.tempname() let command .= ' --dump-header ' . quote . a:settings._file.header . quote let has_output_file = has_key(a:settings, 'outputFile') if has_output_file - let output_file = s:_file_resolve(a:settings.outputFile) + let output_file = s:Core.file_resolve(a:settings.outputFile) else - let output_file = s:_tempname() + let output_file = s:Core.tempname() let a:settings._file.content = output_file endif let command .= ' --output ' . quote . output_file . quote @@ -408,7 +292,7 @@ function! s:clients.curl.request(settings) abort let command .= '-X ' . a:settings.method endif let command .= ' --max-redirs ' . a:settings.maxRedirect - let command .= s:_make_header_args(a:settings.headers, '-H ', quote) + let command .= s:Core.make_header_args(a:settings.headers, '-H ', quote) let timeout = get(a:settings, 'timeout', '') let command .= ' --retry ' . a:settings.retry if timeout =~# '^\d\+$' @@ -432,7 +316,7 @@ function! s:clients.curl.request(settings) abort let command .= ' --oauth2-bearer ' . quote . a:settings.bearerToken . quote endif if has_key(a:settings, 'data') - let a:settings._file.post = s:_make_postfile(a:settings.data) + let a:settings._file.post = s:Core.make_postfile(a:settings.data) let command .= ' --data-binary @' . quote . a:settings._file.post . quote endif let command .= ' ' . quote . a:settings.url . quote @@ -440,7 +324,7 @@ function! s:clients.curl.request(settings) abort call s:Process.system(command) let retcode = s:Process.get_last_status() - let headerstr = s:_readfile(a:settings._file.header) + let headerstr = s:Core.readfile(a:settings._file.header) let header_chunks = split(headerstr, "\r\n\r\n") let headers = map(header_chunks, 'split(v:val, "\r\n")') if retcode != 0 && empty(headers) @@ -458,7 +342,7 @@ function! s:clients.curl.request(settings) abort if has_output_file || a:settings.method ==? 'HEAD' let content = '' else - let content = s:_readfile(output_file) + let content = s:Core.readfile(output_file) endif let responses[-1][1] = content return responses @@ -491,7 +375,7 @@ function! s:clients.wget.request(settings) abort if has_key(a:settings, 'unixSocket') throw 'vital: Web.HTTP: unixSocket only can be used with the curl.' endif - let quote = s:_quote() + let quote = s:Core.quote() let command = self._command(a:settings) let method = a:settings.method if method ==# 'HEAD' @@ -499,19 +383,19 @@ function! s:clients.wget.request(settings) abort elseif method !=# 'GET' && method !=# 'POST' let a:settings.headers['X-HTTP-Method-Override'] = a:settings.method endif - let a:settings._file.header = s:_tempname() + let a:settings._file.header = s:Core.tempname() let command .= ' -o ' . quote . a:settings._file.header . quote let has_output_file = has_key(a:settings, 'outputFile') if has_output_file - let output_file = s:_file_resolve(a:settings.outputFile) + let output_file = s:Core.file_resolve(a:settings.outputFile) else - let output_file = s:_tempname() + let output_file = s:Core.tempname() let a:settings._file.content = output_file endif let command .= ' -O ' . quote . output_file . quote let command .= ' --server-response -q -L ' let command .= ' --max-redirect=' . a:settings.maxRedirect - let command .= s:_make_header_args(a:settings.headers, '--header=', quote) + let command .= s:Core.make_header_args(a:settings.headers, '--header=', quote) let timeout = get(a:settings, 'timeout', '') let command .= ' --tries=' . a:settings.retry if timeout =~# '^\d\+$' @@ -528,7 +412,7 @@ function! s:clients.wget.request(settings) abort endif let command .= ' ' . quote . a:settings.url . quote if has_key(a:settings, 'data') - let a:settings._file.post = s:_make_postfile(a:settings.data) + let a:settings._file.post = s:Core.make_postfile(a:settings.data) let command .= ' --post-file=' . quote . a:settings._file.post . quote endif @@ -552,7 +436,7 @@ function! s:clients.wget.request(settings) abort if has_output_file let content = '' else - let content = s:_readfile(output_file) + let content = s:Core.readfile(output_file) endif let responses[-1][1] = content return responses @@ -621,10 +505,6 @@ function! s:clients.python2.request(settings) abort endfunction -function! s:_quote() abort - return &shell =~# 'sh$' ? "'" : '"' -endfunction - let &cpo = s:save_cpo unlet s:save_cpo diff --git a/autoload/vital/__vital__/Web/HTTP/Core.vim b/autoload/vital/__vital__/Web/HTTP/Core.vim new file mode 100644 index 000000000..f1cc18604 --- /dev/null +++ b/autoload/vital/__vital__/Web/HTTP/Core.vim @@ -0,0 +1,143 @@ +let s:save_cpo = &cpo +set cpo&vim + +function! s:_vital_loaded(V) abort + let s:V = a:V + let s:Prelude = s:V.import('Prelude') + let s:Process = s:V.import('Process') + let s:String = s:V.import('Data.String') +endfunction + +function! s:_vital_depends() abort + return { + \ 'modules':['Prelude', 'Data.String', 'Process'] , + \ 'files': ['HTTP_python2.py', 'HTTP_python3.py'], + \} +endfunction + +function! s:__urlencode_char(c) abort + return printf('%%%02X', char2nr(a:c)) +endfunction + +function! s:readfile(file) abort + if filereadable(a:file) + return join(readfile(a:file, 'b'), "\n") + endif + return '' +endfunction + +function! s:make_postfile(data) abort + let fname = s:tempname() + call writefile(a:data, fname, 'b') + return fname +endfunction + +function! s:tempname() abort + return s:file_resolve(tempname()) +endfunction + +function! s:file_resolve(file) abort + return fnamemodify(a:file, ':p:gs?\\?/?') +endfunction + +function! s:postdata(data) abort + if s:Prelude.is_dict(a:data) + return [s:encodeURI(a:data)] + elseif s:Prelude.is_list(a:data) + return a:data + else + return split(a:data, "\n") + endif +endfunction + +function! s:build_response(header, content) abort + let response = { + \ 'header' : a:header, + \ 'content': a:content, + \ 'status': 0, + \ 'statusText': '', + \ 'success': 0, + \ } + + if !empty(a:header) + let status_line = get(a:header, 0) + let matched = matchlist(status_line, '^HTTP/\%(1\.\d\|2\)\s\+\(\d\+\)\s\+\(.*\)') + if !empty(matched) + let [status, status_text] = matched[1 : 2] + let response.status = status - 0 + let response.statusText = status_text + let response.success = status =~# '^2' + call remove(a:header, 0) + endif + endif + return response +endfunction + +function! s:build_last_response(responses) abort + let all_headers = [] + for response in a:responses + call extend(all_headers, response.header) + endfor + let last_response = remove(a:responses, -1) + let last_response.redirectInfo = a:responses + let last_response.allHeaders = all_headers + return last_response +endfunction + +function! s:build_settings(args) abort + let settings = { + \ 'method': 'GET', + \ 'headers': {}, + \ 'client': ['python', 'curl', 'wget', 'python3', 'python2'], + \ 'maxRedirect': 20, + \ 'retry': 1, + \ } + let args = copy(a:args) + if len(args) == 0 + throw 'vital: Web.HTTP: request() needs one or more arguments.' + endif + if s:Prelude.is_dict(args[-1]) + call extend(settings, remove(args, -1)) + endif + if len(args) == 2 + let settings.method = remove(args, 0) + endif + if !empty(args) + let settings.url = args[0] + endif + + return settings +endfunction + +function! s:make_header_args(headdata, option, quote) abort + let args = '' + for [key, value] in items(a:headdata) + if s:Prelude.is_windows() + let value = substitute(value, '"', '"""', 'g') + endif + let args .= ' ' . a:option . a:quote . key . ': ' . value . a:quote + endfor + return args +endfunction + +function! s:escape(str) abort + let result = '' + for i in range(len(a:str)) + if a:str[i] =~# '^[a-zA-Z0-9_.~-]$' + let result .= a:str[i] + else + let result .= s:__urlencode_char(a:str[i]) + endif + endfor + return result +endfunction + +function! s:quote() abort + return &shell =~# 'sh$' ? "'" : '"' +endfunction + + +let &cpo = s:save_cpo +unlet s:save_cpo + +" vim:set et ts=2 sts=2 sw=2 tw=0: diff --git a/doc/vital/Web/AsyncHTTP.txt b/doc/vital/Web/AsyncHTTP.txt new file mode 100644 index 000000000..af719706c --- /dev/null +++ b/doc/vital/Web/AsyncHTTP.txt @@ -0,0 +1,202 @@ +*vital/Web/AsyncHTTP.txt* simple Async HTTP client library. + +Maintainer: mattn + thinca + +============================================================================== +CONTENTS *Vital.Web.AsyncHTTP-contents* + +INTRODUCTION |Vital.Web.AsyncHTTP-introduction| +INTERFACE |Vital.Web.AsyncHTTP-interface| + Functions |Vital.Web.AsyncHTTP-functions| + Response |Vital.Web.AsyncHTTP-response| + +============================================================================== +INTRODUCTION *Vital.Web.AsyncHTTP-introduction* + +*Vital.Web.AsyncHTTP* is an Async HTTP Utilities Library. It provides a simple +Async HTTP client. + +============================================================================== +INTERFACE *Vital.Web.AsyncHTTP-interface* +------------------------------------------------------------------------------ +FUNCTIONS *Vital.Web.AsyncHTTP-functions* + +get({url} [, {param} [, {header}]]) *Vital.Web.AsyncHTTP.get()* + Send a GET request to the server. + This is just a wrapper of |Vital.Web.AsyncHTTP.request()|. + +post({url} [, {param} [, {header}]]) *Vital.Web.AsyncHTTP.post()* + Send a POST request to the server. + This is just a wrapper of |Vital.Web.AsyncHTTP.request()|. + +request({settings}) *Vital.Web.AsyncHTTP.request()* +request({url} [, {settings}]) +request({method}, {url} [, {settings}]) + Send a request to the server. + This function requires one of the clients, "curl" or "wget". + {settings} is a |Dictionary| which contains the following items: + + "url" Required + URL of a server. + + "method" Default: "GET" + HTTP Method, such as GET, HEAD, POST, PUT, DELETE, or PATCH. + + "param" Default: (None) + GET parameters. This is a string or a dictionary. + If dictionary, it is converted to a string by + |Vital.Web.HTTP.encodeURI()|. + This is appended to url. + + "data" Default: (None) + POST data. This is a string, a list, or a dictionary. + If it is a dictionary, it is converted to a string by + |Vital.Web.HTTP.encodeURI()|. + + "headers" Default: (None) + Request headers. This is a dictionary. + + "contentType" Default: (None) + Content-Type for "data". + This is one of "headers". This is used preferentially. + + "outputFile" Default: (None) + Output the result to this file. + "content" of the result become empty when this is specified. + + "timeout" Default: (None) + Network timeout by seconds. + + "username" Default: (None) + User name for an HTTP authentication. + + "password" Default: (None) + Password for an HTTP authentication. + + "bearerToken" Default: (None) + Bearer token for an HTTP authentication (OAuth2). + + "maxRedirect" Default: 20 + Maximum number of redirections. + The default is 20, which is usually far more than necessary. + + "retry" Default: 1 + Maximum number of retries. + + "client" Default: ["python", "curl", "wget", + "python3", "python2"] + Candidate list of HTTP client to use for a request. + The first available one is used. + A string as an HTTP client is also possible. + See also |Vital.Web.AsyncHTTP-client|. + + "command" + Command name for a client. You should use with "client". + This is a |Dictionary| that has client name as key and has the + command as value. + This maybe becomes like the following. > + { + "curl": "/usr/bin/curl", + "wget": "/usr/local/bin/wget", + } +< + "authMethod" Default: (None) + (This is only valid for "curl" interface.) + Specify the authorization method. + The value must be in ['basic', 'digest', 'ntlm', 'negotiate', + 'oauth2'] + The default value is None, and then use "anyauth". + + "gzipDecompress" Default: 0 + Attempt to decompress response data as if it was gzipped + + "unixSocket" Default: (None) + Use --unix-sokect (only curl >= 7.40.0) + + "user_cb" Default: (None) + A |function| of callback function called when the process + outputs some data. The callback function has one argument, + response(|Dictionary| of response). + +------------------------------------------------------------------------------ +RESPONSE *Vital.Web.AsyncHTTP-response* + +|Vital.Web.AsyncHTTP.request()|, |Vital.Web.AsyncHTTP.get()|, and +|Vital.Web.AsyncHTTP.post()| pass it to the resonse argument of |user_cb|. +Data structure as |Directory| like following. +> + { + "header": [ + "Content-Type: text/html", + "Content-Length: 310" + ], + "allHeaders": [ + "Set-Cookie: k1=v1; Path=/", + "Content-Type: text/html", + "Content-Length: 310" + ], + "content": " .....", + "status": 200, + "statusText": "OK", + "success": 1, + "redirectInfo": [], + } +< +"header" + The header lines of a response. This can convert to + |Dictionary| by |Vital.Web.HTTP.parseHeader()|. + +"allHeaders" + All of header lines that includes redirectInfo. + +"content" + The content of a response. + +"status" + The http status code of a response. + If the code couldn't take, this is 0. + +"statusText" + The http status code text of a response. + If the code couldn't take, this is the empty string. + +"success" + This is 1 if the "status" is 2xx. + +"redirectInfo" + When the request was redirected, the redirected responses are + stored. Form of these are the same as a response. + + + +------------------------------------------------------------------------------ +CLIENT *Vital.Web.AsyncHTTP-client* + +The following can be used. +(TODO: More document. Especially about limitation.) + +python *Vital.Web.AsyncHTTP-client-python* + "python" as "python3" or "python2" auto select. + (High priority for "python3".) + +curl *Vital.Web.AsyncHTTP-client-curl* + Use curl command. + + http://curl.haxx.se/ + +wget *Vital.Web.AsyncHTTP-client-wget* + Use wget command. + + http://www.gnu.org/software/wget/ + +python3 *Vital.Web.AsyncHTTP-client-python3* + Use Python3's urllib.request library via |if_pyth.txt|. + +python2 *Vital.Web.AsyncHTTP-client-python2* + Use Python2's urllib2 library via |if_pyth.txt|. + + +============================================================================== +vim:tw=78:fo=tcq2mM:ts=8:ft=help:norl + diff --git a/test/Web/AsyncHTTP.vimspec b/test/Web/AsyncHTTP.vimspec new file mode 100644 index 000000000..86038f65a --- /dev/null +++ b/test/Web/AsyncHTTP.vimspec @@ -0,0 +1,81 @@ +scriptencoding utf-8 + +let s:is_windows = has('win32') + +function! s:wait_until(cond, timeout) abort + let start = reltime() + while 1 + if a:cond() + return 1 + endif + if reltimefloat(reltime(start)) * 1000 > a:timeout + return 0 + endif + sleep 10m + endwhile +endfunction + +Describe Web.AsyncHTTP + Before all + let AsyncHTTP = vital#vital#new().import('Web.AsyncHTTP') + End + + After all + unlet g:response + End + + Before + let g:response = {} + End + + Describe .request() + It client curl + if s:is_windows && exists('$GITHUB_ACTIONS') && $GITHUB_ACTIONS ==# 'true' + Skip Windows on GitHub Actions, occur unknown write error... + endif + + function! s:user_cb(response) abort + let g:response = a:response + endfunction + + let current_dir = fnamemodify(getcwd(), ":gs?\\?/?") + let response = AsyncHTTP.request({ + \ 'url': 'file:///' .. current_dir .. '/test/_testdata/Web/test.html', + \ 'client': ['curl'], + \ 'user_cb': function('s:user_cb'), + \ }) + + call s:wait_until({-> g:response !=# {}}, 2000) + + Assert Equals(g:response.content, "テスト\n") + End + + It option_outputFile + if s:is_windows && exists('$GITHUB_ACTIONS') && $GITHUB_ACTIONS ==# 'true' + Skip Windows on GitHub Actions, occur unknown write error... + endif + + function! s:user_cb(response) abort + let g:response = a:response + endfunction + + let current_dir = fnamemodify(getcwd(), ":gs?\\?/?") + let output_file = tempname() + let response = AsyncHTTP.request({ + \ 'url': 'file:///' .. current_dir .. '/test/_testdata/Web/test.html', + \ 'outputFile': output_file, + \ 'client': ['curl'], + \ }) + + call s:wait_until({-> g:response !=# {}}, 2000) + + let output_file_content = readfile(output_file) + + Assert Equals(len(output_file_content), 1) + Assert Equals(output_file_content[0], "テスト") + + call delete(output_file) + End + End +End +