cosmopolitan/tool/net/fetch.inc
Justine Tunney ad3944a3b6
Support any HTTP method
It's now possible to use redbean Fetch() with arbitrary HTTP methods,
e.g. LIST which is used by Hashicorp. There's an eight char limit and
uppercase canonicalization still happens. This change also includes a
better function for launching a browser tab, that won't deadlock on a
headless workstation running Debian.

Closes #1107
2024-02-22 14:12:18 -08:00

580 lines
19 KiB
SourcePawn

#define FetchHasHeader(H) (!!msg.headers[H].a)
#define FetchHeaderData(H) (inbuf.p + msg.headers[H].a)
#define FetchHeaderLength(H) (msg.headers[H].b - msg.headers[H].a)
#define FetchHeaderEqualCase(H, S) \
SlicesEqualCase(S, strlen(S), FetchHeaderData(H), FetchHeaderLength(H))
#define kaNONE 0
#define kaOPEN 1
#define kaKEEP 2
#define kaCLOSE 3
static int LuaFetch(lua_State *L) {
#define ssl nope // TODO(jart): make this file less huge
ssize_t rc;
bool usingssl;
uint32_t ip;
struct Url url;
int t, ret, sock = -1, hdridx;
const char *host, *port;
char *request;
struct TlsBio *bio;
struct addrinfo *addr;
struct Buffer inbuf; // shadowing intentional
struct HttpMessage msg; // shadowing intentional
struct HttpUnchunker u;
const char *urlarg, *body, *method;
char *conlenhdr = "";
char *headers = 0;
const char *hosthdr = 0;
const char *connhdr = 0;
const char *agenthdr = brand;
const char *key, *val, *hdr;
size_t keylen, vallen;
size_t urlarglen, requestlen, paylen, bodylen;
size_t i, g, hdrsize;
int keepalive = kaNONE;
char canmethod[9] = {0};
uint64_t imethod;
int numredirects = 0, maxredirects = 5;
bool followredirect = true;
struct addrinfo hints = {.ai_family = AF_INET,
.ai_socktype = SOCK_STREAM,
.ai_protocol = IPPROTO_TCP,
.ai_flags = AI_NUMERICSERV};
(void)ret;
(void)usingssl;
/*
* Get args: url [, body | {method = "PUT", body = "..."}]
*/
urlarg = luaL_checklstring(L, 1, &urlarglen);
if (lua_istable(L, 2)) {
lua_settop(L, 2); // discard any extra arguments
lua_getfield(L, 2, "body");
body = luaL_optlstring(L, -1, "", &bodylen);
lua_getfield(L, 2, "method");
// use GET by default if no method is provided
method = luaL_optstring(L, -1, "GET");
if ((imethod = ParseHttpMethod(method, -1))) {
WRITE64LE(canmethod, imethod);
method = canmethod;
} else {
return LuaNilError(L, "bad method");
}
lua_getfield(L, 2, "followredirect");
if (lua_isboolean(L, -1)) followredirect = lua_toboolean(L, -1);
lua_getfield(L, 2, "maxredirects");
maxredirects = luaL_optinteger(L, -1, maxredirects);
lua_getfield(L, 2, "numredirects");
numredirects = luaL_optinteger(L, -1, numredirects);
lua_getfield(L, 2, "keepalive");
if (!lua_isnil(L, -1)) {
if (lua_istable(L, -1)) {
keepalive = kaOPEN; // will be updated based on host later
} else if (lua_isboolean(L, -1)) {
keepalive = lua_toboolean(L, -1) ? kaOPEN : kaNONE;
if (keepalive) {
lua_createtable(L, 0, 1);
lua_setfield(L, 2, "keepalive");
}
} else {
return luaL_argerror(L, 2,
"invalid keepalive value;"
" boolean or table expected");
}
}
lua_getfield(L, 2, "headers");
if (!lua_isnil(L, -1)) {
if (!lua_istable(L, -1))
return luaL_argerror(L, 2, "invalid headers value; table expected");
lua_pushnil(L);
while (lua_next(L, -2)) {
if (lua_type(L, -2) == LUA_TSTRING) { // skip any non-string keys
key = lua_tolstring(L, -2, &keylen);
if (!IsValidHttpToken(key, keylen))
return LuaNilError(L, "invalid header name: %s", key);
val = lua_tolstring(L, -1, &vallen);
if (!(hdr = gc(EncodeHttpHeaderValue(val, vallen, 0))))
return LuaNilError(L, "invalid header %s value encoding", key);
// Content-Length will be overwritten; skip it to avoid duplicates;
// also allow unknown headers
if ((hdridx = GetHttpHeader(key, keylen)) == -1 ||
hdridx != kHttpContentLength) {
if (hdridx == kHttpUserAgent) {
agenthdr = hdr;
} else if (hdridx == kHttpHost) {
hosthdr = hdr;
} else if (hdridx == kHttpConnection) {
connhdr = hdr;
} else {
appendd(&headers, key, keylen);
appendw(&headers, READ16LE(": "));
appends(&headers, hdr);
appendw(&headers, READ16LE("\r\n"));
}
}
}
lua_pop(L, 1); // pop the value, keep the key for the next iteration
}
}
lua_settop(L, 2); // drop all added elements to keep the stack balanced
} else if (lua_isnoneornil(L, 2)) {
body = "";
bodylen = 0;
method = "GET";
} else {
body = luaL_checklstring(L, 2, &bodylen);
method = "POST";
}
// provide Content-Length header unless it's zero and not expected
imethod = ParseHttpMethod(method, -1);
if (bodylen > 0 ||
!(imethod == kHttpGet || imethod == kHttpHead || imethod == kHttpTrace ||
imethod == kHttpDelete || imethod == kHttpConnect)) {
conlenhdr = gc(xasprintf("Content-Length: %zu\r\n", bodylen));
}
/*
* Parse URL.
*/
gc(ParseUrl(urlarg, urlarglen, &url, true));
gc(url.params.p);
DEBUGF("(ftch) client fetching %`'s (host=%`'.*s, port=%.*s, path=%`'.*s)",
urlarg, url.host.n, url.host.p, url.port.n, url.port.p, url.path.n,
url.path.p);
usingssl = false;
if (url.scheme.n) {
#ifndef UNSECURE
if (!unsecure && url.scheme.n == 5 &&
!memcasecmp(url.scheme.p, "https", 5)) {
usingssl = true;
} else
#endif
if (!(url.scheme.n == 4 && !memcasecmp(url.scheme.p, "http", 4))) {
return LuaNilError(L, "bad scheme");
}
}
#ifndef UNSECURE
if (usingssl) keepalive = kaNONE;
if (usingssl && !sslinitialized) TlsInit();
#endif
if (url.host.n) {
host = gc(strndup(url.host.p, url.host.n));
if (url.port.n) {
port = gc(strndup(url.port.p, url.port.n));
#ifndef UNSECURE
} else if (usingssl) {
port = "443";
#endif
} else {
port = "80";
}
} else if ((ip = ParseIp(urlarg, -1)) != -1) {
host = urlarg;
port = "80";
} else {
return LuaNilError(L, "invalid host");
}
if (!IsAcceptableHost(host, -1)) {
return LuaNilError(L, "invalid host");
}
if (!IsAcceptablePort(port, -1)) {
return LuaNilError(L, "invalid port");
}
if (!hosthdr) hosthdr = gc(xasprintf("%s:%s", host, port));
// check if hosthdr is in keepalive table
if (keepalive && lua_istable(L, 2)) {
lua_getfield(L, 2, "keepalive");
lua_getfield(L, -1, "close"); // aft: -2=tbl, -1=close
lua_getfield(L, -2, hosthdr); // aft: -3=tbl, -2=close, -1=hosthdr
if (lua_isinteger(L, -1)) {
sock = lua_tointeger(L, -1);
keepalive = lua_toboolean(L, -2) ? kaCLOSE : kaKEEP;
// remove host mapping, as the socket is ether being closed
// (so needs to be removed) or will be added after the request is done;
// this also helps to keep the mapping clean in case of an error
lua_pushnil(L); // aft: -4=tbl, -3=close, -2=hosthdr, -1=nil
lua_setfield(L, -4, hosthdr);
VERBOSEF("(ftch) reuse socket %d for host %s (and %s)", sock, hosthdr,
keepalive == kaCLOSE ? "close" : "keep");
}
lua_settop(L, 2); // drop all added elements to keep the stack balanced
}
url.fragment.p = 0, url.fragment.n = 0;
url.scheme.p = 0, url.scheme.n = 0;
url.user.p = 0, url.user.n = 0;
url.pass.p = 0, url.pass.n = 0;
url.host.p = 0, url.host.n = 0;
url.port.p = 0, url.port.n = 0;
if (!url.path.n || url.path.p[0] != '/') {
void *p = gc(xmalloc(1 + url.path.n));
mempcpy(mempcpy(p, "/", 1), url.path.p, url.path.n);
url.path.p = p;
++url.path.n;
}
/*
* Create HTTP message.
*/
request = 0;
appendf(&request,
"%s %s HTTP/1.1\r\n"
"Host: %s\r\n"
"Connection: %s\r\n"
"User-Agent: %s\r\n"
"%s%s"
"\r\n",
method, gc(EncodeUrl(&url, 0)), hosthdr,
(keepalive == kaNONE || keepalive == kaCLOSE)
? "close"
: (connhdr ? connhdr : "keep-alive"),
agenthdr, conlenhdr, headers ? headers : "");
appendd(&request, body, bodylen);
requestlen = appendz(request).i;
gc(request);
if (keepalive == kaNONE || keepalive == kaOPEN) {
/*
* Perform DNS lookup.
*/
DEBUGF("(ftch) client resolving %s", host);
if ((rc = getaddrinfo(host, port, &hints, &addr)) != 0) {
return LuaNilError(L, "getaddrinfo(%s:%s) error: EAI_%s %s", host, port,
gai_strerror(rc), strerror(errno));
}
/*
* Connect to server.
*/
ip = ntohl(((struct sockaddr_in *)addr->ai_addr)->sin_addr.s_addr);
DEBUGF("(ftch) client connecting %hhu.%hhu.%hhu.%hhu:%d", ip >> 24,
ip >> 16, ip >> 8, ip,
ntohs(((struct sockaddr_in *)addr->ai_addr)->sin_port));
CHECK_NE(-1, (sock = GoodSocket(addr->ai_family, addr->ai_socktype,
addr->ai_protocol, false, &timeout)));
rc = connect(sock, addr->ai_addr, addr->ai_addrlen);
freeaddrinfo(addr), addr = 0;
if (rc == -1) {
close(sock);
return LuaNilError(L, "connect(%s:%s) error: %s", host, port,
strerror(errno));
}
}
(void)bio;
#ifndef UNSECURE
if (usingssl) {
if (sslcliused) {
mbedtls_ssl_session_reset(&sslcli);
} else {
ReseedRng(&rngcli, "child");
}
sslcliused = true;
DEBUGF("(ftch) client handshaking %`'s", host);
if (!evadedragnetsurveillance) {
mbedtls_ssl_set_hostname(&sslcli, host);
}
bio = gc(malloc(sizeof(struct TlsBio)));
bio->fd = sock;
bio->a = 0;
bio->b = 0;
bio->c = -1;
mbedtls_ssl_set_bio(&sslcli, bio, TlsSend, 0, TlsRecvImpl);
while ((ret = mbedtls_ssl_handshake(&sslcli))) {
switch (ret) {
case MBEDTLS_ERR_SSL_WANT_READ:
break;
case MBEDTLS_ERR_X509_CERT_VERIFY_FAILED:
goto VerifyFailed;
default:
close(sock);
return LuaNilTlsError(L, "handshake", ret);
}
}
LockInc(&shared->c.sslhandshakes);
VERBOSEF("(ftch) shaken %s:%s %s %s", host, port,
mbedtls_ssl_get_ciphersuite(&sslcli),
mbedtls_ssl_get_version(&sslcli));
}
#endif /* UNSECURE */
/*
* Send HTTP Message.
*/
DEBUGF("(ftch) client sending %s request", method);
for (i = 0; i < requestlen; i += rc) {
#ifndef UNSECURE
if (usingssl) {
rc = mbedtls_ssl_write(&sslcli, request + i, requestlen - i);
if (rc <= 0) {
if (rc == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED) goto VerifyFailed;
close(sock);
return LuaNilTlsError(L, "write", rc);
}
} else
#endif
if ((rc = WRITE(sock, request + i, requestlen - i)) <= 0) {
close(sock);
return LuaNilError(L, "write error: %s", strerror(errno));
}
}
if (logmessages) {
LogMessage("sent", request, requestlen);
}
/*
* Handle response.
*/
bzero(&inbuf, sizeof(inbuf));
InitHttpMessage(&msg, kHttpResponse);
for (hdrsize = paylen = t = 0;;) {
if (inbuf.n == inbuf.c) {
inbuf.c += 1000;
inbuf.c += inbuf.c >> 1;
inbuf.p = realloc(inbuf.p, inbuf.c);
}
NOISEF("(ftch) client reading");
#ifndef UNSECURE
if (usingssl) {
if ((rc = mbedtls_ssl_read(&sslcli, inbuf.p + inbuf.n,
inbuf.c - inbuf.n)) < 0) {
if (rc == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
rc = 0;
} else {
close(sock);
free(inbuf.p);
DestroyHttpMessage(&msg);
return LuaNilTlsError(L, "read", rc);
}
}
} else
#endif
if ((rc = READ(sock, inbuf.p + inbuf.n, inbuf.c - inbuf.n)) == -1) {
close(sock);
free(inbuf.p);
DestroyHttpMessage(&msg);
return LuaNilError(L, "read error: %s", strerror(errno));
}
g = rc;
inbuf.n += g;
switch (t) {
case kHttpClientStateHeaders:
if (!g) {
WARNF("(ftch) HTTP client %s error", "EOF headers");
goto TransportError;
}
rc = ParseHttpMessage(&msg, inbuf.p, inbuf.n);
if (rc == -1) {
WARNF("(ftch) HTTP client %s error", "ParseHttpMessage");
goto TransportError;
}
if (rc) {
DEBUGF("(ftch) content-length is %`'.*s",
FetchHeaderLength(kHttpContentLength),
FetchHeaderData(kHttpContentLength));
hdrsize = rc;
if (logmessages) {
LogMessage("received", inbuf.p, hdrsize);
}
if (100 <= msg.status && msg.status <= 199) {
if ((FetchHasHeader(kHttpContentLength) &&
!FetchHeaderEqualCase(kHttpContentLength, "0")) ||
(FetchHasHeader(kHttpTransferEncoding) &&
!FetchHeaderEqualCase(kHttpTransferEncoding, "identity"))) {
WARNF("(ftch) HTTP client %s error", "Content-Length #1");
goto TransportError;
}
DestroyHttpMessage(&msg);
InitHttpMessage(&msg, kHttpResponse);
memmove(inbuf.p, inbuf.p + hdrsize, inbuf.n - hdrsize);
inbuf.n -= hdrsize;
break;
}
if (msg.status == 204 || msg.status == 304) {
goto Finished;
}
if (FetchHasHeader(kHttpTransferEncoding) &&
!FetchHeaderEqualCase(kHttpTransferEncoding, "identity")) {
if (FetchHeaderEqualCase(kHttpTransferEncoding, "chunked")) {
t = kHttpClientStateBodyChunked;
bzero(&u, sizeof(u));
goto Chunked;
} else {
WARNF("(ftch) HTTP client %s error", "Transfer-Encoding");
goto TransportError;
}
} else if (FetchHasHeader(kHttpContentLength)) {
rc = ParseContentLength(FetchHeaderData(kHttpContentLength),
FetchHeaderLength(kHttpContentLength));
if (rc == -1) {
WARNF("(ftch) ParseContentLength(%`'.*s) failed",
FetchHeaderLength(kHttpContentLength),
FetchHeaderData(kHttpContentLength));
goto TransportError;
}
if ((paylen = rc) <= inbuf.n - hdrsize) {
goto Finished;
} else {
t = kHttpClientStateBodyLengthed;
}
} else {
t = kHttpClientStateBody;
}
}
break;
case kHttpClientStateBody:
if (!g) {
paylen = inbuf.n - hdrsize;
goto Finished;
}
break;
case kHttpClientStateBodyLengthed:
if (!g) {
WARNF("(ftch) HTTP client %s error", "EOF body");
goto TransportError;
}
if (inbuf.n - hdrsize >= paylen) {
goto Finished;
}
break;
case kHttpClientStateBodyChunked:
Chunked:
rc = Unchunk(&u, inbuf.p + hdrsize, inbuf.n - hdrsize, &paylen);
if (rc == -1) {
WARNF("(ftch) HTTP client %s error", "Unchunk");
goto TransportError;
}
if (rc) goto Finished;
break;
default:
__builtin_unreachable();
}
}
Finished:
if (paylen && logbodies) LogBody("received", inbuf.p + hdrsize, paylen);
VERBOSEF("(ftch) completed %s HTTP%02d %d %s %`'.*s", method, msg.version,
msg.status, urlarg, FetchHeaderLength(kHttpServer),
FetchHeaderData(kHttpServer));
// check if the server has requested to close the connection
// https://www.rfc-editor.org/rfc/rfc2616#section-14.10
if (keepalive && keepalive != kaCLOSE && FetchHasHeader(kHttpConnection) &&
FetchHeaderEqualCase(kHttpConnection, "close")) {
VERBOSEF("(ftch) close keepalive on server request");
keepalive = kaCLOSE;
}
// need to save updated sock for keepalive
if (keepalive && keepalive != kaCLOSE && lua_istable(L, 2)) {
lua_getfield(L, 2, "keepalive");
lua_pushinteger(L, sock);
lua_setfield(L, -2, hosthdr);
lua_pop(L, 1);
}
if (followredirect && FetchHasHeader(kHttpLocation) &&
(msg.status == 301 || msg.status == 308 || // permanent redirects
msg.status == 302 || msg.status == 307 || // temporary redirects
msg.status == 303 /* see other; non-GET changes to GET, body lost */) &&
numredirects < maxredirects) {
// if 303, then remove body and set method to GET
if (msg.status == 303) {
body = "";
bodylen = 0;
method = "GET";
}
// create table if needed
if (!lua_istable(L, 2)) {
lua_settop(L, 1); // pop body if present
lua_createtable(L, 0, 3); // body, method, numredirects
}
lua_pushlstring(L, body, bodylen);
lua_setfield(L, -2, "body");
lua_pushstring(L, method);
lua_setfield(L, -2, "method");
lua_pushinteger(L, numredirects + 1);
lua_setfield(L, -2, "numredirects");
// replace URL with Location header, which
// can be a relative or absolute URL:
// https://www.rfc-editor.org/rfc/rfc3986#section-4.2
gc(ParseUrl(FetchHeaderData(kHttpLocation),
FetchHeaderLength(kHttpLocation), &url, true));
free(url.params.p);
VERBOSEF("(ftch) client redirecting %`'.*s "
"(scheme=%`'.*s, host=%`'.*s, port=%.*s, path=%`'.*s)",
FetchHeaderLength(kHttpLocation), FetchHeaderData(kHttpLocation),
url.scheme.n, url.scheme.p, url.host.n, url.host.p, url.port.n,
url.port.p, url.path.n, url.path.p);
// while it's possible to check for IsAcceptableHost/IsAcceptablePort
// it's not clear what to do if they are not;
// if they are invalid, redirect returns "invalid host" message
if (url.host.n && url.scheme.n) {
lua_pushlstring(L, FetchHeaderData(kHttpLocation),
FetchHeaderLength(kHttpLocation));
} else {
gc(ParseUrl(urlarg, urlarglen, &url, true));
free(url.params.p);
// remove user/pass/fragment for the redirect
url.fragment.p = 0, url.fragment.n = 0;
url.user.p = 0, url.user.n = 0;
url.pass.p = 0, url.pass.n = 0;
if (FetchHeaderData(kHttpLocation)[0] == '/') {
// if the path is absolute, then use it
// so `/redir/more` -> `/less` becomes `/less`
url.path.n = 0; // replace the path
} else {
// if the path is relative, then merge it,
// so `/redir/more` -> `less` becomes `/redir/less`
while (url.path.n > 0 && url.path.p[url.path.n - 1] != '/') {
--url.path.n;
}
}
url.path.p = gc(xasprintf("%.*s%.*s", url.path.n, url.path.p,
FetchHeaderLength(kHttpLocation),
FetchHeaderData(kHttpLocation)));
url.path.n = strlen(url.path.p);
lua_pushstring(L, gc(EncodeUrl(&url, 0)));
}
lua_replace(L, -3);
DestroyHttpMessage(&msg);
free(inbuf.p);
if (!keepalive || keepalive == kaCLOSE) close(sock);
return LuaFetch(L);
} else {
lua_pushinteger(L, msg.status);
LuaPushHeaders(L, &msg, inbuf.p);
lua_pushlstring(L, inbuf.p + hdrsize, paylen);
DestroyHttpMessage(&msg);
free(inbuf.p);
if (!keepalive || keepalive == kaCLOSE) close(sock);
return 3;
}
TransportError:
DestroyHttpMessage(&msg);
free(inbuf.p);
close(sock);
return LuaNilError(L, "transport error");
#ifndef UNSECURE
VerifyFailed:
LockInc(&shared->c.sslverifyfailed);
close(sock);
return LuaNilTlsError(
L, gc(DescribeSslVerifyFailure(sslcli.session_negotiate->verify_result)),
ret);
#endif
#undef ssl
}