Network, HTTP, and URL functions
from fastcore.test import *
from nbdev.showdoc import *
from fastcore.nb_imports import *



Update url's path with urllib.parse.quote

urlquote("[email protected]{}…master")
'[email protected]'


urlwrap(url, data=None, headers=None)

Wrap url in a urllib Request with urlquote

class HTTP4xxClientError[source]

HTTP4xxClientError(url, code, msg, hdrs, fp) :: HTTPError

Base class for client exceptions (code 4xx) from url* functions

class HTTP5xxServerError[source]

HTTP5xxServerError(url, code, msg, hdrs, fp) :: HTTPError

Base class for server exceptions (code 5xx) from url* functions


urlopen(url, data=None, headers=None, timeout=None, **kwargs)

Like urllib.request.urlopen, but first urlwrap the url, and encode data


urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs)

Retrieve url, using data dict or kwargs to POST if present


urljson(url, data=None, timeout=None)

Retrieve url and decode json

test_eq(urljson('')['headers']['User-Agent'], url_default_headers['User-Agent'])


urlcheck(url, timeout=10)



Remove fragment, params, and querystring from url if present

test_eq(urlclean(''), '')


urlretrieve(url, filename=None, reporthook=None, data=None)

Retrieve a URL into a temporary location on disk.

Requires a URL argument. If a filename is passed, it is used as the temporary file location. The reporthook argument should be a callable that accepts a block number, a read size, and the total file size of the URL target. The data argument should be valid URL encoded data.

If a filename is passed and the URL points to a local resource, the result is a copy from local file to new file.

Returns a tuple containing the path to the newly created data file as well as the resulting HTTPMessage object.


urldest(url, dest=None)


urlsave(url, dest=None, reporthook=None, timeout=None)

Retrieve url and save based on its name

with tempfile.TemporaryDirectory() as d: urlsave('', d)



Test if x is a valid URL

assert urlvalid('')
assert not urlvalid('')
assert not urlvalid(1)


urlrequest(url, verb, headers=None, route=None, query=None, data=None, json_data=True)

Request for url with optional route params replaced by route, plus query string, and post data

hdr = {'Hdr1':'1', 'Hdr2':'2'}
req = urlrequest('{foo}/1', 'POST',
                 headers=hdr, route={'foo':'3'}, query={'q':'4'}, data={'d':'5'})

test_eq(req.headers, hdr)
test_eq(req.full_url, '')
test_eq(req.method, 'POST')
test_eq(, b'{"d": "5"}')
req = urlrequest('{foo}/1', 'POST', data={'d':'5','e':'6'}, headers=hdr, json_data=False)
test_eq(, b'd=5&e=6')



Summary containing full_url, headers, method, and data, removing skip from headers

{'full_url': '{foo}/1',
 'method': 'POST',
 'data': b'd=5&e=6',
 'headers': {'Hdr2': '2'}}


urlsend(url, verb, headers=None, route=None, query=None, data=None, json_data=True, return_json=True, return_headers=False, debug=None)

Send request with urlrequest, converting result to json if return_json

url = ""
header={'Accept': 'application/vnd.github.v3+json'}
route={'artifact_id': 1234, 'archive_format': 'zip'} 

res = urlsend(url, "get", headers=header, route=route)
assert type(res) is bytes

def _org_urlsend(url, verb, headers=None, route=None, query=None, data=None, json_data=True,
            return_json=True, return_headers=False, debug=None):
    req = urlrequest(url, verb, headers, route=route, query=query, data=data, json_data=json_data)
    if debug: debug(req)
    return urlread(req, return_json=return_json, return_headers=return_headers)

test_fail(_org_urlsend, args=dict(url=url, verb="get", headers=header, route=route), msg="'utf-8' codec can't decode")


do_request(url, post=False, headers=None, **data)

Call GET or json-encoded POST on url, depending on post


start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None)

Create a socket server on port, with optional host, of type dgram

You can create a TCP client and server pass an int as port and optional host. host defaults to your main network interface if not provided. You can create a Unix socket client and server by passing a string to port. A SOCK_STREAM socket is created by default, unless you pass dgram=True, in which case a SOCK_DGRAM socket is created. n_queue sets the listening queue size.


start_client(port, host=None, dgram=False)

Create a socket client on port, with optional host, of type dgram