Network functionality

Network, HTTP, and URL functions 
from fastcore.test import *
from nbdev.showdoc import *
from fastcore.nb_imports import *

Every URL function here goes through urlopen. It quotes the URL, sends url_default_headers, and encodes data. On an HTTP error it raises the status-specific exception for the code, such as HTTP404NotFoundError or HTTP503ServiceUnavailableError. Each is a subclass of HTTP4xxClientError or HTTP5xxServerError. Its message includes the response body. The module also has TCP and Unix-socket clients and servers, and helpers that wait for a condition or a free port.

URLs

source

urlquote

def urlquote(
    url
):

Update url’s path with urllib.parse.quote

urlquote("https://github.com/fastai/fastai/compare/master@{1.day.ago}…master")
'https://github.com/fastai/fastai/compare/master@%7B1.day.ago%7D%E2%80%A6master'
urlquote("https://www.google.com/search?q=你好")
'https://www.google.com/search?q=%E4%BD%A0%E5%A5%BD'

source

urlwrap

def urlwrap(
    url, data:NoneType=None, headers:NoneType=None
):

Wrap url in a urllib Request with urlquote

source

HTTP4xxClientError

def HTTP4xxClientError(
    url, code, msg, hdrs, fp
):

Base class for client exceptions (code 4xx) from url* functions

source

HTTP5xxServerError

def HTTP5xxServerError(
    url, code, msg, hdrs, fp
):

Base class for server exceptions (code 5xx) from url* functions

source

urlopener

def urlopener():

source

urlopen

def urlopen(
    url, data:NoneType=None, headers:NoneType=None, timeout:NoneType=None, **kwargs
):

Like urllib.request.urlopen, but first urlwrap the url, and encode data

On an HTTP error, urlopen raises the status-specific exception for the code, with the response body added to its message:

try: urlopen('https://api.github.com/v3')
except HTTPError as e: 
    print(e.code, e.msg)
    assert 'documentation_url' in e.msg
404 Not Found
====Error Body====
{
  "message": "Not Found",
  "documentation_url": "https://docs.github.com/rest",
  "status": "404"
}

source

urlread

def urlread(
    url, data:NoneType=None, headers:NoneType=None, decode:bool=True, return_json:bool=False,
    return_headers:bool=False, timeout:NoneType=None, **kwargs
):

Retrieve url, using data dict or kwargs to POST if present

source

urljson

def urljson(
    url, data:NoneType=None, headers:NoneType=None, timeout:NoneType=None
):

Retrieve url and decode json

test_eq(urljson('https://httpbin.org/get')['headers']['User-Agent'], url_default_headers['User-Agent'])

source

urlclean

def urlclean(
    url
):

Remove fragment, params, and querystring from url if present

test_eq(urlclean('http://a.com/b?c=1#d'), 'http://a.com/b')

source

urlretrieve

def urlretrieve(
    url, filename:NoneType=None, reporthook:NoneType=None, data:NoneType=None, headers:NoneType=None,
    timeout:NoneType=None
):

Same as urllib.request.urlretrieve but also works with Request objects

source

urldest

def urldest(
    url, dest:NoneType=None
):

source

urlsave

def urlsave(
    url, dest:NoneType=None, reporthook:NoneType=None, headers:NoneType=None, timeout:NoneType=None
):

Retrieve url and save based on its name

#skip
with tempfile.TemporaryDirectory() as d: urlsave('http://www.google.com/index.html', d)

The examples below use a local server that answers each request with the status code in its path, so they run without network access.

Every URL function raises the status-specific exception for 4xx and 5xx codes, with the response body in its message. A code missing from the table raises the base class for its range:

with expect_fail(HTTP404NotFoundError): urlread(f'{base_url}/404')
with expect_fail(HTTP503ServiceUnavailableError, contains='details from the server'): urljson(f'{base_url}/503')
with tempfile.TemporaryDirectory() as d, expect_fail(HTTP500InternalServerError): urlsave(f'{base_url}/500', d)
with expect_fail(HTTP4xxClientError): urlopen(f'{base_url}/499')
try: urlread(f'{base_url}/502')
except HTTP5xxServerError as e: res = type(e).__name__, e.code
res

source

urlvalid

def urlvalid(
    x
):

Test if x is a valid URL

assert urlvalid('http://www.google.com/')
assert not urlvalid('www.google.com/')
assert not urlvalid(1)

Basic client/server

source

start_server

def start_server(
    port, host:NoneType=None, dgram:bool=False, reuse_addr:bool=True, n_queue:NoneType=None
):

Create a socket server on port, with optional host, of type dgram

You can create a TCP client and server by passing an int as port and optional host. host defaults to your main network interface if not provided. You can create a Unix socket client and server by passing a string to port. A SOCK_STREAM socket is created by default, unless you pass dgram=True, in which case a SOCK_DGRAM socket is created. n_queue sets the listening queue size.

source

start_client

def start_client(
    port, host:NoneType=None, dgram:bool=False
):

Create a socket client on port, with optional host, of type dgram

source

tobytes

def tobytes(
    s:str
)->bytes:

Convert s into HTTP-ready bytes format

test_eq(tobytes('foo\nbar'), b'foo\r\nbar')

source

http_response

def http_response(
    body:NoneType=None, status:int=200, hdrs:NoneType=None, **kwargs
):

Create an HTTP-ready response, adding kwargs to hdrs

exp = b'HTTP/1.1 200 OK\r\nUser-Agent: me\r\nContent-Length: 4\r\n\r\nbody'
test_eq(http_response('body', 200, User_Agent='me'), exp)

source

recv_once

def recv_once(
    host:str='localhost', port:int=8000
):

Spawn a thread to receive a single HTTP request and store in d['r']

Waiting

waitfor polls a callable until it returns truthy – often needed when coordinating with servers or other processes that take time to change state. The sync and async variants share one polling loop (_waitfor), so they behave identically apart from how they sleep.

source

waitfor_async

async def waitfor_async(
    f, timeout:int=20, msg:NoneType=None
):

Async version of waitfor

source

waitfor

def waitfor(
    f, timeout:int=20, msg:NoneType=None
):

Call f every 0.1s until it returns truthy, raising TimeoutError with msg after timeout secs

A predicate that succeeds on its third call returns normally; one that never succeeds raises after timeout seconds:

vals = iter([0,0,1])
waitfor(lambda: next(vals))
test_fail(lambda: waitfor(lambda: False, timeout=0.15), contains='Timeout')

vals = iter([0,1])
await waitfor_async(lambda: next(vals))

source

is_port_free

def is_port_free(
    port, host:str='localhost'
):

Is port free on host?

source

free_port

def free_port(
    host:str='localhost'
):

A free TCP port on host, picked by the OS

p = free_port()
assert is_port_free(p)

source

wait_port_free_async

async def wait_port_free_async(
    port, host:str='localhost', max_wait:int=20
):

Async wait for port to be free on host

source

wait_port_free

def wait_port_free(
    port, host:str='localhost', max_wait:int=20
):

Wait for port to be free on host

A port held by a start_server socket is busy until the socket closes, at which point wait_port_free returns promptly. Note that start_server binds the hostname’s interface by default, while the port helpers default to localhost – so when combining them, pass an explicit host to keep both on the same interface:

s = start_server(8877, host='localhost')
assert not is_port_free(8877)
s.close()
wait_port_free(8877)
assert is_port_free(8877)
await wait_port_free_async(8877)