Network functionality

Network, HTTP, and URL functions

from fastcore.test import *
from nbdev.showdoc import *
from fastcore.nb_imports import *

URLs

---

source

urlquote

 urlquote (url)

Update url’s path with urllib.parse.quote

urlquote("https://github.com/fastai/fastai/compare/master@{1.day.ago}…master")

'https://github.com/fastai/fastai/compare/master@%7B1.day.ago%7D%E2%80%A6master'

urlquote("https://www.google.com/search?q=你好")

'https://www.google.com/search?q=%E4%BD%A0%E5%A5%BD'

---

source

urlwrap

 urlwrap (url, data=None, headers=None)

Wrap url in a urllib Request with urlquote

---

source

HTTP4xxClientError

 HTTP4xxClientError (url, code, msg, hdrs, fp)

Base class for client exceptions (code 4xx) from url* functions

---

source

HTTP5xxServerError

 HTTP5xxServerError (url, code, msg, hdrs, fp)

Base class for server exceptions (code 5xx) from url* functions

---

source

urlopener

 urlopener ()

---

source

urlopen

 urlopen (url, data=None, headers=None, timeout=None, **kwargs)

Like urllib.request.urlopen, but first urlwrap the url, and encode data

With urlopen, the body of the response will also be returned in addition to the message if there is an error:

try: urlopen('https://api.github.com/v3')
except HTTPError as e: 
    print(e.code, e.msg)
    assert 'documentation_url' in e.msg

404 Not Found
====Error Body====
{
  "message": "Not Found",
  "documentation_url": "https://docs.github.com/rest",
  "status": "404"
}

---

source

urlread

 urlread (url, data=None, headers=None, decode=True, return_json=False,
          return_headers=False, timeout=None, **kwargs)

Retrieve url, using data dict or kwargs to POST if present

---

source

urljson

 urljson (url, data=None, headers=None, timeout=None)

Retrieve url and decode json

test_eq(urljson('https://httpbin.org/get')['headers']['User-Agent'], url_default_headers['User-Agent'])

---

source

urlcheck

 urlcheck (url, headers=None, timeout=10)

---

source

urlclean

 urlclean (url)

Remove fragment, params, and querystring from url if present

test_eq(urlclean('http://a.com/b?c=1#d'), 'http://a.com/b')

---

source

urlretrieve

 urlretrieve (url, filename=None, reporthook=None, data=None,
              headers=None, timeout=None)

Same as urllib.request.urlretrieve but also works with Request objects

---

source

urldest

 urldest (url, dest=None)

---

source

urlsave

 urlsave (url, dest=None, reporthook=None, headers=None, timeout=None)

Retrieve url and save based on its name

#skip
with tempfile.TemporaryDirectory() as d: urlsave('http://www.google.com/index.html', d)

---

source

urlvalid

 urlvalid (x)

Test if x is a valid URL

assert urlvalid('http://www.google.com/')
assert not urlvalid('www.google.com/')
assert not urlvalid(1)

---

source

urlrequest

 urlrequest (url, verb, headers=None, route=None, query=None, data=None,
             json_data=True)

Request for url with optional route params replaced by route, plus query string, and post data

hdr = {'Hdr1':'1', 'Hdr2':'2'}
req = urlrequest('http://example.com/{foo}/1', 'POST',
                 headers=hdr, route={'foo':'3'}, query={'q':'4'}, data={'d':'5'})

test_eq(req.headers, hdr)
test_eq(req.full_url, 'http://example.com/3/1?q=4')
test_eq(req.method, 'POST')
test_eq(req.data, b'{"d": "5"}')

req = urlrequest('http://example.com/{foo}/1', 'POST', data={'d':'5','e':'6'}, headers=hdr, json_data=False)
test_eq(req.data, b'd=5&e=6')

---

source

Request.summary

 Request.summary (skip=None)

Summary containing full_url, headers, method, and data, removing skip from headers

req.summary(skip='Hdr1')

{'full_url': 'http://example.com/{foo}/1',
 'method': 'POST',
 'data': b'd=5&e=6',
 'headers': {'Hdr2': '2'}}

---

source

urlsend

 urlsend (url, verb, headers=None, decode=True, route=None, query=None,
          data=None, json_data=True, return_json=True,
          return_headers=False, debug=None, timeout=None)

Send request with urlrequest, converting result to json if return_json

---

source

do_request

 do_request (url, post=False, headers=None, **data)

Call GET or json-encoded POST on url, depending on post

Basic client/server

---

source

start_server

 start_server (port, host=None, dgram=False, reuse_addr=True,
               n_queue=None)

Create a socket server on port, with optional host, of type dgram

You can create a TCP client and server pass an int as port and optional host. host defaults to your main network interface if not provided. You can create a Unix socket client and server by passing a string to port. A SOCK_STREAM socket is created by default, unless you pass dgram=True, in which case a SOCK_DGRAM socket is created. n_queue sets the listening queue size.

---

source

start_client

 start_client (port, host=None, dgram=False)

Create a socket client on port, with optional host, of type dgram

---

source

tobytes

 tobytes (s:str)

Convert s into HTTP-ready bytes format

test_eq(tobytes('foo\nbar'), b'foo\r\nbar')

---

source

http_response

 http_response (body=None, status=200, hdrs=None, **kwargs)

Create an HTTP-ready response, adding kwargs to hdrs

exp = b'HTTP/1.1 200 OK\r\nUser-Agent: me\r\nContent-Length: 4\r\n\r\nbody'
test_eq(http_response('body', 200, User_Agent='me'), exp)

---

source

recv_once

 recv_once (host:str='localhost', port:int=8000)

Spawn a thread to receive a single HTTP request and store in d['r']