Network Programming

with Python
IPv4 and IPv6
Sébastien Tixeuil
sebastien.Tixeuil@lip6.fr

IPv4 Names IPv4 Names

postetixeuil4.rsr.lip6.fr
from socket import * from socket import *

print(gethostname()) print(gethostname())
postetixeuil4.rsr.lip6.fr

print(getfqdn()) print(getfqdn())

132.227.104.15
print(gethostbyname('lip6.fr')) print(gethostbyname('lip6.fr'))

print(gethostbyaddr('132.227.104.15')) print(gethostbyaddr('132.227.104.15'))
('ww.lip6.fr', ['15.104.227.132.in-
addr.arpa'], ['132.227.104.15'])

print(gethostbyname(getfqdn())) print(gethostbyname(getfqdn()))
132.227.84.244
 IPv4-IPv6 Names IPv4-IPv6 Names
infolist = getaddrinfo('lip6.fr','www') infolist = getaddrinfo('lip6.fr','www')

print(infolist) print(infolist)
[(<AddressFamily.AF_INET: 2>,
<SocketKind.SOCK_DGRAM: 2>, 17,
'', ('132.227.104.15', 80)),
info = infolist[1] info = infolist[1]
(<AddressFamily.AF_INET: 2>,
<SocketKind.SOCK_STREAM: 1>, 6,
'', ('132.227.104.15', 80))]
print(info) print(info)

(<AddressFamily.AF_INET: 2>,
<SocketKind.SOCK_STREAM: 1>, 6,
s = socket(*info[0:3]) s = socket(*info[0:3])
'', ('132.227.104.15', 80))

s.connect(info[4]) s.connect(info[4])

Strings vs. Bytes

• Strings are meant for general Unicode support

• Bytes are what is sent/received through the network

Strings and Bytes • Encoding of Strings into Bytes before sending

toSend = str.encode(‘utf-8’)

• Decoding Bytes into Strings when receiving

str = received.decode(‘utf-8’)
 UDP Python Client UDP Python Server
from socket import * from socket import *

serverName = ‘A.B.C.D’ serverPort = 1234

serverPort = 1234 serverSocket = socket(AF_INET,SOCK_DGRAM)

clientSocket = socket(AF_INET,SOCK_DGRAM) serverSocket.bind((‘’,serverPort))

print(‘server ready’)
message = input(‘lowercase sentence:’).encode(‘utf-8’)
while True:
clientSocket.sendto(message,(serverName,serverPort))
message, clientAddress = serverSocket.recvfrom(2048)
modifiedMessage, serverAddress = clientSocket.recvfrom(2048)
modifiedMessage = message.decode(‘utf-8’).upper()
print(modifiedMessage.decode(‘utf-8’))
serverSocket.sendto(modifiedMessage.encode(‘utf-8’),
clientSocket.close() clientAddress)

Byte Order over the Network

from struct import *

Numbers and
print(hex(1234))

print(pack('<i',1234))

Byte Order print(pack('>i',1234))

print(pack('!i',1234))

print(unpack('>i',b'\x00\x00\x04\xd2'))

print(unpack('!i',b'\x00\x00\x04\xd2'))
Byte Order over the Network
from struct import *

print(hex(1234))
0x4d2

print(pack('<i',1234))
b'\xd2\x04\x00\x00'

b'\x00\x00\x04\xd2'
Network Exceptions
print(pack('>i',1234))
b'\x00\x00\x04\xd2'
print(pack('!i',1234))

(1234,)
print(unpack('>i',b'\x00\x00\x04\xd2'))
(1234,)

print(unpack('!i',b'\x00\x00\x04\xd2'))

Network Exceptions Network Exceptions

from socket import *

• OSError: almost every failure that can happen

during a network connection
try:

• socket.gaierror: address-related error infolist = getaddrinfo('nonexistent.com','www')

• socket.timeout: timeout expired except gaierror:

print("This host does not seem to exist")

 UDP Packet Drops
Server Process Client Process

Create & Bind Socket Create Socket

UPD Packet Drops Read Datagram

BLOCKING
Create, Send Datagram

Send Response
Datagram Read Datagram
BLOCKING

Close Socket

UDP Packet Drops UDP Packet Drops

… …

delay = 0.1 # sec delay = 0.1 # sec

while True: while True:

clientSocket.sendto(message,(serverName,serverPort)) clientSocket.sendto(message,(serverName,serverPort))

clientSocket.settimeout(delay)
clientSocket.settimeout(delay)
try:
try:
modifiedMessage, serverAddress = clientSocket.recvfrom(2048)
modifiedMessage, serverAddress = clientSocket.recvfrom(2048)
except socket.timeout:
except socket.timeout:
delay *= 2
delay *= 2
if delay > 2.0:
else: raise RuntimeError(‘server seems down’)

break else:

print(modifiedMessage.decode(‘utf-8’)) break

clientSocket.close() print(modifiedMessage.decode(‘utf-8’))
 UDP Broadcast Client
from socket import *

broadcastAddr = ‘W.X.Y.255’ # assuming 255.255.255.0 netmask

serverPort = 1234

UDP Broadcast clientSocket = socket(AF_INET,SOCK_DGRAM)

clientSocket.setsockopt(SOL_SOCKET,SO_BROADCAST,1)

message = input(‘lowercase sentence:’).encode(‘utf-8’)

clientSocket.sendto(message,(serverName,serverPort))

modifiedMessage, serverAddress = clientSocket.recvfrom(2048)

print(modifiedMessage.decode(‘utf-8’))

clientSocket.close()

UDP Broadcast Server

from socket import *

serverPort = 1234

serverSocket = socket(AF_INET,SOCK_DGRAM)

serverSocket.bind((‘’,serverPort))

print(‘server ready’)
TCP Sending
while True:

message, clientAddress = serverSocket.recvfrom(2048)

modifiedMessage = message.decode(‘utf-8’).upper()

serverSocket.sendto(modifiedMessage.encode(‘utf-8’),
clientAddress)
 TCP send TCP send Loop
…
• If TCP/IP stack has enough room, send returns
message=input(‘lowercase sentence:’).encode(‘utf-8’)
immediately, and the complete message will be
handled bytes_sent = 0

while bytes_sent < len(message):

• If TCP/IP stack is full, send is blocking
message_remaining = message[bytes_sent:]
• If TCP/IP stack is almost full, send returns
bytes_sent +=
immediately, but only a part of the message will be clientSocket.send(message_remaining)
handled
…

TCP sendall

message = input(‘lowercase
sentence:’).encode(‘utf-8’)
TCP Receiving
clientSocket.sendall(message)

…
 TCP recv TCP recvall ?
• If TCP/IP stack is full enough, recv returns
• Deciding when all data is received is application
immediately, and the specified size message will
specific
be delivered
• Fixed size messages
• If TCP/IP stack is empty, recv is blocking
• Message size is announced before data is sent
• If TCP/IP stack is not empty, recv returns
immediately, but only a fraction of the specified • Special delimiters announce end of data
size will be delivered

TCP recvall
def recvall(sock, length):

blocks = []

while length:

Framing and Quoting block = sock.recv(length)

if not block:

raise EOFError('socket closed with %d bytes left'

' in this block'.format(length))

length -= len(block)

blocks.append(block)

return b''.join(blocks)
 Sending a Block Receiving a Block
from struct import *
from struct import *
from socket import *
from socket import *
header_struct = Struct('!I')
header_struct = Struct('!I')
def put_block(sock, message):
def get_block(sock):
block_length = len(message)
data = recvall(sock, header_struct.size)
sock.sendall(header_struct.pack(
(block_length,) = header_struct.unpack(data)
block_length))
return recvall(sock, block_length)
sock.sendall(message)

JSON
from json import *
from socket import *

print(dumps(['aéçèà',1234,[2,3,4,5,6]]))

XML and JSON print(loads('["a\u00e9\u00e7\u00e8\u00e0", 1234, [2, 3, 4, 5, 6]]'))

s = socket(AF_INET,SOCK_STREAM)

try:
print(dumps(s))

except TypeError:
print("this data does not seem serializable with JSON")
 JSON
from json import *
from socket import *
["a\u00e9\u00e7\u00e8\u00e0", 1234, [2, 3, 4, 5, 6]]

print(dumps(['aéçèà',1234,[2,3,4,5,6]]))

print(loads('["a\u00e9\u00e7\u00e8\u00e0", 1234, [2, 3, 4, 5, 6]]'))

['aéçèà', 1234, [2, 3, 4, 5, 6]]

Compression
s = socket(AF_INET,SOCK_STREAM)

try:
print(dumps(s))

except TypeError:
print("this data does not seem serializable with JSON")

Python Data Compression Python Data Compression

from zlib import * from zlib import *

str = b'A very long test string to evaluate compression and how much it improves str = b'A very long test string to evaluate compression and how much it improves
bandwidth usage' bandwidth usage'

print(len(str)) print(len(str))

88

data = compress(str) data = compress(str) b'x\x9c\x15\xc9\xd1\t\xc0 \x0c\x05\xc0U\xde*

\x1d%\xd5\xa0\x015%\x89J\xb7\xaf\xfd;\xb8\x0b
print(data) print(data) \x8b\xedE\xd3Q\x10\xec\x01\x0f\x93\xdf\n^\xd4&
\x05#i\x7f\x8c\xddE\x07hdT\xdd\xe83UH@N\xe9b
print(len(data)) print(len(data)) \xc7}fK\x8e\x8a\xe9T\xf8\x03\xad\\!\x05’

78

d = decompressobj() d = decompressobj() b'A very long test string to evaluate

compression and how much it improves bandwidth
newdata = d.decompress(data) newdata = d.decompress(data) usage'
print(newdata) print(newdata)
 Sequential TCP Server
…

def handle_client(clientSocket):

while True:

Server-side received = clientSocket.recv(4096)

if not received:

clientSocket.close()

Programming else:

to_send = received.decode('utf-8').upper().encode('utf-8')

clientSocket.sendall(to_send)

while True:

connectionSocket, address = serverSocket.accept()

handle_client(connectionSocket)

Sequential TCP Server Multi-thread TCP Server

Server Process Client Process Server Process Client Process
Create & Bind Create & Bind
Create Socket Create Socket
Server Socket Server Socket

TCP TCP
Wait for Incoming Wait for Incoming
Connection on Server Handshake Create Connection Connection on Server Handshake Create Connection
Socket Blocking Socket Blocking
Blocking Blocking

Create Client-specific
Thread
Send Request
Send Request Read Request from
Read Request from Blocking
Blocking Connection Socket
Connection Socket
Blocking
Blocking

Send Response to
Send Response to Connection Socket
Connection Socket Blocking Read Response
Blocking Read Response Blocking
Blocking

Close
Close Connection Socket
Connection Socket Close Socket
Close Socket

Server Sub-Process
 Multi-thread TCP Server Multi-Thread TCP Server
Client 2 Process Server Process Client 1 Process
from threading import *
Create & Bind
Create Socket Create Socket
Server Socket
…

def handle_client(clientSocket):
TCP TCP
Wait for Incoming
Create Connection Handshake Connection on Server Handshake Create Connection while True:
Blocking Socket Blocking
Blocking
received = clientSocket.recv(4096)
Create Client-specific Create Client-specific
Thread Thread if not received:
Send Request Send Request
Blocking Read Request from Read Request from Blocking
Connection Socket Connection Socket clientSocket.close()
Blocking Blocking
else:

Send Response to Send Response to

to_send = received.decode('utf-8').upper().encode('utf-8')
Connection Socket Connection Socket
Read Response Blocking Blocking Read Response clientSocket.sendall(to_send)
Blocking Blocking

while True:

Close Close connectionSocket, address = serverSocket.accept()

Connection Socket Connection Socket
Close Socket Close Socket
Thread(target=handle_client,args=(connectionSocket,)).start()

Server Sub-Process Server Sub-Process

Thread Pool TCP Server Thread Pool TCP Server

Client 2 Process Server Process Client 1 Process
Create & Bind …
Create Socket Server Socket Create Socket
Create Thread pool def handle_client(listeningSocket):
Create Create
Thread 1 Thread 2
while True:
TCP TCP
Wait for Incoming Wait for Incoming
Create Connection Handshake Connection on Server Connection on Server Handshake Create Connection clientSocket, address = listeningSocket.accept()
Blocking Socket Socket Blocking
Blocking Blocking
while True:

received = clientSocket.recv(4096)
Send Request Send Request
Blocking Read Request from Read Request from Blocking
Connection Socket Connection Socket if not received:
Blocking Blocking
clientSocket.close()

else:
Send Response to Send Response to
Connection Socket Connection Socket
Read Response Blocking Blocking Read Response to_send = received.decode('utf-8').upper().encode('utf-8')
Blocking Blocking

clientSocket.sendall(to_send)

Close Close for i in range(4):

Connection Socket Connection Socket
Close Socket Close Socket
Thread(target=handle_client,args=(serverSocket,)).start()

Server Sub-Process Server Sub-Process

Multiplexing TCP Server Multiplexing TCP Server
from select import *
Server Process …

Create & Bind

my_poll = poll()
Server Socket
my_poll.register(serverSocket,POLLIN)

Wait for
Socket-related event
Blocking sockets = {serverSocket.fileno(): serverSocket}

# retrieve socket object from fileno

received = dict()

Process Incoming
Read Request from Send Response to
# bytes received from fileno, that are not yet processed
Connection on Server Close
Connection Socket Connection Socket
Socket Connection Socket
Non-Blocking Non-Blocking
Non-Blocking to_send = dict()

# bytes to be sent from fileno, that have been processed

Multiplexing TCP Server Multiplexing TCP Server

while True:
elif sockets[fd] is serverSocket:
for fd, event in my_poll.poll():

if event & (POLLHUP|POLLERR|POLLNVAL): connectionSocket, address =

serverSocket.accept()
received.pop(fd)

to_send.pop(fd) sockets[connectionSocket.fileno()] =
connectionSocket
my_poll.unregister(fd)

del sockets[fd] my_poll.register(connectionSocket,

POLLIN)
sockets.pop(fd)
Multiplexing TCP Server Multiplexing TCP Server
else: if event & POLLOUT:

if event & POLLIN: data = received.pop(fd).decode(‘utf-8’)

data = sockets[fd].recv(4096)
data = data.upper().encode(‘utf-8')
if not data:
if fd in to_send:
sockets[fd].close()
data = to_send.pop(fd) + data
continue
n = sockets[fd].send(data)
if fd in received:
if n < len(data):
received[fd] += data

else:
to_send[fd] = data[n:]

received[fd] = data else:

my_poll.modify(fd,POLLIN|POLLOUT) my_poll.modify(fd,POLLIN)

Multiplexing TCP Server Multiplexing TCP Server

from select import *

… Server Process
while True:
Create & Bind
Server Socket
for fd, event in my_poll.poll():

if event & (POLLHUP|POLLERR|POLLNVAL):

… Wait for
Socket-related event
Blocking
elif sockets[fd] is serverSocket:

else:
Process Incoming
Read Request from Send Response to
if event & POLLIN: Connection on Server
Connection Socket Connection Socket
Close
Socket Connection Socket
Non-Blocking Non-Blocking
Non-Blocking
…

if event & POLLOUT:

…
 Remote Procedure Call
• The main objective is to make client-server calls
(almost) transparent to the programmer

XML-RPC • The server defines a set of functions and makes

them available through the network

• The client calls the function (almost) as if they

were local

• No need to define a protocol, a data format, etc.

Python XML-RPC Server Python XML-RPC Server

from operator import *
…

from math import *

server = SimpleXMLRPCServer(('127.0.0.1', 7001))

from xmlrpc.server import *

server.register_introspection_functions()

from functools import *

server.register_multicall_functions()
def addtogether(*things):
server.register_function(addtogether)
return reduce(add,things)

server.register_function(quadratic)
def quadratic(a,b,c):

print("Server ready")
b24ac = sqrt(b*b - 4.0*a*c)

return list((set([(-b-b24ac)/2.0*a,(-b+b24ac)/2.0*a]))) server.serve_forever()

 Python XML-RPC Client Python XML-RPC Client
from xmlrpc.client import * from xmlrpc.client import *

proxy = ServerProxy('http:// proxy = ServerProxy('http://

127.0.0.1:7001') 127.0.0.1:7001')

print(proxy.addtogether('x','y','z')) print(proxy.addtogether('x','y','z')) xyz

print(proxy.addtogether(1,2,3,4,5)) print(proxy.addtogether(1,2,3,4,5)) 15

print(proxy.quadratic(2,-4,0)) print(proxy.quadratic(2,-4,0)) [0.0, 8.0]

print(proxy.quadratic(1,2,1)) print(proxy.quadratic(1,2,1)) [-1.0]

Wireshark Wireshark
POST /RPC2 HTTP/1.1 POST /RPC2 HTTP/1.1
Host: 127.0.0.1:7001
Host: 127.0.0.1:7001 Accept-Encoding: gzip
Accept-Encoding: gzip Content-Type: text/xml
Content-Type: text/xml User-Agent: Python-xmlrpc/3.4
HTTP/1.0 200 OK Content-Length: 330
User-Agent: Python-xmlrpc/3.4 HTTP/1.0 200 OK
Server: BaseHTTP/0.6 Python/3.4.3
Content-Length: 258 <?xml version='1.0'?> Server: BaseHTTP/0.6 Python/3.4.3
Date: Mon, 18 Jan 2016 13:41:45 GMT
<methodCall> Date: Mon, 18 Jan 2016 13:41:45 GMT
Content-type: text/xml <methodName>addtogether</methodName>
<?xml version='1.0'?> Content-type: text/xml
Content-length: 129 <params>
<methodCall> Content-length: 122
<param>
<methodName>addtogether</methodName> <value><int>1</int></value>
<?xml version='1.0'?>
<params> </param> <?xml version='1.0'?>
<methodResponse> <param>
<param> <methodResponse>
<params> <value><int>2</int></value>
<value><string>x</string></value> <params>
<param> </param>
</param> <param> <param>
<value><string>xyz</string></value>
<param> <value><int>3</int></value> <value><int>15</int></value>
</param> </param>
<value><string>y</string></value> </param>
</params> <param>
</param> </params>
</methodResponse> <value><int>4</int></value>
<param> </param> </methodResponse>
<value><string>z</string></value> <param>
</param> <value><int>5</int></value>
</param>
</params> </params>
</methodCall> </methodCall>
 Conclusion

• Python makes network programming really easy

• A number of Python modules have been developed

for popular Internet-based protocols

• The socket API remains important for developing

new protocols, and accessing lower layers