sockets, servers, and public IP addresses

[ihatehippies]

I've been working with python's SocketServer.ThreadingTCPServer a bit and I'm having some difficulty making a publicly available server. Everything works fine when I set the server address as 'localhost'

self.server = SocketServer.ThreadingTCPServer(('localhost', constants.cmdport), Handler)

As long as the both the server and client are on the same computer and both specify 'localhost' the connections are made without a hitch. My program uses pystun to obtain my public ip. When I pass that public ip to the client and try to make a connection over the internet I get [errno 10061] No connection could be made..... as if the port was not opened. Both computers are behind the same router and subsequently have the same public ip, but I've read that the router will direct traffic to the computer with the open port. It seems almost all of the examples out there are for servers on the localhost. I'm sure there is a painfully simple answer, but I can't seem to find it. How do I establish a client/server connection via the real not just on the same computer. Thanks

[Tech B]

Look into port forwarding. You will have to log into your router and set it up there. Here is a link to help ya out.

[ihatehippies]

Yeah, I've looked into that a bit. Just curious, if I wasn't behind a router what would the TCPServer look like? I've seen

self.server = SocketServer.ThreadingTCPServer(('', port), Handler)

Would you ever put your pubic IP as the server_address? How do other applications do it? Is it just because they use common ports (ie HTTP on 80 etc...) If you use a random 4 digit port not regularly used do you have to set up port forwarding?

[ihatehippies]

In case someone else finds this thread looking for answers. Here is the code that I found on the internet that works for me.

Natpunch.py:

# Written by John Hoffman
# derived from NATPortMapping.py by Yejun Yang
# and from example code by Myers Carpenter
# see LICENSE.txt for license information

import socket
from traceback import print_exc
from subnetparse import IP_List
try:
    from clock import clock
except ImportError:
    from time import clock
from __init__ import createPeerID, resetPeerIDs
try:
    True
except:
    True = 1
    False = 0

DEBUG = 1


resetPeerIDs()
EXPIRE_CACHE = 30 # seconds
ID = "BT-"+createPeerID()[-4:]

try:
    import pythoncom, win32com.client
    _supported = 1
except ImportError:
    _supported = 0



class _UPnP1:   # derived from Myers Carpenter's code
                # seems to use the machine's local UPnP
                # system for its operation.  Runs fairly fast

    def __init__(self):
        self.map = None
        self.last_got_map = -10e10

    def _get_map(self):
        if self.last_got_map + EXPIRE_CACHE < clock():
            try:
                dispatcher = win32com.client.Dispatch("HNetCfg.NATUPnP")
                self.map = dispatcher.StaticPortMappingCollection
                self.last_got_map = clock()
            except:
                self.map = None
        return self.map

    def test(self):
        try:
            assert self._get_map()     # make sure a map was found
            success = True
        except:
            success = False
        return success


    def open(self, ip, p):
        map = self._get_map()
        try:
            map.Add(p,'TCP',p,ip,True,ID)
            if DEBUG:
                print 'port opened: '+ip+':'+str(p)
            success = True
        except:
            if DEBUG:
                print "COULDN'T OPEN "+str(p)
                print_exc()
            success = False
        return success


    def close(self, p):
        map = self._get_map()
        try:
            map.Remove(p,'TCP')
            success = True
            if DEBUG:
                print 'port closed: '+str(p)
        except:
            if DEBUG:
                print 'ERROR CLOSING '+str(p)
                print_exc()
            success = False
        return success


    def clean(self, retry = False):
        if not _supported:
            return
        try:
            map = self._get_map()
            ports_in_use = []
            for i in xrange(len(map)):
                try:
                    mapping = map[i]
                    port = mapping.ExternalPort
                    prot = str(mapping.Protocol).lower()
                    desc = str(mapping.Description).lower()
                except:
                    port = None
                if port and prot == 'tcp' and desc[:3] == 'bt-':
                    ports_in_use.append(port)
            success = True
            for port in ports_in_use:
                try:
                    map.Remove(port,'TCP')
                except:
                    success = False
            if not success and not retry:
                self.clean(retry = True)
        except:
            pass


class _UPnP2:   # derived from Yejun Yang's code
                # apparently does a direct search for UPnP hardware
                # may work in some cases where _UPnP1 won't, but is slow
                # still need to implement "clean" method

    def __init__(self):
        self.services = None
        self.last_got_services = -10e10
                           
    def _get_services(self):
        if not self.services or self.last_got_services + EXPIRE_CACHE < clock():
            self.services = []
            try:
                f=win32com.client.Dispatch("UPnP.UPnPDeviceFinder")
                for t in ( "urn:schemas-upnp-org:service:WANIPConnection:1",
                           "urn:schemas-upnp-org:service:WANPPPConnection:1" ):
                    try:
                        conns = f.FindByType(t,0)
                        for c in xrange(len(conns)):
                            try:
                                svcs = conns[c].Services
                                for s in xrange(len(svcs)):
                                    try:
                                        self.services.append(svcs[s])
                                    except:
                                        pass
                            except:
                                pass
                    except:
                        pass
            except:
                pass
            self.last_got_services = clock()
        return self.services

    def test(self):
        try:
            assert self._get_services()    # make sure some services can be found
            success = True
        except:
            success = False
        return success


    def open(self, ip, p):
        svcs = self._get_services()
        success = False
        for s in svcs:
            try:
                s.InvokeAction('AddPortMapping',['',p,'TCP',p,ip,True,ID,0],'')
                success = True
            except:
                pass
        if DEBUG and not success:
            print "COULDN'T OPEN "+str(p)
            print_exc()
        return success


    def close(self, p):
        svcs = self._get_services()
        success = False
        for s in svcs:
            try:
                s.InvokeAction('DeletePortMapping', ['',p,'TCP'], '')
                success = True
            except:
                pass
        if DEBUG and not success:
            print "COULDN'T OPEN "+str(p)
            print_exc()
        return success


class _UPnP:    # master holding class
    def __init__(self):
        self.upnp1 = _UPnP1()
        self.upnp2 = _UPnP2()
        self.upnplist = (None, self.upnp1, self.upnp2)
        self.upnp = None
        self.local_ip = None
        self.last_got_ip = -10e10
        self.test(1)
        
    def get_ip(self):
        if self.last_got_ip + EXPIRE_CACHE < clock():
            local_ips = IP_List()
            local_ips.set_intranet_addresses()
            try:
                for info in socket.getaddrinfo(socket.gethostname(),0,socket.AF_INET):
                            # exception if socket library isn't recent
                    self.local_ip = info[4][0]
                    if local_ips.includes(self.local_ip):
                        self.last_got_ip = clock()
                        if DEBUG:
                            print 'Local IP found: '+self.local_ip
                        break
                else:
                    raise ValueError('couldn\'t find intranet IP')
            except:
                self.local_ip = None
                if DEBUG:
                    print 'Error finding local IP'
                    print_exc()
        return self.local_ip

    def test(self, upnp_type):
        if DEBUG:
            print 'testing UPnP type '+str(upnp_type)
        if not upnp_type or not _supported or self.get_ip() is None:
            if DEBUG:
                print 'not supported'
            return 0
        pythoncom.CoInitialize()                # leave initialized
        self.upnp = self.upnplist[upnp_type]    # cache this
        if self.upnp.test():
            if DEBUG:
                print 'ok'
            return upnp_type
        if DEBUG:
            print 'tested bad'
        return 0

    def open(self, p):
        assert self.upnp, "must run UPnP_test() with the desired UPnP access type first"
        return self.upnp.open(self.get_ip(), p)

    def close(self, p):
        assert self.upnp, "must run UPnP_test() with the desired UPnP access type first"
        return self.upnp.close(p)

    def clean(self):
        return self.upnp1.clean()

_upnp_ = _UPnP()

accompanying __init__.py file:

from time import clock, time
from sha import sha
from types import StringType

try:
    from os import getpid
except ImportError:
    def getpid():
        return 1


_idprefix = 'R'
mapbase64 = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz.-'
_idrandom = [None]

def resetPeerIDs():
    try:
        f = open('/dev/urandom', 'rb')
        x = f.read(20)
        f.close()
    except:
        x = ''

    l1 = 0
    t = clock()
    while t == clock():
        l1 += 1
    l2 = 0
    t = long(time()*100)
    while t == long(time()*100):
        l2 += 1
    l3 = 0
    if l2 < 1000:
        t = long(time()*10)
        while t == long(clock()*10):
            l3 += 1
    x += ( repr(time()) + '/' + str(time()) + '/'
           + str(l1) + '/' + str(l2) + '/' + str(l3) + '/'
           + str(getpid()) )

    s = ''
    for i in sha(x).digest()[-11:]:
        s += mapbase64[ord(i) & 0x3F]
    _idrandom[0] = s

def createPeerID(ins = '---'):
    if type(ins) != StringType:
        raise Exception, "swapper__init__: createPeerID"
    if len(ins) != 3:
        raise Exception, "swapper__init__: createPeerID"
    return _idprefix + ins + _idrandom[0]

A simplified implementation of the code:

port = 13333
from natpunch import _upnp_ as uPnP
import SocketServer
import socket

class Handler(SocketServer.BaseRequestHandler):

   def handle(self):
      print 'command received'
      data = self.request.recv(2048).strip()
      # do something with the data....

uPnP.open(port)

hostname = socket.gethostbyname(socket.gethostname())
# In my actual code I put the server inside it's own thread
server = SocketServer.ThreadingTCPServer((hostname, port), Handler)

This allows me to connect to my public ip address found by using pystun

Hope this is useful to someone

[ihatehippies]

forgot to include a required module

subnetparse.py

# Written by John Hoffman
# see LICENSE.txt for license information

from bisect import bisect, insort

try:
    True
except:
    True = 1
    False = 0
    bool = lambda x: not not x

hexbinmap = {
    '0': '0000',
    '1': '0001',
    '2': '0010',
    '3': '0011',
    '4': '0100',
    '5': '0101',
    '6': '0110',
    '7': '0111',
    '8': '1000',
    '9': '1001',
    'a': '1010',
    'b': '1011',
    'c': '1100',
    'd': '1101',
    'e': '1110',
    'f': '1111',
    'x': '0000',
}

chrbinmap = {}
for n in xrange(256):
    b = []
    nn = n
    for i in xrange(8):
        if nn & 0x80:
            b.append('1')
        else:
            b.append('0')
        nn <<= 1
    chrbinmap[n] = ''.join(b)


def to_bitfield_ipv4(ip):
    ip = ip.split('.')
    if len(ip) != 4:
        raise ValueError, "bad address"
    b = []
    for i in ip:
        b.append(chrbinmap[int(i)])
    return ''.join(b)

def to_bitfield_ipv6(ip):
    b = ''
    doublecolon = False

    if ip == '':
        raise ValueError, "bad address"
    if ip == '::':      # boundary handling
        ip = ''
    elif ip[:2] == '::':
        ip = ip[1:]
    elif ip[0] == ':':
        raise ValueError, "bad address"
    elif ip[-2:] == '::':
        ip = ip[:-1]
    elif ip[-1] == ':':
        raise ValueError, "bad address"
    for n in ip.split(':'):
        if n == '':     # double-colon
            if doublecolon:
                raise ValueError, "bad address"
            doublecolon = True
            b += ':'
            continue
        if n.find('.') >= 0: # IPv4
            n = to_bitfield_ipv4(n)
            b += n + '0'*(32-len(n))
            continue
        n = ('x'*(4-len(n))) + n
        for i in n:
            b += hexbinmap[i]
    if doublecolon:
        pos = b.find(':')
        b = b[:pos]+('0'*(129-len(b)))+b[pos+1:]
    if len(b) != 128:   # always check size
        raise ValueError, "bad address"
    return b

ipv4addrmask = to_bitfield_ipv6('::ffff:0:0')[:96]

class IP_List:
    def __init__(self, entrylist=None):
        self.ipv4list = []
        self.ipv6list = []
        if entrylist:
            for ip, depth in entrylist:
                self._append(ip,depth)
            self.ipv4list.sort()
            self.ipv6list.sort()


    def __nonzero__(self):
        return bool(self.ipv4list or self.ipv6list)


    def _append(self, ip, depth = 256):
        if ip.find(':') < 0:        # IPv4
            self.ipv4list.append(to_bitfield_ipv4(ip)[:depth])
        else:
            b = to_bitfield_ipv6(ip)
            if b.startswith(ipv4addrmask):
                self.ipv4list.append(b[96:][:depth-96])
            else:
                self.ipv6list.append(b[:depth])

    def append(self, ip, depth = 256):
        if ip.find(':') < 0:        # IPv4
            insort(self.ipv4list,to_bitfield_ipv4(ip)[:depth])
        else:
            b = to_bitfield_ipv6(ip)
            if b.startswith(ipv4addrmask):
                insort(self.ipv4list,b[96:][:depth-96])
            else:
                insort(self.ipv6list,b[:depth])


    def includes(self, ip):
        if not (self.ipv4list or self.ipv6list):
            return False
        if ip.find(':') < 0:        # IPv4
            b = to_bitfield_ipv4(ip)
        else:
            b = to_bitfield_ipv6(ip)
            if b.startswith(ipv4addrmask):
                b = b[96:]
        if len(b) > 32:
            l = self.ipv6list
        else:
            l = self.ipv4list
        for map in l[bisect(l,b)-1:]:
            if b.startswith(map):
                return True
            if map > b:
                return False
        return False


    def read_fieldlist(self, file):   # reads a list from a file in the format 'ip/len <whatever>'
        f = open(file, 'r')
        while True:
            line = f.readline()
            if not line:
                break
            line = line.strip().expandtabs()
            if not line or line[0] == '#':
                continue
            try:
                line, garbage = line.split(' ',1)
            except:
                pass
            try:
                line, garbage = line.split('#',1)
            except:
                pass
            try:
                ip, depth = line.split('/')
            except:
                ip = line
                depth = None
            try:
                if depth is not None:                
                    depth = int(depth)
                self._append(ip,depth)
            except:
                print '*** WARNING *** could not parse IP range: '+line
        f.close()
        self.ipv4list.sort()
        self.ipv6list.sort()


    def set_intranet_addresses(self):
        self.append('127.0.0.1',8)
        self.append('10.0.0.0',8)
        self.append('172.16.0.0',12)
        self.append('192.168.0.0',16)
        self.append('169.254.0.0',16)
        self.append('::1')
        self.append('fe80::',16)
        self.append('fec0::',16)

    def set_ipv4_addresses(self):
        self.append('::ffff:0:0',96)

def ipv6_to_ipv4(ip):
    ip = to_bitfield_ipv6(ip)
    if not ip.startswith(ipv4addrmask):
        raise ValueError, "not convertible to IPv4"
    ip = ip[-32:]
    x = ''
    for i in range(4):
        x += str(int(ip[:8],2))
        if i < 3:
            x += '.'
        ip = ip[8:]
    return x

def to_ipv4(ip):
    if is_ipv4(ip):
        _valid_ipv4(ip)
        return ip
    return ipv6_to_ipv4(ip)

def is_ipv4(ip):
    return ip.find(':') < 0

def _valid_ipv4(ip):
    ip = ip.split('.')
    if len(ip) != 4:
        raise ValueError
    for i in ip:
        chr(int(i))

def is_valid_ip(ip):
    try:
        if not ip:
            return False
        if is_ipv4(ip):
            _valid_ipv4(ip)
            return True
        to_bitfield_ipv6(ip)
        return True
    except:
        return False