Binding to Two Local IPs Fails

Question

I have two wireless adapters, one USB adapter and one built in to my laptop.

I'd like to be able to use these two connections. So, in a toy example I bind two different sockets to the two different IP addresses and port numbers and call connect on each one.

However, when I examine my network traffic in wireshark...I only see traffic from one ip!? In fact, I see both calls to connect from one IP address despite the fact that I explicitly bind each socket.

Here is the code I'm using:

Note, I'm also using non-blocking sockets and select. The code I have for this has been verified to work for one internet connection.

#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/socket.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/time.h>
#include <net/if.h>
#include <sys/ioctl.h>


int main () {

    const char * destIp = "213.112.225.102";

    const char * ip1 = "192.168.43.1";//"172.31.55.111";//"198.228.228.28";
    int portNumber1 = 55555;
    int sockFd1 = -1;

    const char * ip2 = "192.168.1.1";//"98.249.5.16";
    int portNumber2 = 7777;
    int sockFd2 = -1;

    struct sockaddr_in serverAddress;
    serverAddress.sin_addr.s_addr = inet_pton(AF_INET, "213.112.225.102", &(serverAddress.sin_addr));
    serverAddress.sin_port = htons(6985);

    ///////////////////////////////////////////
    struct sockaddr * saddr;
    struct addrinfo hints, * ai,  * it;
    char strportnum[] = "6985";
    memset(&hints, '\0', sizeof(hints));
    hints.ai_flags = AI_ADDRCONFIG;
    hints.ai_socktype = SOCK_STREAM;

    getaddrinfo(destIp, strportnum, &hints, &ai);

    saddr = ai->ai_addr;
    saddr->sa_family = AF_INET;

    it = ai;
    ///////////////////////////////////////////////
    //char * opt;
    int res; 
    long arg; 
    fd_set myset; 
    struct timeval tv; 
    int valopt; 
    socklen_t lon; 
    struct sockaddr_in clientAddress;
    struct sockaddr_in clientAddress2;

    printf("it fam == ||%d||, AF_INET == ||%d||\n", it->ai_family, AF_INET);

    printf("ATTEMPTING SOCKET 1!\n");
    //IP 1 CONNECTION----------------------------------------------------------------------------------//
    if ((sockFd1 = socket(it->ai_family, it->ai_socktype, it->ai_protocol)) != -1) {


        system("route add -net 213.112.225.102 netmask 255.255.255.255 gw 192.168.43.1 dev wlp10s0");

        struct ifreq interface1; 
        memset(&interface1, 0, sizeof(interface1));
        strncpy(interface1.ifr_ifrn.ifrn_name, "wlp10s0", IFNAMSIZ);

        if (setsockopt(sockFd1, SOL_SOCKET, SO_BINDTODEVICE, &interface1, sizeof(interface1)) < 0) { 
            printf("error in set sock opt 1... errno == %d strerror == (%s)\n", errno, strerror(errno));

            close(sockFd1); // Error 
            return 1;
        }


        clientAddress.sin_family = AF_INET;
        clientAddress.sin_addr.s_addr = inet_pton(AF_INET, ip1, &(clientAddress.sin_addr));
        clientAddress.sin_port = htons(portNumber1);
        if (bind(sockFd1, (struct sockaddr *) &clientAddress, sizeof(clientAddress)) < 0) {
            fprintf(stderr, "Error with bind, errno == %d (%s)\n", errno, strerror(errno)); 
        }



        // Set non-blocking 
        if( (arg = fcntl(sockFd1, F_GETFL, NULL)) < 0) { 
            fprintf(stderr, "Error fcntl(..., F_GETFL) (%s)\n", strerror(errno)); 
            return 1;
        }
        arg |= O_NONBLOCK; 
        if( fcntl(sockFd1, F_SETFL, arg) < 0) { 
            fprintf(stderr, "Error fcntl(..., F_SETFL) (%s)\n", strerror(errno)); 
            return 1;
        } 

        printf("ATTEMPTING CONNECTION 2!\n");
        // Trying to connect with timeout 
        res = connect(sockFd1, saddr, sizeof(*saddr)); 
        if (res < 0) { 

            if (errno == EINPROGRESS) { 

                fprintf(stderr, "EINPROGRESS in connect() - selecting\n"); 

                do { 

                    //Set timeouts
                    tv.tv_sec = 15; 
                    tv.tv_usec = 0; 

                    FD_ZERO(&myset); 
                    FD_SET(sockFd1, &myset); 

                    res = select(sockFd1 + 1, NULL, &myset, NULL, &tv); 

                    if (res < 0 && errno != EINTR) { 
                        fprintf(stderr, "Error connecting %d - %s\n", errno, strerror(errno)); 
                    } 
                    else if (res > 0) { 

                        // Socket selected for write 
                        lon = sizeof(int); 
                        if (getsockopt(sockFd1, SOL_SOCKET, SO_ERROR, (void *) &valopt, &lon) < 0) { 
                            fprintf(stderr, "Error in getsockopt() %d - %s\n", errno, strerror(errno)); 
                        } 

                        // Check the value returned... 
                        if (valopt) { 
                            fprintf(stderr, "Error in delayed connection() %d - %s\n", valopt, strerror(valopt)); 
                        } 

                        break;
                    } 
                    else { 
                        fprintf(stderr, "Timeout in select() - Cancelling!\n"); 
                        break;
                    } 
                } while (1); 
            } 
            else { 
                fprintf(stderr, "Error connecting %d - %s\n", errno, strerror(errno)); 
            } 
        } 
    }



    printf("ATTEMPTING SOCKET 2!\n");
    //IP 2 CONNECTION----------------------------------------------------------------------------------//
    if ((sockFd2 = socket(it->ai_family, it->ai_socktype, it->ai_protocol)) != -1) {

        system("route add -net 213.112.225.102 netmask 255.255.255.255 gw 192.168.1.1 dev wlp11s0u1");

        struct ifreq interface2; 
        memset(&interface2, 0, sizeof(interface2));
        strncpy(interface2.ifr_ifrn.ifrn_name, "wlp11s0u1", IFNAMSIZ);

        if (setsockopt(sockFd2, SOL_SOCKET, SO_BINDTODEVICE, &interface2, sizeof(interface2)) < 0) { 
            printf("error in set sock opt 2... errno == %d strerror == (%s)\n", errno, strerror(errno));
            close(sockFd2); // Error 
            return 1;
        }


        clientAddress2.sin_family = AF_INET;
        clientAddress2.sin_addr.s_addr = inet_pton(AF_INET, ip2, &(clientAddress.sin_addr));
        clientAddress2.sin_port = htons(portNumber2);
        if (bind(sockFd2, (struct sockaddr *) &clientAddress2, sizeof(clientAddress2)) < 0) {
            fprintf(stderr, "Error with bind (%s)\n", strerror(errno)); 
        }



        // Set non-blocking 
        if( (arg = fcntl(sockFd2, F_GETFL, NULL)) < 0) { 
            fprintf(stderr, "Error fcntl(..., F_GETFL) (%s)\n", strerror(errno)); 
            return 1;
        }
        arg |= O_NONBLOCK; 
        if( fcntl(sockFd2, F_SETFL, arg) < 0) { 
            fprintf(stderr, "Error fcntl(..., F_SETFL) (%s)\n", strerror(errno)); 
            return 1;
        } 

        printf("ATTEMPTING CONNECTION 2!\n");
        // Trying to connect with timeout 
        res = connect(sockFd2, saddr, sizeof(*saddr)); 
        if (res < 0) { 

            if (errno == EINPROGRESS) { 

                fprintf(stderr, "EINPROGRESS in connect() - selecting\n"); 

                do { 

                    //Set timeouts
                    tv.tv_sec = 15; 
                    tv.tv_usec = 0; 

                    FD_ZERO(&myset); 
                    FD_SET(sockFd2, &myset); 

                    res = select(sockFd2 + 1, NULL, &myset, NULL, &tv); 

                    if (res < 0 && errno != EINTR) { 
                        fprintf(stderr, "Error connecting %d - %s\n", errno, strerror(errno)); 
                    } 
                    else if (res > 0) { 

                        // Socket selected for write 
                        lon = sizeof(int); 
                        if (getsockopt(sockFd2, SOL_SOCKET, SO_ERROR, (void *) &valopt, &lon) < 0) { 
                            fprintf(stderr, "Error in getsockopt() %d - %s\n", errno, strerror(errno)); 
                        } 

                        // Check the value returned... 
                        if (valopt) { 
                            fprintf(stderr, "Error in delayed connection() %d - %s\n", valopt, strerror(valopt)); 
                        } 

                        break;
                    } 
                    else { 
                        fprintf(stderr, "Timeout in select() - Cancelling!\n"); 
                        break;
                    } 
                } while (1); 
            } 
            else { 
                fprintf(stderr, "Error connecting %d - %s\n", errno, strerror(errno)); 
            } 
        } 
    }

    return 0;
}

/*
ifreq interface; 
memset(&interface, 0, sizeof(interface));
strncpy(interface.ifr_ifrn.ifrn_name, "eth1", IFNAMSIZ);

if (setsockopt(sd, SOL_SOCKET, SO_BINDTODEVICE, &interface, sizeof(interface)) < 0) { 
    close(sd); // Error 
}
*/

So what gives, why is bind not binding!?

EDIT:

Okay, so thanks to this old post: Multiple Ethernet Interfaces - How to create a separate network and access from C code

I'm taking a different approach now, but I still am not utilizing both networks...

To see a response from address X, you need to send a request to address X. If you only send requests to address Y, where should X come from? — n. 'pronouns' m., Dec 17 '13 at 07:51
I'm not certain what you're saying. I have addresses x1 and x2 and I call connect on both. That sends a tcp packet from x1 to y and x2 to y. But instead they're both coming from x1.... — Ethan, Dec 17 '13 at 07:54
Sorry, misread your question. You have a client, not a server. In this case this is just a question of routing. You probably have one default route through one of your interfaces. Packets destined to a non-local network will only go to that interface. To have it otherwise, you need to be running a routing service. Those normally don't run on laptops but on big boxes made by companies like Cisco. — n. 'pronouns' m., Dec 17 '13 at 08:06
Oh. So there's no way to (simply) programmatically route traffic to each of my two interfaces? — Ethan, Dec 17 '13 at 08:07
It is not impossible to run a routing service on a laptop, but this is offtipic for this site. Try serverfault. Your program is not at fault here. — n. 'pronouns' m., Dec 17 '13 at 08:09
Controlling routing programmatically is very OS-dependent (no guarantee it's pissible for your OS at all) and you will need admin rights anyway. Certainly not easy. — n. 'pronouns' m., Dec 17 '13 at 08:16
This post seems to describe something similar, and it sounds like they got it working... http://stackoverflow.com/questions/18369848/multiple-ethernet-interfaces-how-to-create-a-separate-network-and-access-from — Ethan, Dec 17 '13 at 08:22
I *think* your packets with source address X go out from the interface with address Y, and get lost in the network connected to Y. SO_BINDTODEVICE can help, but it works only if you are root, and only on Linux. — n. 'pronouns' m., Dec 17 '13 at 08:34
Luckily I'm running Linux and this doesn't need to be portable. I updated the code above, but when I run it now I get communication on the main IP (as expected) but then I also see several ICMP packets going from ip2 to ip2....Why would it try to transmit to itself? — Ethan, Dec 17 '13 at 08:39
Note that if you are running Linux, you can do it with iptables, then your program doesn't have to be root (but you need root to configure iptables of course). Google `iptables source based routing` or `iptables policy routing`. — n. 'pronouns' m., Dec 17 '13 at 09:10
So, for clarification. If I do static routing, do I just need to call bind to each distinct ip address and thats it? Or is there more that must be done? — Ethan, Dec 18 '13 at 03:10

rodolk · Accepted Answer · 2013-12-20T12:17:05.907

1

I haven't tried it but just an idea. Without having looked at kernel code, maybe you can cheat the kernel but I'm not sure whether it'll work. With the risk of saying something awfully wrong.

Before connecting the first time, through "wlp11s0u1", you set the route to "213.112.225.102" through that interface. route add -host 213.112.225.102 gw 192.168.0.1 dev wlp11s0u1 (or the GW IP)

Then, before connecting the second time, through "wlp10s0" you set the route: route add -host 213.112.225.102 gw 192.168.2.1 dev wlp10s0 (or the corresponding gateway)

You can use system() function to run "route" command.

If the kernel does not evaluate the route after a connection was established, it might work. If the kernel evaluates the route for every datagram sent through a connected socket, it won't work. You could try as an easy solution.

EDIT: I tried also using Loose source route option, and it did not work for me for some reason, maybe my router does not allow that IP option. I can pass you the code if you are interested.

Other possibility that will certainly work is that you use packet sockets but you will have to add the IP and TCP headers, as well as manage ARP, IP and TCP algorithms. That is fun.

edited Dec 20 '13 at 12:17

answered Dec 18 '13 at 03:34

rodolk

4,969
3
22
32

Why would I set the route to 213.112.225? That is the IP address that I am connecting to. – Ethan Dec 18 '13 at 03:37
You are telling the kernel that for datagrams destined to that IP, it must send the datagram through the said interface. – rodolk Dec 18 '13 at 13:49
1

You are telling the kernel for datagrams destined to IP 213.112.225.102, it must send the datagram through the said interface. You are trying to connect to IP address 213.112.225.102, right? I'm assuming you have a gateway (gw) (you must set your gateway's IP address, replacing the one I wrote). Again, I'm not sure if it will work but it's easy to try. – rodolk Dec 18 '13 at 13:57
1

BTW, I suppose you are trying to connect to the same IP address from two different network interfaces for increasing the bandwidth used to download/upload files/data, or something similar? – rodolk Dec 18 '13 at 13:59
Yes, your guess is mostly correct. I am also using two different networks. I'll give your suggestion a shot! – Ethan Dec 18 '13 at 16:21
1

Great! It sounds very interesting. I'll try to look into the kernel in the following two days to see if there is any other alternative. – rodolk Dec 18 '13 at 17:22
1

I was looking at the kernel. At first glimpse it seems this method could work but I still need to check better. – rodolk Dec 18 '13 at 19:47
So, that almost works....I'm able to get out a SYN and receive an ACK on the network that isn't set as the default, but then I get a bunch of retransmissions and nothing. However the network that is set as default can connect just fine. Seems like that comes close though! – Ethan Dec 18 '13 at 23:35
Also, I had to change the route commands slightly because they were not accepted as valid. They ended up being: "route add -net 213.112.225.102 netmask 255.255.255.255 gw 192.168.43.1 dev wlp10s0" and "route add -net 213.112.225.102 netmask 255.255.255.255 gw 192.168.1.1 dev wlp11s0u1" – Ethan Dec 18 '13 at 23:36
Got it to work for my toy example above, I had to add corresponding "route del" statements at the end of the code so that it would run consistently (and not break my internet)! However, in my larger application (a bittorrent client) the connection goes to only using 1 route again after a little while....Any idea what could be the cause of this? – Ethan Dec 19 '13 at 01:33
It seems as if using this configuration I can only send from a specific IP...now I need to figure out a way to receive on a specific IP as well – Ethan Dec 19 '13 at 02:09
Definitely, the solution goes in that direction. I'm still looking at the kernel, maybe there is some configuration that let you achieve the goal. Just as an easy alternative, do you have the possibility of assigning to different IP addresses to the server? Is it your server? – rodolk Dec 19 '13 at 14:51
I didn't find any other way to do it. Please note the route command is not fully correct, it should be: route add -host 213.112.225.102 gw X.X.X.X dev nnnnn". What happens if you use it before sending each packet and set the corresponding route before sending with each socket? – rodolk Dec 20 '13 at 04:48
I edited my response with additional research I carried out. But if you test the alternative I suggested in my previous comment, it might work. – rodolk Dec 20 '13 at 05:01

Binding to Two Local IPs Fails

1 Answers1