标题：TCP Port Multiplexing
出处：Felix021
时间：Fri, 24 Feb 2012 17:00:48 +0000
作者：felix021
地址：https://www.felix021.com/blog/read.php?2066

内容：
　　很久很久以前就想过这个问题，是不是可以写这样一个frontend proxy，通过client的请求来判断需要连接的端口号，实现http/ssh共用80端口呢？

　　不久后我发现，通过apache的mod_proxy+mod_proxy_connect可以使用HTTP的CONNECT HOST:PORT命令来代理22端口，也就是tunneling ssh over http, google可以发现很多都用到了proxytunnel，当作openssh client的代理；securecrt内建支持，使用Option->Global里面的firewall，其实就是proxy。

　　于是这个想法被搁置了，好几年。为什么又想起了呢…………是因为，proxy_connect模块貌似不能限制被代理的HOST。也就是说，通过这台服务器可以PROXY到任意机器这样是有风险的。

　　于是昨天花了半个下午+半个晚上看libevent的文档。其实本来是打算找个速成教程的，但是好像没有，所以还是老老实实看文档。当天晚上写出了第一个版本的multiplexer，各种BUG。无奈，第二天干脆推倒重来，写了第二个版本，还是各种BUG。各种蛋疼之后，终于实现了一个基本可用的multiplexer，250行左右，挺短的代码。

　　遇到的主要障碍包括：

1. bufferevent *bev的read_cb函数被调用以后，应该用一个while循环把bev里面所有的数据都读出来（比如使用bufferevent_read）；

2. set_nonblocking的操作应该在bind/listen/connect等操作之后完成；否则这些操作也是nonblocking了……

3. 最好加上 |EV_PERSIST ，免得每次callback的时候都要再enable或者add一次；

4. ssh协议对client/server哪个先说话貌似没要求，比如openssh是等server先说话，而securecrt是自己先说话，所以这个multiplexer的实现很蛋疼，需要设置一个timeout，如果client过了一会儿没说话，就forward给sshd。

5. 对于4，更蛋疼的是，（我用的libevent2.0.17-stable）bufferevent_set_timeouts(bev, NULL, NULL)貌似不能清除timeout（或许是个BUG?），所以使用了一个struct timeval tv = {86400*1024, 0} 来绕过timeout的问题。

大概就这些，代码如下
//tested on ubuntu 10.04 AMD64, with libevent-2.0.17-stable.
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <assert.h>

/* For sockaddr_in */
#include <netinet/in.h>
/* For socket functions */
#include <sys/socket.h>
/* For fcntl */
#include <fcntl.h>

#include <event2/event.h>
#include <event2/buffer.h>
#include <event2/bufferevent.h>

#define ALLOC(type, n) ((type *)malloc(sizeof(type) * n))
#define READ_ONCE   512
#define LISTEN_BACKLOG  4096

#define LISTEN_PORT 9999
#define HTTP_PORT   80
#define SSH_PORT    22
    
struct sockctx {
    evutil_socket_t self;
    struct sockctx *partner;
    struct event_base *base;
    struct bufferevent *bev;
};
typedef struct sockctx sockctx;

sockctx *sockctx_new(evutil_socket_t fd, struct event_base *base, struct bufferevent *bev)
{
    sockctx *sc = ALLOC(sockctx, 1);
    if (sc == NULL)
        return NULL;
    sc->self = fd;
    sc->partner = NULL;
    sc->base = base;
    sc->bev = bev;
    return sc;
}

void sockctx_free(struct sockctx *sc)
{
    if (sc == NULL) return;
    if (sc->partner != NULL) {
        printf("close fd[%u]\n", sc->partner->self);
        bufferevent_free(sc->partner->bev);
        free(sc->partner);
    }
    printf("close fd[%u]\n", sc->self);
    bufferevent_free(sc->bev);
    free(sc);
}

void do_accept(evutil_socket_t listener, short event, void *arg);
void read_cb(struct bufferevent *bev, void *arg);
void error_cb(struct bufferevent *bev, short error, void *arg);
int create_partner(struct sockctx *sc, unsigned port);
unsigned guess_port(const char *line, int n);

int main(int argc, char *argv[])
{
    struct event_base *base;
    base = event_base_new();
    assert(base != NULL);

    struct sockaddr_in sin;

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = 0;
    sin.sin_port = htons(LISTEN_PORT);

    evutil_socket_t listener;
    listener = socket(AF_INET, SOCK_STREAM, 0);
    assert(listener > 0);
    evutil_make_listen_socket_reuseable(listener);

    if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
        perror("bind");
        return 1;
    }
    if (listen(listener, LISTEN_BACKLOG) < 0) {
        perror("listen");
        return 1;
    }
    evutil_make_socket_nonblocking(listener);

    printf("listen ok!\n");
    struct event *listen_event;
    listen_event = event_new(base, listener, EV_READ|EV_PERSIST, do_accept, (void *)base);
    event_add(listen_event, NULL);
    event_base_dispatch(base);
    return 0;
}

void do_accept(evutil_socket_t listener, short event, void *arg)
{
    struct event_base *base = (struct event_base *)arg;
    struct sockaddr_in sin;
    socklen_t slen = sizeof(sin);

    evutil_socket_t fd;

    fd = accept(listener, (struct sockaddr *)&sin, &slen);
    if (fd < 0) {
        perror("accept");
        return;
    }
    if (fd > FD_SETSIZE) {
        perror("fd > FD_SETSIZE");
        return;
    }

    printf("Accept: fd = %u\n", fd);
    evutil_make_socket_nonblocking(fd);

    struct bufferevent *bev;
    bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE);

    struct sockctx *sc = sockctx_new(fd, base, bev);
    if (sc == NULL) {
        bufferevent_free(bev);
        perror("alloc sc failed");
        return;
    }

    //*
    struct timeval tv = {1, 0};
    bufferevent_set_timeouts(bev, &tv, NULL);
    //*/

    bufferevent_setcb(bev, read_cb, NULL, error_cb, sc);
    bufferevent_enable(bev, EV_READ|EV_WRITE|EV_PERSIST);
}

unsigned guess_port(const char *line, int n)
{
    if (n >= 3 && strncmp(line, "SSH", 3) == 0) {
        return SSH_PORT;
    }
    else {
        return HTTP_PORT;
    }
}

void read_cb(struct bufferevent *bev, void *arg)
{
    struct sockctx *sc = (struct sockctx *)arg;
    char line[READ_ONCE + 1];
    int n;
    while (n = bufferevent_read(bev, line, READ_ONCE), n > 0) {
        line[n] = '\0';
        //printf("read_cb: fd = %u, n = %d, line = %s\n", sc->self, n, line);

        if (sc->partner == NULL) {
            unsigned port = guess_port(line, n);
            if (create_partner(sc, port) < 0) {
                perror("create partner failed");
                sockctx_free(sc);
                return;
            }
        }

        //printf("forwarding to fd[%u]\n", sc->partner->self);
        bufferevent_write(sc->partner->bev, line, n);
    }
}

void error_cb(struct bufferevent *bev, short error, void *arg)
{
    struct sockctx *sc = (struct sockctx *)arg;
    
    if (error & BEV_EVENT_TIMEOUT) {
        if (sc->partner == NULL) {
            printf("[fd = %u] timed out, try ssh\n", sc->self);
            if (create_partner(sc, SSH_PORT) < 0) {
                perror("create partner failed");
                sockctx_free(sc);
            }
            bufferevent_setcb(bev, read_cb, NULL, error_cb, sc);
            bufferevent_enable(bev, EV_READ|EV_WRITE|EV_PERSIST);
        }
        else {
            bufferevent_setcb(bev, read_cb, NULL, error_cb, sc);
            bufferevent_enable(bev, EV_READ|EV_WRITE|EV_PERSIST);
        }
        //workaround for not able to delete timeout!
        struct timeval tv = {86400 * 1024, 0}; 
        bufferevent_set_timeouts(bev, &tv, NULL);
        return;
    }
    else if (error & BEV_EVENT_EOF) {
        printf("[fd=%u] connection closed\n", sc->self);
    }
    else if (error & BEV_EVENT_ERROR) {
        printf("[fd=%u] unknown error\n", sc->self);
    }
    sockctx_free(sc);
}

int create_partner(struct sockctx *sc, unsigned port)
{
    evutil_socket_t fd;
    fd = socket(AF_INET, SOCK_STREAM, 0);
    if (fd < 0) {
        perror("socket");
        return -1;
    }

    struct sockaddr_in sin;
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = htonl(0x7f000001); //127.0.0.1;
    sin.sin_port = htons(port);

    if (connect(fd, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
        perror("connect");
        return -1;
    }

    printf("partner fd: %u\n", fd);

    evutil_make_socket_nonblocking(fd);

    struct bufferevent *bev;
    bev = bufferevent_socket_new(sc->base, fd, BEV_OPT_CLOSE_ON_FREE);

    sc->partner = sockctx_new(fd, sc->base, bev);
    if (sc->partner == NULL) {
        bufferevent_free(bev);
        perror("sockctx_new");
        return -1;
    }
    sc->partner->partner = sc;

    bufferevent_setcb(bev, read_cb, NULL, error_cb, sc->partner);
    bufferevent_enable(bev, EV_READ|EV_WRITE|EV_PERSIST);
    return 0;
}



Generated by Bo-blog 2.1.0