Port driver communication witout copy of binaries

[Reto Kramer]

Raimo,

I wish I could replicate your outputv result. Perhaps you could help  
me spot my mistake. The example I've added further down (iovec.erl,  
iovec_driver.c and build_iovec) is run on OS X 10.4.7 using R11. gcc  
is 3.3. The port is opened as binary as it should.

$ ./build_iovec
Erlang (BEAM) emulator version 5.5 [source] [async-threads:32] [hipe]

Eshell V5.5  (abort with ^G)
1>ev->vsize 2
ev-size (total bytes size) 8
ev->binv[0] = null
ev->iov[0]->iov_len = 0
ev->binv[1]->orig_size = 8
ev->iov[1]->iov_len = 8
ev->iov[1]->iov_base[0] = 1
ev->iov[1]->iov_base[1] = 2
ev->iov[1]->iov_base[2] = 3
ev->iov[1]->iov_base[3] = 4
ev->iov[1]->iov_base[4] = 5
ev->iov[1]->iov_base[5] = 6
ev->iov[1]->iov_base[6] = 7
ev->iov[1]->iov_base[7] = 8

^C

In iovec.erl, the following iolist is sent:
   encode() ->
       [1,<<2,3,4>>,5,6|<<7,8>>].

It's really as if I used "output" (instead of "outputv") in that the  
iolist seems to be flattened before handed to the driver (and hence,  
copied, which I wanted to avoid by way of using outputv).

The 2nd puzzle is that the first iov entry is always length 0. This  
is so, even if I pass (encode) just a single binary.

Any sharp eyed comment on iovec_driver.c would be much appreciated.

- Reto
PS: the async aspect is irrelevant here, I just left it in there  
since this is a trimmed version of a larger driver that is async.

------------------------------------------------------------------------ 
-----
** iovec_driver.c **

#include <stdio.h>
#include "erl_driver.h"

typedef struct {
   ErlDrvPort port;
} example_data;

typedef struct {
   int res;
} example_async_data;

typedef void ASYNC_INVOKE(void*);

void async_iovectest(void* async_data) {
   example_async_data* the_data = (example_async_data*)async_data;
}

static ErlDrvData example_drv_start(ErlDrvPort port, char *buff) {
   example_data* d = (example_data*)driver_alloc(sizeof(example_data));
   d->port = port;
   return (ErlDrvData)d;
}

static void example_drv_stop(ErlDrvData handle) {
   driver_free((char*)handle);
}

static void example_drv_outputv(ErlDrvData handle, ErlIOVec *ev) {
   example_data* d = (example_data*)handle;
   example_async_data* the_async_data = driver_alloc(sizeof 
(example_async_data));
   ASYNC_INVOKE* fun;
   printf("\n\r");
   printf("ev->vsize %i\n\r", ev->vsize);
   printf("ev-size (total bytes size) %i\n\r", ev->size);
   int i;
   for (i = 0; i < (ev->vsize); i++) {
     SysIOVec b = ev->iov[i];
     ErlDrvBinary* binv = ev->binv[i];
     if (binv == NULL) {
       printf("ev->binv[%i] = null\n\r", i);
     } else {
       printf("ev->binv[%i]->orig_size = %ld\n\r", i, binv->orig_size);
     }
     printf("ev->iov[%i]->iov_len = %i\n\r", i, b.iov_len);
     int j;
     for (j=0; j < b.iov_len; j++) {
       printf("ev->iov[%i]->iov_base[%i] = %i\n\r", i, j, b.iov_base 
[j]);
     }
   }
   fun = &async_iovectest;
}

static void example_ready_async(ErlDrvData handle, ErlDrvThreadData  
async_data)
{
   example_data* d = (example_data*)handle;
   example_async_data* the_async_data = (example_async_data*)async_data;
   ErlDrvTermData spec[] = {
     ERL_DRV_PORT, driver_mk_port(d->port),
     ERL_DRV_ATOM, driver_mk_atom("ok"),
     ERL_DRV_TUPLE, 2 };
   driver_output_term(d->port, spec, sizeof(spec) / sizeof(spec[0]));
   driver_free(the_async_data);
}

ErlDrvEntry example_driver_entry = {
   NULL, /* F_PTR init, N/A */
   example_drv_start, /* L_PTR start, called when port is opened */
   example_drv_stop, /* F_PTR stop, called when port is closed */
   NULL, /* F_PTR output, called when erlang has sent */
   NULL, /* F_PTR ready_input, called when input descriptor ready */
   NULL, /* F_PTR ready_output, called when output descriptor ready */
   "iovec_drv", /* char *driver_name, the argument to open_port */
   NULL,                       /* finish */
   NULL,                       /* handle */
   NULL,                       /* control */
   NULL,                       /* timeout */
   example_drv_outputv,        /* outputv */
   example_ready_async,
   NULL,                       /* flush */
   NULL,                       /* call */
   NULL                        /* event */
};
DRIVER_INIT(example_drv) /* must match name in driver_entry */ {
   return &example_driver_entry;
}

------------------------------------------------------------------------ 
-----
** iovec.erl **
-module(iovec).

-export([start/1, init/1, test/0]).

-define(DRIVER, iovec_drv).

test() ->
     ok = start(?DRIVER),
     timer:sleep(100), % HACK: give process some time to start and  
load .so
     Port = init(?DRIVER),
     test(Port).

start(SharedLib) ->
     case erl_ddll:load_driver(".", SharedLib) of
	ok -> ok;
	{error, already_loaded} -> ok;
	Error -> exit({error, could_not_load_driver, Error})
     end.

init(SharedLib) ->
     Port = open_port({spawn, SharedLib}, [binary]),
     Port.

test(Port) ->
     true = port_command(Port, encode()),
     receive
	{Port, ok} ->
	    io:format("ok!~n", []);
	{'EXIT', Port, Reason} ->
	    io:format("~p ~n", [Reason]),
	    exit(port_terminated)
     end.

encode() ->
     [1,<<2,3,4>>,5,6|<<7,8>>].

------------------------------------------------------------------------ 
-----
** build_iovec **
cc -no-cpp-precomp -fPIC -fno-common -bundle -flat_namespace - 
undefined suppress -I/usr/local/lib/erlang/usr/include -o  
iovec_drv.so complex.c iovec_driver.c && erlc +debug_info iovec.erl  
&& erl +A 32 -s iovec test

Thanks,
- Reto


On Mar 28, 2006, at 3:42 AM, Raimo Niskanen wrote:

> What you really want to do can not be done (as far as I know)
> but you might get it done with some tricks...
>
> To avoid copying your driver must implement the
> ->outputv() entry point and you must send it I/O lists
> being lists of binaries (might even be an improper list,
> that is a binary in the tail). You will have to map
> your tuples into that.
>
> If you send [1,<<2,3,4>>,5,6|<<7,8>>] to the driver,
> void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev) will get:
>
> ev->iov[0].iov_len = 1;
> ev->iov[0].iov_base -> {1};
> ev->binv[0] = NULL;
> ev->iov[1].iov_len = 3;
> ev->iov[1].iov_base -> ev->binv[1]->orig_bytes;
> ev->binv[1]->orig_size = 3;
> ev->binv[1]->orig_bytes = {2,3,4};
> ev->iov[2].iov_len = 2;
> ev->iov[2].iov_base -> {5,6};
> ev->binv[2] = NULL;
> ev->iov[3].iov_len = 2;
> ev->iov[3].iov_base -> ev->binv[3]->orig_bytes;
> ev->binv[3]->orig_size = 2;
> ev->binv[3]->orig_bytes = {7,8};
>
> approximately, excuse my syntax :-)
>
> Binaries will be binaries and intermediate bytes
> will be loose vectors. If your driver wants to
> hang on to the data, it will have to use the
> reference count in the binary to avoid premature freeing.
>
> To send data back without copying your driver will
> have to use driver_outputv() and it arrives to erlang as
> a header list of integers followed by a list of
> binaries. Conversion to tuple format will have to
> be done in erlang.
>
> Keep on dreaming...
>
>
>
> Have a look at efile_drv.c in the sources...
>
>
>
> rlenglet@REDACTED (Romain Lenglet) writes:
>
>> Hi,
>>
>>
>> I have the following need: I want to wrap C functions in Erlang.
>> Those functions get big binaries as input parameters, and return
>> big binaries, among other kinds of data.
>> For efficiency, I would like to avoid to copy those binaries
>> around when communicating. Therefore, I am forced to implement a
>> C port driver, since this is the only available mechanism that
>> does not create a separate system process (and hence does not
>> require inter-process data copy when communicating).
>>
>> If I needed only to send one binary in every message, that would
>> be OK, e.g.:
>>
>> % in Erlang:
>> Binary = <<...>>,
>> port_command(Port, Binary),
>>
>> // in the C port implem:
>> void myoutput(ErlDrvData drv_data, char *buf, int len) {
>> ...
>> }
>>
>> I guess that the Binary is not copied, and its data in the Erlang
>> heap is directly pointed by the *buf argument.
>> By the way, is that true??? Sending binaries that way is what is
>> done in prim_inet for sending IP data, so I guess that no copy
>> is done here.
>>
>>
>> However, I want to send and receive more complex data, which must
>> be manipulated by the driver, typically a tuple of simple terms
>> and binaries (which may be large), e.g. the tuple:
>> Tuple = {ContextHandle, QopReq, Message}
>> %%  ContextHandle = small binary()
>> %%  QopReq = integer() | atom()
>> %%  Message = large binary()
>>
>> Such a tuple cannot be passed to port_command(Port, Tuple), since
>> it is not an IO list. And if I encode it into a binary, by
>> calling encode(Tuple), I guess that the binaries in the tuple
>> will get copied in the process (can anybody confirm this?).
>>
>>
>> I have the same problem in the Driver -> Erlang direction, e.g.
>> to send the tuple:
>> Tuple = {MajorStatus, MinorStatus, ConfState, QopState,
>> OutputMessage}
>> %%      MajorStatus = integer()
>> %%      MinorStatus = integer()
>> %%      ConfState = bool()
>> %%      QopState = integer()
>> %%      OutputMessage = large binary()
>> I hope that using the driver_output_term() C function and the
>> ErlDrvTermData construction technique, the binary data in the
>> tuple above will not be copied. Can anybody confirm this?
>>
>>
>>
>> Is there any clean solution to my problem? Or am I doomed to
>> write my own BIFs and use my custom erts? Or to send data in
>> multiple messages, in sequence?
>>
>> I dream of a way to extend the BIFs list at runtime, by loading
>> native libraries dynamically...
>>
>> -- 
>> Romain LENGLET
>
> -- 
>
> / Raimo Niskanen, Erlang/OTP, Ericsson AB