l'essentiel est invisible pour les yeux

Monday, June 30, 2008

Tutorial for how to create Erlang linked-in driver.

Erlang provides mechanism to communicate between Erlang and other programs written by C/C++ or other. This entry introduce how to create simple linked-in driver. Create shared library and Erlang interface to use Erlang's linked-in driver.

Goal: To create HAVAL bindings for Erlang.
HAVAL is fast cryptographic hash algorithm is invented by Yuliang Zheng, Josef Pieprzyk, and Jennifer Seberry in 1992. This library is pretty simple, so this is good choice for this tutorial.

Step 0: Files


config.h (Generated by configure script in HAVAL package)
haval.h
haval.c
haval.erl
haval_drv.cc
Makefile


haval.c and haval.h are included haval-1.1.tar.gz (C source code).

You can checkout these sample sources with Git. see haval_erlang.

% git clone git://github.com/rakuto/erlang_haval.git erlang_haval


Step 1: Create Erlang's interface
Erlang provides port mechanism in order to communicate between Erlang and other loaded shared object. See also Writing an Erlang Port using OTP Principles


-module(haval).
-author('Rakuto Furutani <xri://=rakuto>').

-define(DRV_INFO, 1).
-define(DRV_HAVAL_STRING, 2).
-define(DRV_HAVAL_FILE, 3).

-export([start/0, stop/0]).
-export([info/0, haval_string/1, haval_file/1]).
-ifdef(debug).
-export([test/0, test/1]).
-endif.

%%
%% Public interface
%%
start() ->
start("haval_drv"),
ok.

start(SharedLib) ->
case erl_ddll:load_driver(".", SharedLib) of
ok -> ok;
{error, already_loaded} -> ok;
_ -> exit({error, could_not_load_driver})
end,
register_lib(SharedLib).

stop() ->
[{port, Port}| _] = ets:lookup(haval_table, port),
Port ! {close, self()},
ok.

%% TODO: Implement this function return the information for module
info() -> ok.

%% Caluculate a value of HAVAL from string
haval_string(Str) ->
binary_to_term(control(?DRV_HAVAL_STRING, Str)).

%% Caluculate a value of HAVAL from file
haval_file(FileName) ->
binary_to_term(control(?DRV_HAVAL_FILE, FileName)).

-ifdef(debug).
test() ->
haval:start(),
Str = "I love Erlang.",
FileName = "haval.erl",
Hash1 = haval:haval_string(Str),
Hash2 = haval:haval_file(FileName),
io:format("HAVAL(~p) = ~p ~n", [Str, Hash1]),
io:format("HAVAL(~p) = ~p ~n", [FileName, Hash2]),
haval:stop(),
halt().

test([Str|_]) ->
haval:start(),
Hash = haval:haval_string(Str),
io:format("HAVAL(~p) = ~p ~n", [Str, Hash]),
haval:stop(),
halt().
-endif.

%%
%% Internal functions
%%
register_lib(SharedLib) ->
Port = open_port({spawn, SharedLib}, []),
Tab = ets:new(haval_table, [set, protected, named_table]),
ets:insert(Tab, {port, Port}).

control(Cmd, Data) ->
[{port, Port}| _] = ets:lookup(haval_table, port),
erlang:port_control(Port, Cmd, Data).



Step 2: Create C/C++ shared library
Shared library writted by C++ is loaded by Erlang interface when call erl_ddll::load_driver/2. This driver uses ei library in order to manipulate Erlang binary term.


/**
* HAVAL (cryptographic hash function) bindings for Erlang
*
* @author Rakuto Furutani <xri://=rakuto>
* @date 2008/06/28
*/
#include <stdio.h>
#include <string.h>
#include <stdarg.h>
#include <new>
#include "ei.h"
#include "erl_driver.h"
#include "erl_interface.h"
extern "C" {
// Define constant for haval.h
#define PASS 3
#define NUMBER_OF_BLOCKS 5000
#define FPTLEN 256
#ifndef LITTLE_ENDIAN
#define LITTLE_ENDIAN 1
#endif

#include "haval.h"
}

// Functions are provided by this driver
#define DRV_INFO 1
#define DRV_HAVAL_STRING 2
#define DRV_HAVAL_FILE 3

/* Driver Interface Declarations */
static ErlDrvData start_haval_driver(ErlDrvPort port, char *command);
static void stop_haval_driver(ErlDrvData drv_data);
static int control(ErlDrvData drv_data, unsigned int command, char *buf,
int len, char **rbuf, int rlen);
static void haval_to_hex(unsigned char*, char*);
static ErlDrvBinary* ei_x_to_new_binary(const ei_x_buff*);

/* Driver Entry */
static ErlDrvEntry haval_driver_entry = {
NULL,
start_haval_driver,
stop_haval_driver,
NULL,
NULL,
NULL,
"haval_drv",
NULL,
NULL,
control,
NULL,
NULL
};

typedef struct _drv_data {
ErlDrvPort port;
} drv_data_t;

/* DRIVER INTERFACE */

static ErlDrvData start_haval_driver(ErlDrvPort port, char *command)
{
drv_data_t *data;

data = (drv_data_t*) driver_alloc(sizeof(drv_data_t));
data->port = port;
set_port_control_flags(port, PORT_CONTROL_FLAG_BINARY);
return (ErlDrvData) data;
}

static void stop_haval_driver(ErlDrvData drv_data)
{
driver_free((char*) drv_data);
}

static int control(ErlDrvData drv_data, unsigned int command, char *buf, int len, char **rbuf, int rlen)
{
int ret = -1;
char hex[(FPTLEN >> 3) << 1];
unsigned char fingerprint[FPTLEN >> 3];
char *arg1;
ei_x_buff x_buff;

try {
// argument
arg1 = new char[len + 1];
strncpy(arg1, buf, len);
strcat(arg1, "�0");

ei_x_new_with_version(&x_buff);
switch(command) {
case DRV_INFO:
ei_x_encode_string(&x_buff, "info");
ret = sizeof("info") * sizeof(char);
break;
case DRV_HAVAL_STRING:
haval_string(arg1, fingerprint);
haval_to_hex(&fingerprint[0], &hex[0]);
ret = sizeof(hex);
ei_x_encode_string(&x_buff, hex);
break;
case DRV_HAVAL_FILE:
if(!haval_file(arg1, fingerprint)) {
haval_to_hex(&fingerprint[0], &hex[0]);
ret = sizeof(hex);
ei_x_encode_string(&x_buff, hex);
} else {
erl_err_sys("haval_file");
}
break;
}
if(ret > 0) *rbuf = reinterpret_cast<char*>(ei_x_to_new_binary(&x_buff));
ei_x_free(&x_buff);
} catch(std::bad_alloc) {
erl_err_sys("can not allocate memory");
}

return ret;
}

static void haval_to_hex(unsigned char *fingerprint, char *hex)
{
for(int i=0; i < FPTLEN >> 3; ++i) {
sprintf(&hex[i << 1], "%02X", fingerprint[i]);
}
}

// Init the driver
extern "C" DRIVER_INIT(haval_drv)
{
return &haval_driver_entry;
}

// Utilities
static ErlDrvBinary* ei_x_to_new_binary(const ei_x_buff *x_buff)
{
ErlDrvBinary *bin = driver_alloc_binary(x_buff->index);
if(bin != NULL) {
memcpy(bin->orig_bytes, x_buff->buff, x_buff->index);
}
return bin;
}


Step 3: Compile all sources
Create Makefile for build.
FYI: If you want to build linked-in driveron Mac OS, see Compile Erlang linked-in driver on Mac OSX (Darwin)


.PHONY: clean
.SUFFIXES: .o .c .cc .erl .beam

OS= ${shell uname}
CC=gcc
CXX=g++
CXXFLAGS=-Wall -g

# Erlang
ERL_INCLUDE = -I/usr/local/lib/erlang/usr/include
ERL_LIBS = -L/usr/local/lib/erlang/usr/lib \
-lerts
EI_INCLUDE = -I/usr/local/lib/erlang/lib/erl_interface-3.5.6/include
EI_LIBS = -L/usr/local/lib/erlang/lib/erl_interface-3.5.6/lib \
-lei \
-lerl_interface

TARGET_LIB = haval_drv.so
ifeq ($(OS), Darwin)
EXTRA_OPTIONS = -fno-common -bundle -undefined suppress -flat_namespace
endif
ALL: $(TARGET_LIB) haval.beam

.erl.beam:
erlc -W -Ddebug $<

.c.o:
$(CC) $(CFLAGS) -c $<

.cc.o:
$(CXX) $(CXXFLAGS) $(ERL_INCLUDE) $(EI_INCLUDE) -c $<

haval_drv.so: haval.o haval_drv.o
$(CXX) -o $@ $^ $(ERL_LIBS) $(EI_LIBS) $(EXTRA_OPTIONS) -fpic -O2

clean:
rm -f *.beam *.o *.so



Step 4: Run and test

% erl -noshell -run haval test
HAVAL("I love Erlang.") = "3AECEDF5133ED147C704A25C2CCA9A994FBB984FBCB22C1A523F31963E418498"
HAVAL("haval.erl") = "E006098555DFF788E73C7A263615DF60CC82396B1BCE7AEB5FB7050C376F2B03"

Saturday, June 28, 2008

Compile Erlang linked-in driver on Mac OSX (Darwin)

In order to compile the Erlang linked-in driver on Darwin, we must specify some options when it will be linked. If you don't specify these options, you might see errors is as follows:

% gcc -o my_drv.dymlib my_drv.c -dynamiclib -fpic

/usr/bin/ld: Undefined symbols:
_driver_alloc
_driver_free
_driver_output

That's invalid, so it must specify some options below in order to create the shared object.

% gcc -o my_drv.so my_drv.c -bundle -flat_namespace -udefined suppress


see README in source code of Erlang/OTP for more details.

Friday, June 27, 2008

Asynchronous driver in Erlang using driver_async

Erlang provides some mechanism is to communicate other language, this can be used for to reuse module, speed things up (Erlang is ten times as slow as slow than C), to access OS resource and so on. The below mechanism are provided.

Pipe Drivers
The "pipe drivers" enables bi-directional communication. The external process can be implemented in C or any language, and it can use erl_interface and ei library in order to manipulate data of Erlang as ETERM.

Benefit:

  • Frexible, it can use some types data.
  • Any language

Demerit:
  • Communication overhead

Linked-in Drivers
A driver in Erlang is a library written in C/C++, that is linked to the Erlang emulator and called from erlang. A driver can be dynamically loaded, as a shared library (known as a DLL on windows), or statically loaded, linked with the emulator when it is compiled and linked.

Benefit:
* Faster than "Pipe Drivers"

Demerit:
* Not frexible, to pass the complex data between C/C++ and Erlang is difficult. (see also ei)


Asynchonous Linked-in Driver
The "Linked-in Drivers" is faster than "Pipe Drivers", but it can not scale with a lot of process, since it run synchronously. So If you'd like to take advantage of platforms with multiple CPUs, then you must run some Erlang virtual machine since it run on single thread.

It sounds like great!, but we should care the some problems, multi-thread problem and so on.

Resources:

Next, I'll write an entry about how to create asynchronous driver.

Wednesday, June 18, 2008

[EC2] mount /dev/sdc device since /dev/sdb is not available

AWS team announced about persistent storage, see below the link.

http://www.allthingsdistributed.com/2008/04/persistent_storage_for_amazon.html

That's looks like pretty cool, but it's closed beta now, so we should use S3 or EC2 ephemeral storage (aka. Instance Storage). The 160GB storage is available on m1.large AMI, and some below devices are available.

/dev/sda1 Formatted and mounted as root (/) on all instance types
/dev/sda2 Formatted and mounted as /mnt on small instances
/dev/sda3 Formatted and mounted as /swap on small instances
/dev/sdb Formatted and mounted as /mnt on large and extra large instances
/dev/sdc Available on large and extra large instances; not mounted
/dev/sdd Available on extra large instances; not mounted
/dev/sde Available on extra large instances; not mounted


ATTENTION: The /dev/sdb isn't mounted to /mnt on m1.large AMI, see below a link for more details.
But the /dev/sdb device looks like that isn't mounted to /mnt.


% df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda1 9.9G 8.6G 801M 92% /
tmpfs 3.8G 0 3.8G 0% /lib/init/rw
udev 10M 10M 0 100% /dev
tmpfs 3.8G 4.0K 3.8G 1% /dev/shm
%


/dev/sdb is recognized in /etc/fstab, but it isn't mounted.

# Default /etc/fstab
# Supplied by: ec2-ami-tools-1.3-20041
/dev/sda1 / ext3 defaults 1 1
/dev/sdb /mnt ext3 defaults 0 0
none /dev/pts devpts gid=5,mode=620 0 0
none /proc proc defaults 0 0
none /sys sysfs defaults 0 0



% sudo mount /mnt
mount: /dev/sdb is not a valid block device


So we should initialize device to use as file system.
At first, optimize virtual devices before use them in production.

dd if=/dev/zero of=/dev/sdb bs=1M
dd if=/dev/zero of=/dev/sdc bs=1M
dd if=/dev/zero of=/dev/sdd bs=1M (m1.xlarge only)
dd if=/dev/zero of=/dev/sde bs=1M (m1.xlarge only)


Create a file system and mount it, and mounted /dev/sdc device is available now.

mkfs -t ext3 /dev/sdc
mount -t ext3 /dev/sdc /www



Filesystem Size Used Avail Use% Mounted on
/dev/sda1 9.9G 8.2G 1.2G 88% /
tmpfs 3.8G 0 3.8G 0% /lib/init/rw
udev 10M 10M 0 100% /dev
tmpfs 3.8G 4.0K 3.8G 1% /dev/shm
/dev/sdc 414G 6.7G 386G 2% /www

Tuesday, June 17, 2008

グラデーションブックスで本を探そう。amazonのおすすめ本ばかり読んでいたら世界が広がらない。


Photo via Flickr

グラデーションブックス

amazonの本ランキングや、楽天ブックスの本ランキングは、読者に新しい発見や本をブラウズする楽しみをもたらしているだろうか?実は、深いつながりがある、経済学の本と、最近話題のKYについて書いた本を一緒にリコメンドするだろうか?島嶼http://www.blogger.com/img/gl.link.gif生物学と企業の経営について、二つの本を結びつける事ができるだろうか?

先日、クローズアップ現代「ランキンhttp://www.blogger.com/img/gl.link.gifグ依存が止まらない」で、出版業界が抱える問題にスポットを当てた特集が放送された。とても素晴らしいまとめ記事が2008-06-04 今日の「クローズアップ現代」にある。
出版ニュース社の清田代表は、「ランキングで売れているからそれでいいというわけではない。出版の世界は奥が深いし、多様なものがあって、いろんな面白い本がある。むしろ、売れない本、売りにくい本こそ、いい本があって、埋もれてしまっている状況だ」


そのような状況の中、本を扱うサイト「グラデーションブックス」がオープンした。サイトの運営元である、澁川直子氏の言葉を引用する。

「従来の目的型検索やリコメンデーションサービスでは出会うことができな
かった、思いもよらない本を見つけ出せる喜びを提供できればと思います。」


amazonのリコメンドでは、思いもよらなかった発見が無い。極端な言い方をしてしまえば、amazonのオススメ本ばかり読んでいては、世界観が広がらない。だからこそ、人は、本屋に足を運び、本を買う。本好きの知人から、面白い本について教えてもらう。憧れている人が読んでいる本を、自分も読む。

上記に引用したブログ中にも書かれている通り、ランキングを見て本を購入する読者は、年に本を数冊しか読まないユーザである。たくさんの本を読んでいる人は、複数のジャンルを超えて、面白い本を発見し、つながりを見つけ出す英知を持っている。年間数冊しか読まない読者も、年間200冊読む読者と同じように、面白い本に出会えるサイトが、グラデーションブックスであるような気がしている。

技術本やビジネス本などの実用書しか読まなかった私も、このサービスをきっかけに小説やミステリーを読み出してみた。そんな、きっかけを与えてくれた面白い本を集めてくれている本棚をいくつかリンク。

chibinaoさん
Junさん
yukoさん
Saitoさん