msgpack hello world

* msgpack cplusplus api lets you build an object graph with std structures
and it will serialize it.

* minimal cplusplus program to write a msgpack
================
#include
#include
#include
#include

using namespace std;

int main()
{
map > outer;
map inner;
inner["foo"] = 42;
inner["bar"] = 69;
outer["baz"]=inner;

msgpack::sbuffer sbuf;
msgpack::pack(sbuf, outer);
cout << sbuf.data() << endl;
}
================

* compilation
g++ -lmsgpack hello_msgpack_write.cpp -o hello_msgpack_write

* run
./hello_msgpack_write > x.msgpack

* unpack from python
python msgpack-to-json.py x.msgpack
{
"baz": {
"foo": 42,
"bar": 69
}
}

* msgpack-to-json.py
import sys
import msgpack
import json
print json.dumps(msgpack.loads(file(sys.argv[1]).read()), indent=4)

* msgpack C api

* packing bolierplate
msgpack_sbuffer sbuf;
msgpack_sbuffer_init(&sbuf);
msgpack_packer pk;
msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write);

* packing a list
msgpack_pack_array(&pk, 3);
msgpack_pack_int(&pk, 1);
msgpack_pack_true(&pk);
msgpack_pack_raw(&pk, 7);
msgpack_pack_raw_body(&pk, "example", 7);

* packing a map
/*
* packing a map {"foo"=>42, "bar"=>true}
* how to pack hierarchies? like lol or list of maps?
*/
msgpack_pack_map(&pk, 2); /* must know the number of items ahead of time. */
/* key */
msgpack_pack_raw(&pk, 3); /* length prefix?? there is no pack_string? */
msgpack_pack_raw_body(&pk, "foo", 3);
/* val */
msgpack_pack_int(&pk, 42);
/* another key */
msgpack_pack_raw(&pk, 3);
msgpack_pack_raw_body(&pk, "bar", 3);
/* another val */
msgpack_pack_true(&pk);

* reference: msgpack/example/simple.c in source distribution

* packing composite object

/* packing a list: [1, true, "example", {"foo"=>42, "bar"=>807}, 816] */

/* number of items must be known ahead of time */

msgpack_pack_array(&pk, 5); 

msgpack_pack_int(&pk, 1);

msgpack_pack_true(&pk);

/* packing a string is tedious .. */

msgpack_pack_raw(&pk, 7);

msgpack_pack_raw_body(&pk, "example", 7);

/* map of two items */

msgpack_pack_map(&pk, 2); 

/* key1 */

msgpack_pack_raw(&pk, 3);

msgpack_pack_raw_body(&pk, "foo", 3);

/* val1 */

msgpack_pack_int(&pk, 42);

/* key2 */

msgpack_pack_raw(&pk, 3);

msgpack_pack_raw_body(&pk, "bar", 3);

/* val2 */

msgpack_pack_int(&pk, 807);

/*

* element after the map. 

* we know this goes into the list, not the map because 

* the slots in the maps are exhausted.

*/

msgpack_pack_int(&pk, 816);

Permalink | Leave a comment  »

* module directory:
$ ls nginx-hello/
config
ngx_http_hello_module.c

# start with the simplest of non-filtering module, the empty gif.
cat ~/src/nginx-1.0.5/src/http/modules/ngx_http_empty_gif_module.c \
| sed 's/empty_gif/hello/g' \
> ngx_http_hello_module.c

# change the data from the empty gif to a string message.
# see source for detail.
vi ngx_http_hello_module.c

$ cat config
ngx_addon_name=ngx_http_hello_module
HTTP_MODULES="$HTTP_MODULES ngx_http_hello_module"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_hello_module.c"

* building nginx with your module

# fetch latest http://nginx.org/en/download.html
wget http://nginx.org/download/nginx-1.0.5.tar.gz

# first, build nginx as it is.
tar xzf nginx-1.0.5.tar.gz
cd nginx-1.0.5/
./configure
# it wants pcre..
sudo apt-get install libpcre3-dev libpcre3
./configure
make
#
# now rebuild with your module, pointing configure to the module directory.
#
./configure --add-module=/home/tengu/nginx
make
#
# setting up the runtime environment
#
sudo mkdir -p /usr/local/nginx/logs
mkdir /usr/local/nginx/conf
cp -r conf /usr/local/nginx/
#
# test run before installing
#
objs/nginx
# make it runnable as a regular user.
sed -i.dist 's|listen 80;|listen 8000;|'
/usr/local/nginx/conf/nginx.conf
#
# configuring the hello module: add this to "server" section of nginx.conf
# any url ending in hello will respond with lolcat greeting.
#
location ~ hello$ {
hello;
}

$ curl http://localhost:8000/hello
OH HAI

Permalink | Leave a comment  »

Permalink | Leave a comment  »

#
# dump hankaku-zenkaku correspondance for katakana chars in utf8.
#

sub hexstr { sprintf("%x", ord($_[0])) }

map {
my $z=chr($_);
my $h=Unicode::Japanese->new($z)->z2h->getu;
if ($z eq $h) {
say join "\t", $z, hexstr($z), '-', '-';
} else {
say join "\t", $z, hexstr($z), $h, hexstr($h);
}
} (0x30A0..0x30FF); # zenkaku katakana range

my$output= 30a0 - -
ァ 30a1 % ff67
ア 30a2 % ff71
ィ 30a3 % ff68
イ 30a4 % ff72
ゥ 30a5 % ff69
ウ 30a6 % ff73
ェ 30a7 % ff6a
エ 30a8 % ff74
ォ 30a9 % ff6b
オ 30aa % ff75
カ 30ab % ff76
ガ 30ac % ff76
キ 30ad % ff77
ギ 30ae % ff77
ク 30af % ff78
グ 30b0 % ff78
ケ 30b1 % ff79
ゲ 30b2 % ff79
コ 30b3 % ff7a
ゴ 30b4 % ff7a
サ 30b5 % ff7b
ザ 30b6 % ff7b
シ 30b7 % ff7c
ジ 30b8 % ff7c
ス 30b9 % ff7d
ズ 30ba % ff7d
セ 30bb % ff7e
ゼ 30bc % ff7e
ソ 30bd % ff7f
ゾ 30be % ff7f
タ 30bf % ff80
ダ 30c0 % ff80
チ 30c1 % ff81
ヂ 30c2 % ff81
ッ 30c3 % ff6f
ツ 30c4 % ff82
ヅ 30c5 % ff82
テ 30c6 % ff83
デ 30c7 % ff83
ト 30c8 % ff84
ド 30c9 % ff84
ナ 30ca % ff85
ニ 30cb % ff86
ヌ 30cc % ff87
ネ 30cd % ff88
ノ 30ce % ff89
ハ 30cf % ff8a
バ 30d0 % ff8a
パ 30d1 % ff8a
ヒ 30d2 % ff8b
ビ 30d3 % ff8b
ピ 30d4 % ff8b
フ 30d5 % ff8c
ブ 30d6 % ff8c
プ 30d7 % ff8c
ヘ 30d8 % ff8d
ベ 30d9 % ff8d
ペ 30da % ff8d
ホ 30db % ff8e
ボ 30dc % ff8e
ポ 30dd % ff8e
マ 30de % ff8f
ミ 30df % ff90
ム 30e0 % ff91
メ 30e1 % ff92
モ 30e2 % ff93
ャ 30e3 % ff6c
ヤ 30e4 % ff94
ュ 30e5 % ff6d
ユ 30e6 % ff95
ョ 30e7 % ff6e
ヨ 30e8 % ff96
ラ 30e9 % ff97
リ 30ea % ff98
ル 30eb % ff99
レ 30ec % ff9a
ロ 30ed % ff9b
ヮ 30ee - -
ワ 30ef % ff9c
ヰ 30f0 - -
ヱ 30f1 - -
ヲ 30f2 % ff66
ン 30f3 % ff9d
ヴ 30f4 % ff73
ヵ 30f5 - -
ヶ 30f6 - -
30f7 - -
30f8 - -
30f9 - -
30fa - -
・ 30fb ff65
ー 30fc ff70
ヽ 30fd - -
ヾ 30fe - -
30ff - -
__END__

Permalink | Leave a comment  »

064

$ curl http://localhost:8983/solr/update/ \
-H 'Content-type:text/xml; charset=utf-8' \
--data-binary ''

037

reference: see $solr_src/example/exampledocs/post.sh

Permalink | Leave a comment  »

#include "schema_types.xml"

...

This externalizes a large stable chunk of XML.
(type declarations in solr schema.xml in this case)

cpp invocation: cpp -x c -ansi -P

-x c specifies C as the language.
-ansi makes it strict
Combined, these options makes // not a comment, so that urls are preserved.

-P supresses the emission of linemarkers (lines prefixed by #)

with this gnu make rule %.xml : %.src.xml
cpp -x c -ansi -P $
make schema.xml would resolve #includes in schema.src.xml.

Permalink | Leave a comment  »

building
$ python setup.py build_ext --inplace
running build_ext
cythoning hello.pyx to hello.c
building 'hello' extension
gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c hello.c -o build/temp.linux-i686-2.6/hello.o
gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions build/temp.linux-i686-2.6/hello.o -o hello.so

$ ls Makefile
build # work dir
hello.c # generated
hello.pyx # source
hello.so # generated
setup.py # Makefile for cython

You supply setup.py and .pyx. build/ .c .so are generated.

You can import the generated shared library like:
>>> import hello
OH HAI

Permalink | Leave a comment  »

Permalink | Leave a comment  »

Most popular "scripting" languages claim to support closure, but they do it differently. I developed my expectation for closure from what little I know of scheme and perl. This expectation has been betrayed in other languages. Specifically, python and javascript does not seem to properly close on local variables in a loop block. I have compiled examples here to remind me of the differences, so that I don't get bitten in the backside again.

perl closing on loop local variable

#!/usr/bin/perl -w
use strict;
# perl closes on local variables in loop.
# this to me is proper lexical closure.
my @closures;
for(my $i=0; $i<5; $i++) {
    my $localvar="foo" . $i;
    push(@closures, sub { $localvar });
}
map { printf "%s\n", $_->() } @closures;
# output
# foo0
# foo1
# foo2
# foo3
# foo4

python does not close on loop local variables

closures=[]
for i in range(5):
    closures.append(lambda: "foo"+str(i))
for f in closures:
    print f()
"""
foo4
foo4
foo4
foo4
foo4
"""

javascript does not want to close on loop local variable either

var closures=[];
for (var i=0; i<5; i++) {
    var localvar="foo"+i;
    closures.push(function() { return localvar });
}
closures.map(function(f) { print(f()) });
// foo4
// foo4
// foo4
// foo4
// foo4

scheme

Note: None other than Matz has pointed out that this example is bogus in this (japanese) tweet: http://twitter.com/yukihiro_matz/status/26685527109

May be perl has corrupted my mind. I'll need to think about this when mind is clearer.

;; This is a silly exercise for scheme, 
;; since it's all functions and special forms and it's just going to work. 
;; But for the sake of completeness, here goes.
;; tested with chicken interpreter.
(let ([closures '()])
  (for-each 
   (lambda (i) 
     ;; close on i. push the func into a list.
     (set! closures (cons 
                     (lambda () i)
                     closures))
     )
   '(0 1 2 3 4))

  ;; invoke each closure
  (for-each 
   (lambda (f)
     (printf "foo~A\n" (f)))
   (reverse closures))
  )
;; foo0
;; foo1
;; foo2
;; foo3
;; foo4

What about others like ruby and lua?

Permalink | Leave a comment  »

managing kindle with Linux/Unix terminal

Note: If you have a normal GUI environment (osx, windows, ubunto), just use the file browser, instead of reading this.

Attach kindle to Linux via USB.

# find what device it appears as
$ dmesg | grep -i kindl
[40638.161519] scsi 2:0:0:0: Direct-Access Kindle Internal Storage 0100 PQ: 0 ANSI: 2
$ dmesg | grep 2:0:0:0:
...
[40638.195459] sd 2:0:0:0: [sdb] 3129280 512-byte hardware sectors: (1.60 GB/1.49 GiB)
...
# ok its sdb
$ dmesg | grep sdb | grep -v 2:0:0:0
[40638.406613] sdb: sdb1

mount kindle as vfat.

Note: I made the mistake of mounting as msdos. This chops up the
filename and suffix, so kindle will miss the files that you place in
it. For example, Update_kindle2_gw_2.5.4.bin ended up like update_5,
so upgrade option was not enabled. Also, Instapaper-2010-09-09.mobi
get chopped down to like insta~.mob. Kindle will not display this
presumably because of the unrecognized suffix.

# mounting kindle as mass storage device
$ sudo mount -t vfat /dev/sdb1 /mnt

# Kindle top-level directories
$ ls /mnt/
audible documents music system

# Kindle filesystem content
$ (cd /mnt/; find )
./system
./system/com.amazon.ebook.booklet.reader
./system/com.amazon.ebook.booklet.reader/reader.pref
./system/Search Indexes
./documents
./documents/Instapaper-2010-09-09.mobi
./documents/Kindle_Users_Guide.azw
./documents/Kindle_Users_Guide.mbp
./documents/ghuloum-.pdf
./documents/micro-ma.pdf
./music
./audible
...

Reading pdf with kindle. Technical papers are in pdf and they make for long reads, making them
ideal candidates for Kindle. Unfortunately, PDF support by Kindle,
even after v2.5 software upgrade, is suboptimal. You can zoom into
the text. This works for the left column in the typical two column
paper format. When you move to the 2nd column, unless the geometry
just works out, the right side might not fit in the screen. The
horizontal scroll in Kindle does not allow precise positioning. So
you have to play with the zooming size and horizontal scroll until the
right column fits in the screen with a legible character size. This
is a frustrating exercise. I'd rather print the damn pdf on the dead
tree.

There is a guy who wrote a python script to turn two-column pdf into a
single column layout so it can be confortably read in Kindle. But use
kindle at what cost? Is not the point to make it easier?

Instapaper seems to do a good job of turning a web page into a
kindleable format. May be they do that of pdf as well..

Unfortunately not.  from: http://www.instapaper.com/main/support

PDF support?

Instapaper currently does not support PDF files. The reading experience for PDFs isn’t very good on iPhones for a variety of practical and technical reasons, so official PDF support is unlikely to happen for the foreseeable future.

bummer.

Permalink | Leave a comment  »

A method can be added to a python class with a decorator like:

@add_m_url
class Annotation(Model):
...

Inheritance is considered the way to add behavior to a class, but
using a subtype to splice just one method seems like a overkill.
After all, subclassing was meant to simulate a whole range of
characteristics in modeling the world. Rather than gumming up modeling
mechanism with mundane, bookkeeping functionalities, why not use a
lightweight technique for adding a method?

def add_m_url(cls):
""" decorator to inject a method m_url() into the cls """
setattr(cls, 'm_url', lambda self: '/model/%s/%d/' % (self.__class__.__name__, self.id))
return cls

This assumes that the django model class has an integral primary key called 'id', which is the default, so it works unless you have gone out of your way to change the primary key.

The method gives an instance a place where it lives: /model/{ClassName}/{id}/

Python and django have nice meta-programming support, so its easy to
write generic code and attach it to desired objects (classes are
objects, too) with a decorator.

There is probably a fancy name for extending a class like this, but I don't know it. Neither do I know if this is considered a pythonic way. All I know is that with decorator, I can collapse a lot of boilerplate code without fancy methodologies. Decorator really hits the sweetspot in power/simplicity tradeoff.

Permalink | Leave a comment  »

I am a 40 something Japanese programmer working in San Francisco Bay
Area. Stories below here obtained by talking to my peers in Japan over the years.

A software developer in corporate Japan faces a choice in his (let's
face it; they are mostly men) early thirties; Stay with the 'elevator'
system, where a worker's title goes up automatically every couple of
years, or continue with his craft. Choosing the latter is akin to a monk
who takes the vow of poverty; he must sacrifice worldly benefits
forever. It sounds dramatic, but that is exactly the tone with which they speak of the Old Programmer: with a mix of respect and
pity.

Keeping up with the promotion schedule while programming part-time is
not an option, given the schedule pressure. Some companies actually
explicitly forbid workers who crossed the title threshold from
touching the development tools. I forgot what the exact title was.
It seems like combinatorics is used to create enough titles to for
every year of tenure.. But generally, when one becomes a "S.E." he
stops coding. S.E. stands for Systems Engineer, but it is not confined
to low-level engineering. It just means a technical person who
graduated from programming. Programming in Japan is considered a
menial work. SEs produce the systems and programmers type them in.
Why should SEs touch the development tools? All they need is
PowerPoint, excell and word. So it is from vi to PowerPoint at age 33.

This means that the most experienced practicing developer is 35 years
old at most companies. Put another way, 13 years of hands-on
professional experience is the upper-bound. Now most development
houses don't have the benefit of one of the giants of computing still hacking away, teaching younger developers by
examples. But it is clear that retiring programmers in their early
30's creates a technology transfer problem. Actually, it's not even a transfer problem; masters are prevented from occurring.

I suspect that this culture is a major cause of corporate Japan's poor
performance in software. Compare corporate Japan's software track
records to other fields. Can you name any corporate or personal
software that is exported? Where is Honda of software? Other than
games, which is tied to the hardware, exports seems practically
non-existent. Is this not odd for such a technological powerhouse?

Japan's uncompetitiveness does not apply to software as a whole; it's
just Corporate Software. Japan does produce internationally
recognized open source software. There is obviously Ruby. KAME
project brought IPV6 to BSD. It seems young Japanese hackers are
leading the effort to make Perl cool again. So outside the realm of
forced programmer retirement, Japan can produce good software. I have
to conclude that there is a systematic FAIL built into Corporate Japan when it comes to software creation. The culture of non-tolerance of older programmer must be a major factor in this.

What does this mean for silicon valley or US tech scene in general?
Well, it could a lot worse than your salary maxing out at age 40. At
least we are tolerated. (I personally do not see much ageism in the valley. I feel that if I
can impress the interviewers, I could get any job that would typically
go to a 27 year-old. I am grateful that I am able to work in such a
reasonable place. On exception is YC, but PG did make clear about the chance of anybody above 35 in one of his essays..)

But there is a bigger question than how old programmers feel. If my
conjecture is correct about the lack of old, wily programmers hurting
a whole industry, then what about the converse? What could happen if
the old wisdom is not merely tolerated but fully exploited? Could
this boost the valley even further? YC does an amazing job of getting
the maximum value out of young talents. Could the valley sitting on
another untapped resource?

Permalink | Leave a comment  »

#
# minimum three-column layout
# - sidebars
# - horizontal: absolute position by left: right:
# - vertical: absolute position by top:
# - center:
# - horizontal: margins matching the sidebar widths
# - vertical: container's padding-top #
<pre>

html

 head

  style type="text/css"

   ^div { border: 1px solid #ccc }

   ^.sidebar { top: 10px; width: 180px; }

 body

  div id="container"          style="position:relative; padding-top: 10px;"

   div id="center"   style="margin: 0 200 0 200;"

    ^Center

   div id="left" class="sidebar"   style="position:absolute; left:10px"

    ^Left

   div id="left" class="sidebar"   style="position:absolute; right:10px"

    ^Right

Permalink | Leave a comment  »

# hoge_root: path to your sandbox 
$ virtualenv --no-site-packages hoge_root
$ cd hoge_root/

# this sets up the shell variables. you effectively enter the environment with this command.

$ source bin/activate 

# http://pypi.python.org/pypi/pip says version can be specified here like:
# pip install django ==1.2.1
(hoge_root)hoge_root$ pip install django

(hoge_root)hoge_root$ pip install simplejson 

# verify django installation
(hoge_root)hoge_root$ python
>>> import django
>>> django
... info about the module. is it in the sandbox?
>>> django.VERSION
(1, 2, 1, 'final', 0)

pip freeze

note: git control dir under python lib dir(/usr/lib/python2.6/.git) can cause freeze to go haywire.

# take a snapshot $ pip freeze | tee hoge.pip.req
Django==1.2.1
simplejson==2.1.1
wsgiref==0.1.2

# now the environment can be replcated with :
$ virtualenv --no-site-packages another_root
$ pip install -E another_root -r path/to/hoge.pip.req

note: there seems to be no 'distfiles' directory where downloaded tarballs or whatever gets cached.
At least it is not used by default. So stuff gets downloaded every time.

Permalink | Leave a comment  »

# show current branch
$ git branch * master

# -a shows the current branch and 'remote-tracking' branches
$ git branch -a
* master
origin/HEAD
origin/master

# create a branch
$ git branch redsign-1

# is it there?
$ git branch
* master
redsign-1

# switch to the newly created branch with checkout.
$ git checkout redsign-1
M todo.org
Switched to branch "redsign-1"

# note the * $ git branch master
* redsign-1

# not sure if this was necessary, but there were some edits..
$ git commit -m'notes' .
Created commit 1f66060: notes
2 files changed, 14 insertions(+), 0 deletions(-)

# push the brach to the master
$ git push origin redsign-1
Counting objects: 9, done.
...
To example.com:/home/git/yoyodyne
* [new branch] redsign-1 -> redsign-1

# # switch back to master
#
$ git branch
master
* redsign-1

$ git checkout master
Switched to branch "master"

$ git branch
* master
redsign-1

-----------------------------
#
# setting up a new working copy of the new remote branch.
#
# you don't just yank the branch; clone the whole repo first.
#
$ git clone example.com:/home/git/yoyodyne
Initialized empty Git repository in /home/takumi/tmp/yoyodyne/.git/
...
Resolving deltas: 100% (4468/4468), done.

$ cd yoyodyne

# it's here as a remote branch.
yoyodyne$ git branch -a
* master
origin/HEAD
origin/master
origin/redsign-1

#
# the git seems to be to make a local tracking branch.
# git checkout -b
#
yoyodyne$ git checkout -b redesign-1 origin/redsign-1
Branch redesign-1 set up to track remote branch refs/remotes/origin/redsign-1.
Switched to a new branch "redesign-1"

# indeed, I am on the tracking brach.
yoyodyne$ git branch -a
master
* redesign-1
origin/HEAD
origin/master
origin/redsign-1

yoyodyne$ git push origin redesign-1
yoyodyne$ git pull origin redesign-1

Permalink | Leave a comment  »

#
# fdisk to find the device
#
$ sudo fdisk -l
...
Disk /dev/sdb: 80.0 GB, 80060424192 bytes
...

#
# cfdisk to create partition
#
$ sudo cfdisk /dev/sdb
# New >> Primary >> Size >> Write >> Quit

$ ls -l /dev/sdb*
brw-rw---- 1 root disk 8, 16 Jul 22 00:24 /dev/sdb
brw-rw---- 1 root disk 8, 17 Jul 22 00:24 /dev/sdb1

#
# format
#
$ sudo mkfs.ext3 /dev/sdb1

#
# mount
#
$ sudo mount -t ext3 /dev/sdb1 /mnt/
$ df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
/dev/sdb1 74G 180M 70G 1% /mnt

#
# conclusion: I miss FreeBSD and the Handbook..
# I have to look through gawdawful help sites just to # find how to format a drive. There seems to be no # canonical documentation by the maintainers of the # os/distros..
#

Permalink | Leave a comment  »

** format querystring from dictionary
>>> urllib.urlencode(dict(foo=1,bar=2))
'foo=1&bar=2'

** parse querystring into dictionary
>>> urlparse.parse_qs('foo=1&bar=2')
{'foo': ['1'], 'bar': ['2']}

** url-encode string
>>> urllib.quote('foo bar')
'foo%20bar'

** url-decode string
>>> urllib.unquote('foo%20bar')
'foo bar'

** parse full url
# parsing
>>> urlparse.urlparse('http://host.com:8000/dir/file?foo=1&bar=2')
ParseResult(scheme='http', netloc='host.com:8000', path='/dir/file', params='', query='foo=1&bar=2', fragment='')

# parts as tuple
>>> scheme,netloc,path,params,query,frag=pr
>>> scheme,netloc,path,params,query,frag
('http', 'host.com:8000', '/dir/file', '', 'foo=1&bar=2', '')

# access parts as attrs
>>> pr.hostname
'host.com'
>>> pr.netloc
'host.com:8000'
>>> pr.path
'/dir/file'
>>> pr.query
'foo=1&bar=2'

Permalink | Leave a comment  »

test

Permalink | Leave a comment  »