ExtSort

External merge sort algorithm, implemented in Go. Sort arbitrarily large data sets with a predictable amount of memory using disk.

Example:

Sorting lines:

import(
"fmt""github.com/bsm/extsort"
)
funcmain() {
// Init sorter.sorter:=extsort.New(nil)
defersorter.Close()
// Append plain data._=sorter.Append([]byte("foo"))
_=sorter.Append([]byte("bar"))
_=sorter.Append([]byte("baz"))
_=sorter.Append([]byte("dau"))
// Sort and iterate.iter, err:=sorter.Sort()
iferr!=nil {
panic(err)
}
deferiter.Close()
foriter.Next() {
fmt.Println(string(iter.Data()))
}
iferr:=iter.Err(); err!=nil {
panic(err)
}
}

Map-style API with de-duplication:

import(
"fmt""github.com/bsm/extsort"
)
funcmain() {
// Init with de-duplication.sorter:=extsort.New(&extsort.Options{
Dedupe: bytes.Equal,
})
defersorter.Close()
// Put key/value data._=sorter.Put([]byte("foo"), []byte("v1"))
_=sorter.Put([]byte("bar"), []byte("v2"))
_=sorter.Put([]byte("baz"), []byte("v3"))
_=sorter.Put([]byte("bar"), []byte("v4"))	// duplicate_=sorter.Put([]byte("dau"), []byte("v5"))
// Sort and iterate.iter, err:=sorter.Sort()
iferr!=nil {
panic(err)
}
deferiter.Close()
foriter.Next() {
fmt.Println(string(iter.Key()), string(iter.Value()))
}
iferr:=iter.Err(); err!=nil {
panic(err)
}
}