utils.lua

local dl = require 'dataload._env'

-- Temporarily changes the current working directory to call fn, 
-- returning its result.
function dl.withcwd(path, fn)
 local curdir = lfs.currentdir()
 lfs.chdir(path)
 local res = fn()
 lfs.chdir(curdir)
 return res
end

-- Downloads srcurl into dstdir.
-- If existfile is provided then 
-- download only if that file doesn't exist
function dl.downloadfile(dstdir, srcurl, existfile)
 if (not existfile) or (not paths.filep(existfile)) then
 paths.mkdir(dstdir)
 dl.withcwd(
 dstdir, 
 function() 
 local protocol, scpurl, filename = srcurl:match('(.-)://(.*)/(.-)$')
 if protocol == 'scp' then
 os.execute(string.format('%s %s %s', 'scp', scpurl .. '/' .. filename, filename))
 else
 os.execute('wget ' .. srcurl)
 end
 end
 )
 end
 return dstdir
end

-- Decompress a .tar, .tgz or .tar.gz file.
function dl.untar(srcpath, dstpath)
 local dstpath = dstpath or '.'
 paths.mkdir(dstpath)
 if srcpath:match("%.tar$") then
 os.execute('tar -xvf ' .. srcpath .. ' -C ' .. dstpath)
 else
 os.execute('tar -xvzf ' .. srcpath .. ' -C ' .. dstpath)
 end
end

-- unzip a .zip file
function dl.unzip(srcpath, dstpath)
 local dstpath = dstpath or '.'
 paths.mkdir(dstpath)
 os.execute('unzip ' .. srcpath .. ' -d ' .. dstpath)
end

-- gunzip a .gz file
function dl.gunzip(srcpath, dstpath)
 assert(not dstpath, "destination path not supported with gunzip")
 os.execute('gunzip ' .. srcpath)
end

-- Decompresses srczip into dstdir.
-- If existfile is provided then 
-- decompress only if that file doesn't exist.
-- Supported extensions for srczip are : .zip, .tar, .tgz, .gz, .gzip
function dl.decompressfile(dstdir, srczip, existfile, verbose, dstfile)
 paths.mkdir(dstdir)
 
 if (not existfile) or (not paths.filep(existfile)) then
 dl.withcwd(dstdir,
 function()
 if verbose then 
 print("decompressing file: ", srczip) 
 end
 if string.find(srczip, ".zip") then
 dl.unzip(srczip, dstfile)
 elseif string.find(srczip, ".tar") or string.find(srczip, ".tgz") then
 dl.untar(srczip, dstfile)
 elseif string.find(srczip, ".gz") or string.find(srczip, ".gzip") then
 dl.gunzip(srczip, dstfile)
 else
 error("Don't know how to decompress file: ", srczip)
 end
 end
 )
 end
end

-- loads data as ascii, if that doesn't work, loads as binary.
function dl.load(path)
 local success, data = pcall(function() return torch.load(path, "ascii") end)
 if not success then
 data = torch.load(path, "binary")
 end
 return data
end


function dl.rescale(data, min, max, dmin, dmax)
 local range = max - min
 local dmin = dmin or data:min()
 local dmax = dmax or data:max()
 local drange = dmax - dmin

 data:add(-dmin)
 data:mul(range)
 data:mul(1/drange)
 data:add(min)
 
 return data
end

function dl.binarize(x, threshold)
 x[x:lt(threshold)] = 0
 x[x:ge(threshold)] = 1
 return x
end

-- text utility functions

-- this can be inefficent, I wouldn't use it for datasets larger than PTB
function dl.buildVocab(tokens, minfreq)
 assert(torch.type(tokens) == 'table', 'Expecting table')
 assert(torch.type(tokens[1]) == 'string', 'Expecting table of strings')
 minfreq = minfreq or -1
 assert(torch.type(minfreq) == 'number')
 local wordfreq = {}
 
 for i=1,#tokens do
 local word = tokens[i]
 wordfreq[word] = (wordfreq[word] or 0) + 1
 end
 
 local vocab, ivocab = {}, {}
 local wordseq = 0
 
 local _ = require 'moses'
 -- make sure ordering is consistent
 local words = _.sort(_.keys(wordfreq))
 
 local oov = 0
 for i, word in ipairs(words) do
 local freq = wordfreq[word]
 if freq >= minfreq then
 wordseq = wordseq + 1
 vocab[word] = wordseq
 ivocab[wordseq] = word
 else
 oov = oov + freq
 end
 end
 
 if oov > 0 then
 wordseq = wordseq + 1
 wordfreq['<OOV>'] = oov
 vocab['<OOV>'] = wordseq
 ivocab[wordseq] = '<OOV>'
 end
 
 return vocab, ivocab, wordfreq
end

function dl.text2tensor(tokens, vocab)
 local oov = vocab['<OOV>']
 
 local tensor = torch.IntTensor(#tokens):fill(0)
 
 for i, word in ipairs(tokens) do
 local wordid = vocab[word] 
 
 if not wordid then
 assert(oov)
 wordid = oov
 end
 
 tensor[i] = wordid
 end
 
 return tensor
end

function dl.splitString(str,sep)
 local sep, fields = sep or ":", {}
 local pattern = string.format("([^%s]+)", sep)
 str:gsub(pattern, function(c) fields[#fields+1] = c end)
 return fields
end

function dl.getNumberOfLines(filename)
 local ctr = 0
 for _ in io.lines(filename) do
 ctr = ctr +1
 end
 return ctr
end

-- misc.

function dl.hostname()
 local f = io.popen ("/bin/hostname")
 if not f then 
 return 'localhost'
 end
 local hostname = f:read("*a") or ""
 f:close()
 hostname =string.gsub(hostname, "\n$", "")
 return hostname
end

-- Generates a globally unique identifier.
-- If a namespace is provided it is concatenated with 
-- the time of the call, and the next value from a sequence
-- to get a pseudo-globally-unique name.
-- Otherwise, we concatenate the linux hostname
local counter = 1
function dl.uniqueid(namespace, separator)
 local separator = separator or ':'
 local namespace = namespace or dl.hostname()
 local uid = namespace..separator..os.time()..separator..counter
 counter = counter + 1
 return uid
end

-- table

--http://lua-users.org/wiki/TableUtils
function table.val_to_str ( v )
 if "string" == type( v ) then
 v = string.gsub( v, "\n", "\\n" )
 if string.match( string.gsub(v,"[^'\"]",""), '^"+$' ) then
 return "'" .. v .. "'"
 end
 return '"' .. string.gsub(v,'"', '\\"' ) .. '"'
 else
 return "table" == type( v ) and table.tostring( v ) or tostring( v )
 end
end

function table.key_to_str ( k )
 if "string" == type( k ) and string.match( k, "^[_%a][_%a%d]*$" ) then
 return k
 else
 return "[" .. table.val_to_str( k ) .. "]"
 end
end

function table.tostring(tbl, newline, sort)
 local result, done = {}, {}
 for k, v in ipairs( tbl ) do
 table.insert( result, table.val_to_str( v ) )
 done[ k ] = true
 end
 local s = "="
 if newline then
 s = " : "
 end
 for k, v in pairs( tbl ) do
 if not done[ k ] then
 local line = table.key_to_str( k ) .. s .. table.val_to_str( v )
 table.insert(result, line)
 end
 end
 if sort then
 local _ = require 'moses'
 _.sort(result)
 end
 local res
 if newline then
 res = "{\n " .. table.concat( result, "\n " ) .. "\n}"
 else
 res = "{" .. table.concat( result, "," ) .. "}"
 end
 return res
end

function table.print(tbl)
 assert(torch.type(tbl) == 'table', "expecting table")
 print(table.tostring(tbl, true, true))
end