python_test.clj

(ns libpython-clj2.python-test
 (:require [libpython-clj2.python :as py :refer [py. py.. py.- py* py**]]
 ;;support for tensor/numpy integration
 [libpython-clj2.python.np-array]
 [libpython-clj2.python.ffi :as py-ffi]
 [tech.v3.datatype :as dtype]
 [tech.v3.datatype.functional :as dfn]
 [tech.v3.datatype.ffi :as dt-ffi]
 [tech.v3.tensor :as dtt]
 [clojure.test :refer :all]
 [clojure.repl :refer [doc]]
 libpython-clj2.python.bridge-as-python)
 (:import [java.io StringWriter]
 [java.util Map List]
 [tech.v3.datatype.ffi Pointer]))


(py/initialize!)
(py-ffi/enable-gil-check!)
;; Useful to see how many times we convert a string to a python object.
;; (dt-ffi/enable-string->c-stats!)
;; (.addShutdownHook (Runtime/getRuntime) (Thread.
;; (fn []
;; (clojure.pprint/pprint
;; (dt-ffi/string->c-data-histogram)))))

(deftest stdout-and-stderr
 (is (= "hey\n" (with-out-str
 (py/run-simple-string "print('hey')"))))
 (is (= "hey\n" (let [custom-writer (StringWriter.)]
 (with-bindings {#'*err* custom-writer}
 (py/run-simple-string "import sys\nprint('hey', file=sys.stderr)"))
 (.toString custom-writer))))
 ;;Python exceptions get translated into actual java exceptions.
 (is (thrown? Throwable (py/run-simple-string "import sys\nprint('hey', stderr"))))

(deftest dicts
 (py/with-gil-stack-rc-context
 (let [py-dict (py/->python {:a 1 :b 2})]
 (is (= :dict (py/python-type py-dict)))
 (is (= 2 (-> (py/call-attr py-dict "__len__")
 (py/->jvm))))
 (is (= {"a" 1 "b" 2}
 (->> (py/->jvm py-dict)
 (into {}))))
 (let [bridge-dict (py/as-jvm py-dict)]
 (is (instance? Map bridge-dict))
 (is (= #{"a" "b"} (->> (keys bridge-dict)
 set)))
 (is (= #{1 2} (->> (vals bridge-dict)
 set)))
 (is (= {"a" 1 "b" 2}
 (into {} bridge-dict)))))))

(deftest lists
 (let [py-list (py/->py-list [4 3 2 1])]
 (is (= :list (py/python-type py-list)))
 (is (= 4 (py/call-attr py-list "__len__")))
 (is (= [4 3 2 1]
 (->> (py/->jvm py-list)
 (into []))))

 (let [bridge-list (py/as-jvm py-list)]
 (is (instance? List bridge-list))
 (is (= [4 3 2 1] (into [] bridge-list)))
 ;;This actually calls the python sort function.
 (.sort ^List bridge-list nil)
 (is (= [1 2 3 4] (into [] bridge-list))))))

(deftest global-dict
 (let [main-module (py/add-module "__main__")
 ^Map globals (-> (py/module-dict main-module)
 (py/as-jvm))]
 (is (instance? Map globals))
 (.put globals "item" 100)
 (py/set-item! globals "item2" 200)
 ;;During run-simple-string, if nothing else is specified the
 ;;global map is used as a local map.
 (py/run-simple-string "item3 = item + item2")
 (is (= 300 (globals "item3")))))


(deftest numpy-and-back
 (let [jvm-tens (dtt/->tensor (->> (range 9)
 (partition 3))
 :datatype :float64)]
 (let [py-tens (py/->python jvm-tens)]
 (is (= [[0.0 1.0 2.0] [3.0 4.0 5.0] [6.0 7.0 8.0]]
 (-> (dtt/as-tensor py-tens)
 dtt/->jvm)))
 ;;This operation is in-place
 (let [py-trans (py/call-attr py-tens "transpose" [1 0])]
 (is (= [[0.0 1.0 2.0] [3.0 4.0 5.0] [6.0 7.0 8.0]]
 (-> (dtt/as-tensor py-tens)
 dtt/->jvm)))
 (is (= [[0.0 3.0 6.0] [1.0 4.0 7.0] [2.0 5.0 8.0]]
 (-> (dtt/as-tensor py-trans)
 dtt/->jvm)))
 ;;But they are sharing backing store, so mutation will travel both
 ;;ways. Creepy action at a distance indeed
 (dtype/copy! [5 6 7] (nth (dtt/as-tensor py-trans) 0))
 (is (= [[5.0 1.0 2.0] [6.0 4.0 5.0] [7.0 7.0 8.0]]
 (-> (dtt/as-tensor py-tens)
 dtt/->jvm))))
 (let [main-module (py/add-module "__main__")
 ^Map globals (-> (py/module-dict main-module)
 (py/as-jvm))]
 (py/set-item! globals "np_ary" py-tens)
 (py/run-simple-string "np_ary[2,2] = 100")
 (is (= [[5.0 1.0 2.0] [6.0 4.0 5.0] [7.0 7.0 100.0]]
 ;;zero copy almost always works the other way, however. So there is
 ;;py/->tensor available. Copying the numpy object will allow the
 ;;zero copy pathway to work.
 (-> (dtt/as-tensor py-tens)
 dtt/->jvm)))))))

(deftest numpy-scalars
 (let [np (py/import-module "numpy")
 scalar-constructors (concat ["float64"
 "float32"]
 (for [int-type ["int" "uint"]
 int-width [8 16 32 64]]
 (str int-type int-width)))]
 (doseq [constructor scalar-constructors]
 (is (= 3.0 (-> (py/call-attr np constructor 3.0)
 double))
 (str "Item type " constructor)))))

(deftest dict-with-complex-key
 (let [py-dict (py/->python {["a" "b"] 1
 ["c" "d"] 2})
 bridged (py/as-jvm py-dict)]
 (is (= #{["a" "b"]
 ["c" "d"]}
 (->> (keys bridged)
 ;;Bridged tuples are lists, not persistent vectors.
 (map vec)
 set)))))

(deftest simple-print-crashed
 (let [numpy (py/import-module "numpy")]
 (println (dtt/as-tensor (py/call-attr numpy "ones" [3 3])))))

(deftest true-false-list
 (is (= [false true]
 (-> '(false true)
 py/->py-list
 py/->jvm))))

(deftest true-false-true-numpy
 (let [numpy (py/import-module "numpy")]
 (is (= [true false true]
 (->> (for [a (py/call-attr numpy "array" [true false true])]
 a)
 vec)))))

(py/from-import testcode defaults_fn)

(deftest from-import-adds-arglists-metadata
 (is (= '([& [{top "."
 topdown true
 onerror nil
 follow_symlinks false
 dir_fd nil}]]
 [& [{top "."
 topdown true
 follow_symlinks false
 dir_fd nil}]]
 [& [{top "."
 follow_symlinks false
 dir_fd nil}]]
 [& [{follow_symlinks false
 dir_fd nil}]])
 (:arglists (meta #'defaults_fn)))))

(deftest from-import-doc-renders-arglists
 (let [doc-output (with-out-str (doc defaults_fn))]
 (is (re-find #"defaults_fn" doc-output))
 (is (re-find #"\[\[& \[\{top \"\.\"" doc-output))
 (is (re-find #"Function with Python defaults for metadata tests\." doc-output))))

(deftest aspy-iter
 (let [testcode-module (py/import-module "testcode")]
 (is (= [1 2 3 4 5]
 (-> (py/call-attr testcode-module
 "for_iter" (py/as-python [1 2 3 4 5]))
 (py/->jvm))))
 (is (= ["a" "b" "c"]
 (-> (py/call-attr testcode-module
 "for_iter" (py/as-python {"a" 1 "b" 2 "c" 3}))
 (py/->jvm))))))

(deftest basic-with-test
 (let [testcode-module (py/import-module "testcode")]
 (let [fn-list (py/->py-list [])]
 (is (nil?
 (py/with [item (py/call-attr testcode-module "WithObjClass" true fn-list)]
 (py/call-attr item "doit_err"))))
 (is (= ["enter" "exit: ('Spam', 'Eggs')"]
 (py/->jvm fn-list))))
 (let [fn-list (py/->py-list [])]
 (is (= 1
 (py/with [item (py/call-attr testcode-module "WithObjClass" true fn-list)]
 (py/call-attr item "doit_noerr"))))
 (is (= ["enter" "exit: None"]
 (py/->jvm fn-list))))
 (let [fn-list (py/->py-list [])]
 (is (thrown? Throwable
 (py/with [item (py/call-attr testcode-module "WithObjClass"
 false fn-list)]
 (py/call-attr item "doit_err"))))
 (is (= ["enter" "exit: ('Spam', 'Eggs')"]
 (py/->jvm fn-list))))
 (let [fn-list (py/->py-list [])]
 (py/with [item (py/call-attr testcode-module "WithObjClass"
 false fn-list)]
 (py/call-attr item "doit_noerr"))
 (is (= ["enter" "exit: None"]
 (py/->jvm fn-list))))))

(deftest with-enter-returns-different-object
 (testing "py/with should bind the return value of __enter__, not the context manager"
 (let [testcode (py/import-module "testcode")]
 (py/with [f (py/call-attr testcode "FileWrapper" "test content")]
 ;; f should be the StringIO object returned by __enter__, not FileWrapper
 (is (= "test content" (py/call-attr f "read")))))))

(deftest arrow-as-fns-with-nil
 (is (= nil (py/->jvm nil)))
 (is (= nil (py/as-jvm nil))))

(deftest pydict-nil-get
 (let [dict (py/->python {:a 1 :b {:a 1 :b 2}})
 bridged (py/as-jvm dict)]
 (is (= nil (bridged nil)))))

(deftest bridged-dict-to-jvm
 (let [py-dict (py/->py-dict {:a 1 :b 2})
 bridged (py/as-jvm py-dict)
 copied-back (py/->jvm bridged)]
 (is (instance? clojure.lang.PersistentArrayMap copied-back))))

(deftest calling-conventions
 (let [np (py/import-module "numpy")
 linspace (py/py.. np -linspace)]

 (is (dfn/equals [2.000 2.250 2.500 2.750 3.000]
 (dtt/as-tensor (linspace 2 3 :num 5))))
 (is (dfn/equals [2.000 2.250 2.500 2.750 3.000]
 (dtt/as-tensor (py/$c linspace 2 3 :num 5))))
 (is (dfn/equals [2.000 2.250 2.500 2.750 3.000]
 (dtt/as-tensor (py/$a np linspace 2 3 :num 5))))))

(deftest syntax-sugar
 (py/initialize!)
 (let [np (py/import-module "numpy")]
 (is (= (str (py/py.- np linspace))
 (str (py/get-attr np "linspace"))))
 (is (= (str (py/py.. np -random -shuffle))
 (str (-> (py/get-attr np "random")
 (py/get-attr "shuffle"))))))


 (let [builtins (py/import-module "builtins")
 l (py/call-attr builtins "list")]
 (is (= (py/py. l __len__) 0))
 (py/py. l append 1)
 (is (= (py/py. l __len__) 1))
 (py/py.. l (extend [1 2 3]))
 (is (= ((py/py.- l __len__)) 4)))

 (let [sys (py/import-module "sys")]
 (is (int? (py/py.. sys -path __len__))))


 (let [{{Foo :Foo} :globals}
 (py/run-simple-string "
class Foo:

 def __init__(self, a, b):
 self.a = a
 self.b = b
 self._res = []

 def res(self):
 return self._res

 def extend(self, arg):
 self._res.extend(arg)
 return self

 def count(self):
 return len(self._res)

 def append(self, *args):
 return self.extend(args)


 def math(self, c):
 return self.append(self.a + self.b + c)

")
 f (Foo 1 2)]
 (is (= [] ((py/py.- f res)) (py/py. f res)))
 (is (= 6 (py/py.. f (append 1)
 (append 2)
 (math 3)
 (extend [4 5 6]) count)))
 (is (= [1 2 6 4 5 6] ((py/py.- f res)) (py/py. f res)))
 (is (= 6
 (py/py.. f res __len__)
 ((py/py.. f res -__len__))
 (py/py.. f count)
 ((py/py.. f -count)))))


 (let [builtins (py/import-module "builtins")
 dict (py/get-attr builtins "dict")
 d (dict)]
 (py* d update nil {:a 1})
 (is (= 1 (py* d get [:a])))
 (let [iterable [[:a 2]]]
 (py* d update [iterable]))
 (is (= 2 (py* d get [:a])))
 (let [iterable [[:a 1] [:b 2]]
 kwargs {:name "taco"}]
 (py* d update [iterable] kwargs))
 (is (= {"name" "taco" "a" 1 "b" 2} d))

 (py. d clear)

 (py** d update {:a 1})
 (is (= d {"a" 1}))

 (py** d update [[:a 2]] {:c 3})
 (is (= d {"a" 2 "c" 3}))


 (py. d clear)

 (doto d
 (py.. (*update nil {:a 1}))
 (py.. (*update [[[:b 2]]]))
 (py.. (**update {:c 3}))
 (py.. (**update [[:d 4]] {:e 5})))

 (is (= d {"a" 1 "b" 2 "c" 3 "d" 4 "e" 5}))))


(deftest infinite-seq
 (let [islice (-> (py/import-module "itertools")
 (py/get-attr "islice"))]

 (is (= (vec (range 10))
 ;;Range is an infinite sequence
 (-> (range)
 (islice 0 10)
 (vec))))))

(deftest persistent-vector-nparray
 (testing "Create numpy array from nested persistent vectors"
 (let [ary-data (-> (py/import-module "numpy")
 (py/$a array [[1 2 3]
 [4 5 6]]))]
 (is (dfn/equals (dtt/->tensor [[1 2 3]
 [4 5 6]])
 (dtt/as-tensor ary-data))))))

(deftest python-tuple-equals
 (testing "Python tuples have nice equal semantics."
 (let [lhs (py/->py-tuple [1 2])
 same (py/->py-tuple [1 2])
 not-same (py/->py-tuple [3 4])]
 (is (= lhs same))
 (is (not= lhs not-same))
 (is (= (.hashCode lhs)
 (.hashCode same)))
 (is (not= (.hashCode lhs)
 (.hashCode not-same))))))

(deftest range-nparray
 (let [ary-data (-> (py/import-module "numpy")
 (py/$a array (range 10)))]
 (is (dfn/equals (dtt/->tensor (range 10))
 (dtt/as-tensor ary-data)))))

(deftest false-is-always-py-false
 (let [py-false (py-ffi/py-false)
 ->false (py/->python false)
 as-false (py/as-python false)]
 (is (= (dt-ffi/->pointer py-false)
 (dt-ffi/->pointer ->false)))
 (is (= (dt-ffi/->pointer py-false)
 (dt-ffi/->pointer as-false)))))

(deftest instance-abc-classes
 (let [py-dict (py/->python {"a" 1 "b" 2})
 bridged-dict (py/as-python {"a" 1 "b" 2})
 bridged-iter (py/as-python (repeat 5 1))
 bridged-list (py/as-python (vec (range 10)))
 pycol (py/import-module "collections.abc")
 mapping-type (py/get-attr pycol "Mapping")
 iter-type (py/get-attr pycol "Iterable")
 sequence-type (py/get-attr pycol "Sequence")]
 (is (py/is-instance? py-dict mapping-type))
 (is (py/is-instance? bridged-dict mapping-type))
 (is (py/is-instance? bridged-iter iter-type))
 (is (py/is-instance? bridged-list sequence-type))))

(deftest nested-map-and-back
 (let [py-dict (-> (py/run-simple-string "testdata={'camera_id': 'CODOT-10106-12067', 'country': 'United States', 'state': 'Colorado', 'city': 'Fountain', 'provider': 'CO DOT', 'description': '0.6 mi N of Ray Nixon Rd Int', 'direction': 'North', 'video': False, 'links': {'jpeg': {'url': 'https://www.cotrip.org/dimages/camera?imageURL=remote/CTMCCAM025S125-20-N.jpg'}}, 'tags': ['auto_rerated'], 'ratings': {'road_weather': 4, 'visibility': 4}, 'health': {}, 'created_at': '2016-04-19T20:27:45.030Z', 'time_zone_offset': -25200}")
 (get-in [:globals "testdata"]))
 jvm-dict (py/->jvm py-dict)]
 (is (= (get jvm-dict "ratings")
 (py/->jvm (py/get-item py-dict "ratings"))))))

(deftest characters
 (is (= (py/->jvm (py/->python "c"))
 (py/->jvm (py/->python \c)))))


(deftest numpy-all
 (let [np (py/import-module "numpy")]
 (is (= true (py/call-attr np "all"
 (py/call-attr np "array" [true true true]))))
 (is (= false (py/call-attr np "all"
 (py/call-attr np "array" [true false true]))))))


(deftest np-dot
 (let [np (py/import-module "numpy")
 np-dot (py/get-attr np "dot")
 np-ary (py/call-attr np "array" [1 2 3])]
 (is (== 14 (np-dot np-ary np-ary)))))


(deftest ->python-expands-sequences
 (let [test-dict {:a (seq [1 2 3 4 5 7 8 9])}
 py-dict (py/->python test-dict)
 py-data (py/get-item py-dict "a")]
 (is (= 8 (py/call-attr py-data "__len__")))))


(deftest python-as-jvm-destructuring
 (let [py-dict (py/->python {:a 1})
 jvm-py-dict (py/as-jvm py-dict)
 {:keys [a b]} jvm-py-dict]
 (is (nil? b))))


(deftest incorrect-numpy-marshal
 (let [np (py/import-module "numpy")
 p (py/$a np full [2 5] 7)
 tp (py/get-attr np "transpose")
 pp (tp p)]
 (is (= [5 2]
 (py/->jvm (py/get-attr pp "shape"))))))

(deftest iter-should-not-npe
 ;; credit Carsten Behring
 (is (true? (boolean (py. (libpython-clj2.python.bridge-as-python/map-as-python {}) __iter__)))))


(deftest call-attr-test
 (let [np (py/import-module "numpy")
 random (py/get-attr np "random")
 data (py/call-attr random :randn 8 6)]
 (is (= [8 6] (dtype/shape (py/call-attr random :randn 8 6))))))


(comment
 (require '[libpython-clj.require :refer [require-python]])

 (require-python '[pandas :as pd])
 (require-python '[plotly.express :as px])
 (def px (py/import-module "plotly.express"))

 (let [data (doto (pd/DataFrame {:index [1 2] :value [2 3] :variable [1 1]})
 (py. melt :id_vars "index"))]
 (py. px line :data_frame data :x "index" :y "value" :color "variable"))


 (let [data (doto (pd/DataFrame {:index [1 2] :value [2 3] :variable [1 1]})
 (py. melt :id_vars "index"))]
 ((py.- px line) :data_frame data :x "index" :y "value" :color "variable")))