Add Python bindings for libvillas

.gitlab-ci.yml

@@ -164,7 +164,7 @@ test:python:
  - pytest --verbose .
  - black --line-length=90 --extend-exclude=".*(\\.pyi|_pb2.py)$" --check .
  - flake8 --max-line-length=90 --extend-exclude="*.pyi,*_pb2.py" .
- - mypy .
+ - mypy --explicit-package-bases .
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
  needs:
  - job: "build:source: [fedora]"
@@ -182,6 +182,28 @@ test:cppcheck:
  - cppcheck.log
  expose_as: cppcheck
 
+test:python_unit_integration:
+ stage: test
+ image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
+ before_script:
+ # dependency from node.py, which gets imported because of __init__.py in node/villas/python/node
+ - pip install requests
+ script:
+ # build binding and symlink to villas.node folder
+ # helps with correct imports and compatiblity for CI, binding wrapper and binding install
+ - export PYTHONPATH=$PYTHONPATH:${PWD}/python
+ - cmake --build build ${CMAKE_BUILD_OPTS} --target python_binding
+
+ - binding_path=$(find ${PWD}/build/python/binding/ -name "python_binding*.so" | head -n1)
+ - link_path=${PWD}/python/villas/node/$(basename $binding_path)
+ - ln -sf $binding_path $link_path
+
+ - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-unit-tests run-python-integration-tests
+ - rm $link_path
+ needs:
+ - job: "build:source: [fedora]"
+ artifacts: true
+
 test:unit:
  stage: test
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}

CMakeLists.txt

@@ -4,7 +4,7 @@
 # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
 # SPDX-License-Identifier: Apache-2.0
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.15)
 
 project(villas-node
  VERSION 1.1.0

include/villas/node.h

@@ -6,6 +6,7 @@
  */
 
 #pragma once
+#include <cstdint>
 
 #include <jansson.h>
 #include <stdbool.h>
@@ -67,11 +68,11 @@ unsigned sample_length(vsample *smp);
 
 void sample_decref(vsample *smp);
 
-vsample *sample_pack(unsigned seq, struct timespec *ts_origin,
+vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin,
  struct timespec *ts_received, unsigned len,
  double *values);
 
-void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin,
+void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin,
  struct timespec *ts_received, int *flags, unsigned *len,
  double *values);
 

lib/node.cpp

@@ -440,7 +440,7 @@ Node *NodeFactory::make(json_t *json, const uuid_t &id,
  std::string type;
  Node *n;
 
- if (json_is_object(json))
+ if (!json_is_object(json))
  throw ConfigError(json, "node-config-node",
  "Node configuration must be an object");
 

lib/node_capi.cpp

@@ -152,12 +152,12 @@ unsigned sample_length(vsample *s) {
  return smp->length;
 }
 
-vsample *sample_pack(unsigned seq, struct timespec *ts_origin,
+vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin,
  struct timespec *ts_received, unsigned len,
  double *values) {
  auto *smp = sample_alloc_mem(len);
 
- smp->sequence = seq;
+ smp->sequence = *seq;
  smp->ts.origin = *ts_origin;
  smp->ts.received = *ts_received;
  smp->length = len;
@@ -169,7 +169,7 @@ vsample *sample_pack(unsigned seq, struct timespec *ts_origin,
  return (vsample *)smp;
 }
 
-void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin,
+void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin,
  struct timespec *ts_received, int *flags, unsigned *len,
  double *values) {
  auto *smp = (Sample *)s;

lib/nodes/socket.cpp

@@ -67,6 +67,9 @@ int villas::node::socket_init(NodeCompat *n) {
  auto *s = n->getData<struct Socket>();
 
  s->formatter = nullptr;
+ s->sd = -1;
+ s->in.buf = nullptr;
+ s->out.buf = nullptr;
 
  return 0;
 }
@@ -77,6 +80,17 @@ int villas::node::socket_destroy(NodeCompat *n) {
  if (s->formatter)
  delete s->formatter;
 
+ if (s->in.buf != nullptr) {
+ memset(s->in.buf, 0, s->in.buflen);
+ delete[] s->in.buf;
+ s->in.buf = nullptr;
+ }
+ if (s->out.buf != nullptr) {
+ memset(s->out.buf, 0, s->out.buflen);
+ delete[] s->out.buf;
+ s->out.buf = nullptr;
+ }
+
  return 0;
 }
 
@@ -355,8 +369,16 @@ int villas::node::socket_stop(NodeCompat *n) {
  return ret;
  }
 
- delete[] s->in.buf;
- delete[] s->out.buf;
+ if (s->in.buf != nullptr) {
+ memset(s->in.buf, 0, s->in.buflen);
+ delete[] s->in.buf;
+ s->in.buf = nullptr;
+ }
+ if (s->out.buf != nullptr) {
+ memset(s->out.buf, 0, s->out.buflen);
+ delete[] s->out.buf;
+ s->out.buf = nullptr;
+ }
 
  return 0;
 }

packaging/docker/Dockerfile.debian

@@ -52,6 +52,8 @@ RUN apt-get update && \
 	libssl-dev \
 	libusb-1.0-0-dev \
 	libzmq3-dev \
+	python3-dev \
+	python3-pybind11 \
 	uuid-dev
 
 # Install unpackaged dependencies from source

packaging/docker/Dockerfile.debian-multiarch

@@ -58,6 +58,8 @@ RUN apt-get update && \
 	libssl-dev:${ARCH} \
 	libusb-1.0-0-dev:${ARCH} \
 	libzmq3-dev:${ARCH} \
+	python3-dev:${ARCH} \
+	python3-pybind11:${ARCH} \
 	uuid-dev:${ARCH}
 
 ADD cmake/toolchains/debian-${ARCH}.cmake /

packaging/docker/Dockerfile.fedora

@@ -27,7 +27,7 @@ RUN dnf -y install \
 	openssh-clients \
 	jq nmap-ncat \
 	iproute iproute-tc \
-	python python-devel python-pip \
+	python-devel python-pip \
 	gdb gdb-gdbserver \
 	cppcheck \
 	xmlto dblatex rubygem-asciidoctor \
@@ -63,6 +63,7 @@ RUN dnf -y install \
 	openssl-devel \
 	protobuf-c-devel \
 	protobuf-devel \
+	pybind11-devel \
 	spdlog-devel \
 	zeromq-devel
 

packaging/docker/Dockerfile.rocky

@@ -53,7 +53,9 @@ RUN dnf -y install \
 	nanomsg-devel \
 	libnice-devel \
 	libre-devel \
-	libwebsockets-devel
+	libwebsockets-devel \
+	python-devel \
+	pybind11-devel
 
 # Install unpackaged dependencies from source
 ADD packaging/patches /deps/patches

packaging/docker/Dockerfile.ubuntu

@@ -59,6 +59,8 @@ RUN apt-get update && \
 	libusb-1.0-0-dev \
 	libwebsockets-dev \
 	libzmq3-dev \
+	python3-dev \
+	python3-pybind11 \
 	uuid-dev
 
 # Install unpackaged dependencies from source

python/CMakeLists.txt

@@ -4,6 +4,8 @@
 # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
 # SPDX-License-Identifier: Apache-2.0
 
+add_subdirectory(binding)
+
 if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND)
  add_custom_command(
  OUTPUT

python/binding/CMakeLists.txt

@@ -0,0 +1,33 @@
+# Python-binding CMakeLists.
+#
+# Author: Kevin Vu te Laar <vu.te@rwth-aachen.de>
+# SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University
+# SPDX-License-Identifier: Apache-2.0
+
+set(PYBIND11_FINDPYTHON ON)
+find_package(pybind11 CONFIG)
+
+if(pybind11_FOUND)
+ find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
+
+ execute_process(
+ COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('purelib'))"
+ OUTPUT_VARIABLE PYTHON_SITE_PACKAGES
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+
+ message(STATUS "Found Python version: ${Python_VERSION}")
+ message(STATUS "Python major version: ${Python_VERSION_MAJOR}")
+ message(STATUS "Python minor version: ${Python_VERSION_MINOR}")
+
+ pybind11_add_module(python_binding capi_python_binding.cpp)
+ target_link_libraries(python_binding PUBLIC villas)
+
+ install(
+ TARGETS python_binding
+ COMPONENT lib
+ LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}/villas/node/
+ )
+else()
+ message(STATUS "pybind11 not found. Skipping Python wrapper build.")
+endif()