From 660b0e972222274567d873832682c9c8d1e38cf6 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Tue, 9 Dec 2025 21:37:27 -0600 Subject: [PATCH 01/11] remove-params-cython --- common/SConscript | 4 +- common/params.py | 2 +- common/params_module.cc | 522 ++++++++++++++++++++++++++++++++++++ common/params_pyx.pyx | 196 -------------- common/tests/test_params.py | 18 ++ 5 files changed, 543 insertions(+), 199 deletions(-) create mode 100644 common/params_module.cc delete mode 100644 common/params_pyx.pyx diff --git a/common/SConscript b/common/SConscript index c771ee78b7fd9c..de2299f467fda8 100644 --- a/common/SConscript +++ b/common/SConscript @@ -16,8 +16,8 @@ if GetOption('extras'): ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'], LIBS=[_common, 'json11', 'zmq', 'pthread']) -# Cython bindings -params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11']) +# C++ bindings +params_python = envCython.Program('_params.so', 'params_module.cc', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11']) SConscript([ 'transformations/SConscript', diff --git a/common/params.py b/common/params.py index 494617200f2cfc..11b5c33fc2a5af 100644 --- a/common/params.py +++ b/common/params.py @@ -1,4 +1,4 @@ -from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName +from openpilot.common._params import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName assert Params assert ParamKeyFlag assert ParamKeyType diff --git a/common/params_module.cc b/common/params_module.cc new file mode 100644 index 00000000000000..30005dd1082654 --- /dev/null +++ b/common/params_module.cc @@ -0,0 +1,522 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include + +#include "common/params.h" + +static PyObject *UnknownKeyName; +static PyObject *json_module; +static PyObject *json_dumps; +static PyObject *json_loads; + +static PyTypeObject ParamKeyFlagType = { + PyVarObject_HEAD_INIT(NULL, 0) +}; + +static PyTypeObject ParamKeyTypeType = { + PyVarObject_HEAD_INIT(NULL, 0) +}; + +typedef struct { + PyObject_HEAD + Params *p; + PyObject *d; +} ParamsObject; + +static void Params_dealloc(ParamsObject *self) { + if (self->p) { + delete self->p; + } + Py_XDECREF(self->d); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int Params_init(ParamsObject *self, PyObject *args, PyObject *kwds) { + const char *d = ""; + if (!PyArg_ParseTuple(args, "|s", &d)) { + return -1; + } + self->d = PyUnicode_FromString(d); + if (!self->d) return -1; + + std::string path(d); + Py_BEGIN_ALLOW_THREADS + self->p = new Params(path); + Py_END_ALLOW_THREADS + return 0; +} + +static PyObject* PyUnicode_FromStringAndSize_Bytes(const std::string& s) { + return PyUnicode_FromStringAndSize(s.c_str(), s.size()); +} + +static PyObject* cpp2python(ParamKeyType type, const std::string& value) { + switch (type) { + case ParamKeyType::STRING: + return PyUnicode_FromStringAndSize_Bytes(value); + case ParamKeyType::BOOL: + return PyBool_FromLong(value == "1"); + case ParamKeyType::INT: + try { + return PyLong_FromLong(std::stoi(value)); + } catch (...) { + return NULL; + } + case ParamKeyType::FLOAT: + try { + return PyFloat_FromDouble(std::stof(value)); + } catch (...) { + return NULL; + } + case ParamKeyType::TIME: { + PyObject *dt_module = PyImport_ImportModule("datetime"); + if (!dt_module) return NULL; + PyObject *dt_class = PyObject_GetAttrString(dt_module, "datetime"); + Py_DECREF(dt_module); + if (!dt_class) return NULL; + PyObject *val_str = PyUnicode_FromStringAndSize_Bytes(value); + PyObject *res = PyObject_CallMethod(dt_class, "fromisoformat", "O", val_str); + Py_DECREF(dt_class); + Py_DECREF(val_str); + if (!res) PyErr_Clear(); // Clear error to allow fallback + return res; + } + case ParamKeyType::JSON: { + PyObject *val_str = PyUnicode_FromStringAndSize_Bytes(value); + PyObject *res = PyObject_CallFunctionObjArgs(json_loads, val_str, NULL); + Py_DECREF(val_str); + if (!res) PyErr_Clear(); + return res; + } + case ParamKeyType::BYTES: + return PyBytes_FromStringAndSize(value.c_str(), value.size()); + } + Py_RETURN_NONE; +} + +static std::string python2cpp(ParamKeyType type, PyObject* value) { + switch (type) { + case ParamKeyType::STRING: + if (PyUnicode_Check(value)) { + return PyUnicode_AsUTF8(value); + } + break; + case ParamKeyType::BOOL: + return PyObject_IsTrue(value) ? "1" : "0"; + case ParamKeyType::INT: + case ParamKeyType::FLOAT: { + PyObject* str_val = PyObject_Str(value); + if (str_val) { + std::string s = PyUnicode_AsUTF8(str_val); + Py_DECREF(str_val); + return s; + } + break; + } + case ParamKeyType::TIME: { + if (PyObject_HasAttrString(value, "isoformat")) { + PyObject* iso = PyObject_CallMethod(value, "isoformat", NULL); + if (iso) { + std::string s = PyUnicode_AsUTF8(iso); + Py_DECREF(iso); + return s; + } + } + break; + } + case ParamKeyType::JSON: { + PyObject* dumped = PyObject_CallFunctionObjArgs(json_dumps, value, NULL); + if (dumped) { + std::string s = PyUnicode_AsUTF8(dumped); + Py_DECREF(dumped); + return s; + } + break; + } + case ParamKeyType::BYTES: + if (PyBytes_Check(value)) { + return std::string(PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + } + if (PyUnicode_Check(value)) { + return PyUnicode_AsUTF8(value); + } + break; + } + + if (type == ParamKeyType::BYTES) { + if (PyBytes_Check(value)) return std::string(PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + } + + return ""; +} + +static std::string ensure_bytes(PyObject* v) { + if (PyUnicode_Check(v)) { + return PyUnicode_AsUTF8(v); + } else if (PyBytes_Check(v)) { + return std::string(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v)); + } + return ""; +} + +static std::string check_key(ParamsObject* self, PyObject* key) { + std::string k = ensure_bytes(key); + if (!self->p->checkKey(k)) { + PyErr_SetString(UnknownKeyName, k.c_str()); + return ""; + } + return k; +} + +static PyObject* Params_check_key(ParamsObject* self, PyObject* key) { + std::string k = check_key(self, key); + if (PyErr_Occurred()) return NULL; + return PyBytes_FromStringAndSize(k.c_str(), k.size()); +} + +static PyObject* Params_clear_all(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"tx_flag", NULL}; + int tx_flag = ParamKeyFlag::ALL; + PyObject* tx_flag_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &tx_flag_obj)) { + return NULL; + } + if (tx_flag_obj) { + tx_flag = PyLong_AsLong(tx_flag_obj); + } + + self->p->clearAll((ParamKeyFlag)tx_flag); + Py_RETURN_NONE; +} + +static PyObject* Params_get(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", (char *)"block", (char *)"return_default", NULL}; + PyObject* key_obj; + int block = 0; + int return_default = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|pp", kwlist, &key_obj, &block, &return_default)) { + return NULL; + } + + std::string k = check_key(self, key_obj); + if (PyErr_Occurred()) return NULL; + + ParamKeyType t = self->p->getKeyType(k); + std::optional default_val_opt = self->p->getKeyDefaultValue(k); + + std::string val; + { + Py_BEGIN_ALLOW_THREADS + val = self->p->get(k, block); + Py_END_ALLOW_THREADS + } + + if (val.empty()) { + if (block) { + PyErr_SetNone(PyExc_KeyboardInterrupt); + return NULL; + } else { + if (return_default && default_val_opt.has_value()) { + return cpp2python(t, default_val_opt.value()); + } + Py_RETURN_NONE; + } + } + + PyObject* ret = cpp2python(t, val); + if (!ret) { + if (default_val_opt.has_value()) { + return cpp2python(t, default_val_opt.value()); + } + Py_RETURN_NONE; + } + return ret; +} + +static PyObject* Params_get_bool(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", (char *)"block", NULL}; + PyObject* key_obj; + int block = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|p", kwlist, &key_obj, &block)) { + return NULL; + } + + std::string k = check_key(self, key_obj); + if (PyErr_Occurred()) return NULL; + + bool r; + Py_BEGIN_ALLOW_THREADS + r = self->p->getBool(k, block); + Py_END_ALLOW_THREADS + + if (r) Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +static PyObject* Params_put(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", (char *)"dat", NULL}; + PyObject* key_obj; + PyObject* dat_obj; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &key_obj, &dat_obj)) { + return NULL; + } + + std::string k = check_key(self, key_obj); + if (PyErr_Occurred()) return NULL; + + ParamKeyType t = self->p->getKeyType(k); + std::string dat_bytes = python2cpp(t, dat_obj); + + Py_BEGIN_ALLOW_THREADS + self->p->put(k, dat_bytes); + Py_END_ALLOW_THREADS + + Py_RETURN_NONE; +} + +static PyObject* Params_put_bool(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", (char *)"val", NULL}; + PyObject* key_obj; + int val; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "Op", kwlist, &key_obj, &val)) { + return NULL; + } + + std::string k = check_key(self, key_obj); + if (PyErr_Occurred()) return NULL; + + Py_BEGIN_ALLOW_THREADS + self->p->putBool(k, val); + Py_END_ALLOW_THREADS + + Py_RETURN_NONE; +} + + +static PyObject* Params_put_nonblocking(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", (char *)"dat", NULL}; + PyObject* key_obj; + PyObject* dat_obj; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &key_obj, &dat_obj)) { + return NULL; + } + + std::string k = check_key(self, key_obj); + if (PyErr_Occurred()) return NULL; + + ParamKeyType t = self->p->getKeyType(k); + std::string dat_bytes = python2cpp(t, dat_obj); + + Py_BEGIN_ALLOW_THREADS + self->p->putNonBlocking(k, dat_bytes); + Py_END_ALLOW_THREADS + + Py_RETURN_NONE; +} + +static PyObject* Params_put_bool_nonblocking(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", (char *)"val", NULL}; + PyObject* key_obj; + int val; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "Op", kwlist, &key_obj, &val)) { + return NULL; + } + + std::string k = check_key(self, key_obj); + if (PyErr_Occurred()) return NULL; + + Py_BEGIN_ALLOW_THREADS + self->p->putBoolNonBlocking(k, val); + Py_END_ALLOW_THREADS + + Py_RETURN_NONE; +} + +static PyObject* Params_remove(ParamsObject* self, PyObject* key) { + std::string k = check_key(self, key); + if (PyErr_Occurred()) return NULL; + + Py_BEGIN_ALLOW_THREADS + self->p->remove(k); + Py_END_ALLOW_THREADS + + Py_RETURN_NONE; +} + +static PyObject* Params_get_param_path(ParamsObject* self, PyObject* args, PyObject* kwds) { + static char *kwlist[] = {(char *)"key", NULL}; + PyObject* key_obj = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &key_obj)) { + return NULL; + } + + std::string k; + if (key_obj && key_obj != Py_None) { + k = ensure_bytes(key_obj); + } + + std::string path = self->p->getParamPath(k); + return PyUnicode_FromString(path.c_str()); +} + +static PyObject* Params_get_type(ParamsObject* self, PyObject* key) { + std::string k = check_key(self, key); + if (PyErr_Occurred()) return NULL; + + ParamKeyType t = self->p->getKeyType(k); + return PyLong_FromLong((long)t); +} + +static PyObject* Params_all_keys(ParamsObject* self) { + std::vector keys = self->p->allKeys(); + PyObject* list = PyList_New(keys.size()); + for (size_t i = 0; i < keys.size(); i++) { + PyList_SetItem(list, i, PyBytes_FromStringAndSize(keys[i].c_str(), keys[i].size())); + } + return list; +} + +static PyObject* Params_get_default_value(ParamsObject* self, PyObject* key) { + std::string k = check_key(self, key); + if (PyErr_Occurred()) return NULL; + + ParamKeyType t = self->p->getKeyType(k); + std::optional default_val = self->p->getKeyDefaultValue(k); + if (default_val.has_value()) { + return cpp2python(t, default_val.value()); + } + Py_RETURN_NONE; +} + +static PyObject* Params_reduce(ParamsObject* self) { + return Py_BuildValue("(O(O))", Py_TYPE(self), self->d); +} + +static PyMethodDef Params_methods[] = { + {"__reduce__", (PyCFunction)Params_reduce, METH_NOARGS, "Pickle support"}, + {"clear_all", (PyCFunction)Params_clear_all, METH_VARARGS | METH_KEYWORDS, "Clear all params"}, + {"check_key", (PyCFunction)Params_check_key, METH_O, "Check if key exists"}, + {"get", (PyCFunction)Params_get, METH_VARARGS | METH_KEYWORDS, "Get param value"}, + {"get_bool", (PyCFunction)Params_get_bool, METH_VARARGS | METH_KEYWORDS, "Get param bool value"}, + {"put", (PyCFunction)Params_put, METH_VARARGS | METH_KEYWORDS, "Put param value"}, + {"put_bool", (PyCFunction)Params_put_bool, METH_VARARGS | METH_KEYWORDS, "Put param bool value"}, + {"put_nonblocking", (PyCFunction)Params_put_nonblocking, METH_VARARGS | METH_KEYWORDS, "Put param value non-blocking"}, + {"put_bool_nonblocking", (PyCFunction)Params_put_bool_nonblocking, METH_VARARGS | METH_KEYWORDS, "Put param bool value non-blocking"}, + {"remove", (PyCFunction)Params_remove, METH_O, "Remove param"}, + {"get_param_path", (PyCFunction)Params_get_param_path, METH_VARARGS | METH_KEYWORDS, "Get param path"}, + {"get_type", (PyCFunction)Params_get_type, METH_O, "Get param type"}, + {"all_keys", (PyCFunction)Params_all_keys, METH_NOARGS, "Get all keys"}, + {"get_default_value", (PyCFunction)Params_get_default_value, METH_O, "Get default value"}, + {NULL} +}; + +static PyTypeObject ParamsType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_params.Params", + .tp_basicsize = sizeof(ParamsObject), + .tp_dealloc = (destructor)Params_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = Params_methods, + .tp_init = (initproc)Params_init, + .tp_new = PyType_GenericNew, +}; + +static PyMethodDef params_functions[] = { + {NULL} +}; + +static struct PyModuleDef params_module = { + PyModuleDef_HEAD_INIT, + "_params", + NULL, + -1, + params_functions +}; + +PyMODINIT_FUNC PyInit__params(void) { + PyObject *m; + if (PyType_Ready(&ParamsType) < 0) + return NULL; + + m = PyModule_Create(¶ms_module); + if (m == NULL) + return NULL; + + Py_INCREF(&ParamsType); + if (PyModule_AddObject(m, "Params", (PyObject *)&ParamsType) < 0) { + Py_DECREF(&ParamsType); + Py_DECREF(m); + return NULL; + } + + // Initialize ParamKeyFlagType + ParamKeyFlagType.tp_name = "_params.ParamKeyFlag"; + ParamKeyFlagType.tp_basicsize = sizeof(PyObject); + ParamKeyFlagType.tp_flags = Py_TPFLAGS_DEFAULT; + ParamKeyFlagType.tp_doc = "ParamKeyFlag"; + + if (PyType_Ready(&ParamKeyFlagType) < 0) return NULL; + + // Add constants to ParamKeyFlag + PyObject *dict = ParamKeyFlagType.tp_dict; + PyDict_SetItemString(dict, "PERSISTENT", PyLong_FromLong(ParamKeyFlag::PERSISTENT)); + PyDict_SetItemString(dict, "CLEAR_ON_MANAGER_START", PyLong_FromLong(ParamKeyFlag::CLEAR_ON_MANAGER_START)); + PyDict_SetItemString(dict, "CLEAR_ON_ONROAD_TRANSITION", PyLong_FromLong(ParamKeyFlag::CLEAR_ON_ONROAD_TRANSITION)); + PyDict_SetItemString(dict, "CLEAR_ON_OFFROAD_TRANSITION", PyLong_FromLong(ParamKeyFlag::CLEAR_ON_OFFROAD_TRANSITION)); + PyDict_SetItemString(dict, "DEVELOPMENT_ONLY", PyLong_FromLong(ParamKeyFlag::DEVELOPMENT_ONLY)); + PyDict_SetItemString(dict, "CLEAR_ON_IGNITION_ON", PyLong_FromLong(ParamKeyFlag::CLEAR_ON_IGNITION_ON)); + PyDict_SetItemString(dict, "ALL", PyLong_FromUnsignedLong(ParamKeyFlag::ALL)); + + Py_INCREF(&ParamKeyFlagType); + if (PyModule_AddObject(m, "ParamKeyFlag", (PyObject *)&ParamKeyFlagType) < 0) { + Py_DECREF(&ParamKeyFlagType); + return NULL; + } + + // Initialize ParamKeyTypeType + ParamKeyTypeType.tp_name = "_params.ParamKeyType"; + ParamKeyTypeType.tp_basicsize = sizeof(PyObject); + ParamKeyTypeType.tp_flags = Py_TPFLAGS_DEFAULT; + ParamKeyTypeType.tp_doc = "ParamKeyType"; + + if (PyType_Ready(&ParamKeyTypeType) < 0) return NULL; + + // Add constants to ParamKeyType + dict = ParamKeyTypeType.tp_dict; + PyDict_SetItemString(dict, "STRING", PyLong_FromLong(ParamKeyType::STRING)); + PyDict_SetItemString(dict, "BOOL", PyLong_FromLong(ParamKeyType::BOOL)); + PyDict_SetItemString(dict, "INT", PyLong_FromLong(ParamKeyType::INT)); + PyDict_SetItemString(dict, "FLOAT", PyLong_FromLong(ParamKeyType::FLOAT)); + PyDict_SetItemString(dict, "TIME", PyLong_FromLong(ParamKeyType::TIME)); + PyDict_SetItemString(dict, "JSON", PyLong_FromLong(ParamKeyType::JSON)); + PyDict_SetItemString(dict, "BYTES", PyLong_FromLong(ParamKeyType::BYTES)); + + Py_INCREF(&ParamKeyTypeType); + if (PyModule_AddObject(m, "ParamKeyType", (PyObject *)&ParamKeyTypeType) < 0) { + Py_DECREF(&ParamKeyTypeType); + return NULL; + } + + UnknownKeyName = PyErr_NewException("_params.UnknownKeyName", NULL, NULL); + Py_INCREF(UnknownKeyName); + PyModule_AddObject(m, "UnknownKeyName", UnknownKeyName); + + // Initialize standard modules + json_module = PyImport_ImportModule("json"); + if (json_module) { + json_dumps = PyObject_GetAttrString(json_module, "dumps"); + json_loads = PyObject_GetAttrString(json_module, "loads"); + } + + PyDateTime_IMPORT; + + return m; +} diff --git a/common/params_pyx.pyx b/common/params_pyx.pyx deleted file mode 100644 index 93c550f22a2c8a..00000000000000 --- a/common/params_pyx.pyx +++ /dev/null @@ -1,196 +0,0 @@ -# distutils: language = c++ -# cython: language_level = 3 -import builtins -import datetime -import json -from libcpp cimport bool -from libcpp.string cimport string -from libcpp.vector cimport vector -from libcpp.optional cimport optional - -from openpilot.common.swaglog import cloudlog - -cdef extern from "common/params.h": - cpdef enum ParamKeyFlag: - PERSISTENT - CLEAR_ON_MANAGER_START - CLEAR_ON_ONROAD_TRANSITION - CLEAR_ON_OFFROAD_TRANSITION - DEVELOPMENT_ONLY - CLEAR_ON_IGNITION_ON - ALL - - cpdef enum ParamKeyType: - STRING - BOOL - INT - FLOAT - TIME - JSON - BYTES - - cdef cppclass c_Params "Params": - c_Params(string) except + nogil - string get(string, bool) nogil - bool getBool(string, bool) nogil - int remove(string) nogil - int put(string, string) nogil - void putNonBlocking(string, string) nogil - void putBoolNonBlocking(string, bool) nogil - int putBool(string, bool) nogil - bool checkKey(string) nogil - ParamKeyType getKeyType(string) nogil - optional[string] getKeyDefaultValue(string) nogil - string getParamPath(string) nogil - void clearAll(ParamKeyFlag) - vector[string] allKeys() - -PYTHON_2_CPP = { - (str, STRING): lambda v: v, - (builtins.bool, BOOL): lambda v: "1" if v else "0", - (int, INT): str, - (float, FLOAT): str, - (datetime.datetime, TIME): lambda v: v.isoformat(), - (dict, JSON): json.dumps, - (list, JSON): json.dumps, - (bytes, BYTES): lambda v: v, -} -CPP_2_PYTHON = { - STRING: lambda v: v.decode("utf-8"), - BOOL: lambda v: v == b"1", - INT: int, - FLOAT: float, - TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")), - JSON: json.loads, - BYTES: lambda v: v, -} - -def ensure_bytes(v): - return v.encode() if isinstance(v, str) else v - -class UnknownKeyName(Exception): - pass - -cdef class Params: - cdef c_Params* p - cdef str d - - def __cinit__(self, d=""): - cdef string path = d.encode() - with nogil: - self.p = new c_Params(path) - self.d = d - - def __reduce__(self): - return (type(self), (self.d,)) - - def __dealloc__(self): - del self.p - - def clear_all(self, tx_flag=ParamKeyFlag.ALL): - self.p.clearAll(tx_flag) - - def check_key(self, key): - key = ensure_bytes(key) - if not self.p.checkKey(key): - raise UnknownKeyName(key) - return key - - def python2cpp(self, proposed_type, expected_type, value, key): - cast = PYTHON_2_CPP.get((proposed_type, expected_type)) - if cast: - return cast(value) - raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}") - - def _cpp2python(self, t, value, default, key): - if value is None: - return None - try: - return CPP_2_PYTHON[t](value) - except (KeyError, TypeError, ValueError): - cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}") - return self._cpp2python(t, default, None, key) - - def get(self, key, bool block=False, bool return_default=False): - cdef string k = self.check_key(key) - cdef ParamKeyType t = self.p.getKeyType(k) - cdef optional[string] default = self.p.getKeyDefaultValue(k) - cdef string val - with nogil: - val = self.p.get(k, block) - - default_val = (default.value() if default.has_value() else None) if return_default else None - if val == b"": - if block: - # If we got no value while running in blocked mode - # it means we got an interrupt while waiting - raise KeyboardInterrupt - else: - return self._cpp2python(t, default_val, None, key) - return self._cpp2python(t, val, default_val, key) - - def get_bool(self, key, bool block=False): - cdef string k = self.check_key(key) - cdef bool r - with nogil: - r = self.p.getBool(k, block) - return r - - def _put_cast(self, key, dat): - cdef string k = self.check_key(key) - cdef ParamKeyType t = self.p.getKeyType(k) - return ensure_bytes(self.python2cpp(type(dat), t, dat, key)) - - def put(self, key, dat): - """ - Warning: This function blocks until the param is written to disk! - In very rare cases this can take over a second, and your code will hang. - Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but - in general try to avoid writing params as much as possible. - """ - cdef string k = self.check_key(key) - cdef string dat_bytes = self._put_cast(key, dat) - with nogil: - self.p.put(k, dat_bytes) - - def put_bool(self, key, bool val): - cdef string k = self.check_key(key) - with nogil: - self.p.putBool(k, val) - - def put_nonblocking(self, key, dat): - cdef string k = self.check_key(key) - cdef string dat_bytes = self._put_cast(key, dat) - with nogil: - self.p.putNonBlocking(k, dat_bytes) - - def put_bool_nonblocking(self, key, bool val): - cdef string k = self.check_key(key) - with nogil: - self.p.putBoolNonBlocking(k, val) - - def remove(self, key): - cdef string k = self.check_key(key) - with nogil: - self.p.remove(k) - - def get_param_path(self, key=""): - cdef string key_bytes = ensure_bytes(key) - return self.p.getParamPath(key_bytes).decode("utf-8") - - def get_type(self, key): - return self.p.getKeyType(self.check_key(key)) - - def all_keys(self): - return self.p.allKeys() - - def get_default_value(self, key): - cdef string k = self.check_key(key) - cdef ParamKeyType t = self.p.getKeyType(k) - cdef optional[string] default = self.p.getKeyDefaultValue(k) - return self._cpp2python(t, default.value(), None, key) if default.has_value() else None - - def cpp2python(self, key, value): - cdef string k = self.check_key(key) - cdef ParamKeyType t = self.p.getKeyType(k) - return self._cpp2python(t, value, None, key) diff --git a/common/tests/test_params.py b/common/tests/test_params.py index 592bf2c4b24cbc..c1545a31d32de6 100644 --- a/common/tests/test_params.py +++ b/common/tests/test_params.py @@ -139,3 +139,21 @@ def test_params_get_type(self): now = datetime.datetime.now(datetime.UTC) self.params.put("InstallDate", now) assert self.params.get("InstallDate") == now + + def test_params_type_conversion_errors(self): + p = self.params.get_param_path("LongitudinalPersonality") + with open(p, "w") as f: + f.write("not an int") + + val = self.params.get("LongitudinalPersonality", return_default=True) + assert isinstance(val, int) + + val = self.params.get("LongitudinalPersonality", return_default=False) + assert isinstance(val, int) + + p = self.params.get_param_path("ApiCache_FirehoseStats") + with open(p, "w") as f: + f.write("{ invalid json") + + val = self.params.get("ApiCache_FirehoseStats") + assert val is None From 97ce9882fd3e0693ac8276964a5891afb061260d Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Tue, 9 Dec 2025 22:15:29 -0600 Subject: [PATCH 02/11] remove-transformations-cython --- common/transformations/SConscript | 7 +- .../transformations/tests/test_coordinates.py | 26 + common/transformations/transformations.pyx | 173 ------- .../transformations/transformations_module.cc | 469 ++++++++++++++++++ 4 files changed, 501 insertions(+), 174 deletions(-) delete mode 100644 common/transformations/transformations.pyx create mode 100644 common/transformations/transformations_module.cc diff --git a/common/transformations/SConscript b/common/transformations/SConscript index 4ac73a165e4ba2..35b1cc00eec742 100644 --- a/common/transformations/SConscript +++ b/common/transformations/SConscript @@ -1,5 +1,10 @@ Import('env', 'envCython') +import numpy as np + transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc']) -transformations_python = envCython.Program('transformations.so', 'transformations.pyx') + +envCython.Append(CPPPATH=[np.get_include()]) +transformations_python = envCython.Program('transformations.so', ['transformations_module.cc'], + LIBS=[transformations]) Export('transformations', 'transformations_python') diff --git a/common/transformations/tests/test_coordinates.py b/common/transformations/tests/test_coordinates.py index 11a6bf70eefabb..360ac904fe95f7 100644 --- a/common/transformations/tests/test_coordinates.py +++ b/common/transformations/tests/test_coordinates.py @@ -102,3 +102,29 @@ def test_ned_batch(self): np.testing.assert_allclose(converter.ned2ecef(ned_offsets_batch), ecef_positions_offset_batch, rtol=1e-9, atol=1e-7) + + def test_errors(self): + # Test wrong length + with np.testing.assert_raises(ValueError): + coord.geodetic2ecef([0, 0]) + + with np.testing.assert_raises(ValueError): + coord.geodetic2ecef([0, 0, 0, 0]) + + # Test wrong type (scalar causes IndexError in numpy_wrap) + with np.testing.assert_raises(IndexError): + coord.geodetic2ecef(1) + + # Test invalid element types (valid length but strings) + with np.testing.assert_raises(TypeError): + coord.geodetic2ecef(['a', 'b', 'c']) + + # Test LocalCoord constructor errors + with np.testing.assert_raises(ValueError): + coord.LocalCoord.from_geodetic([0, 0]) + + with np.testing.assert_raises(ValueError): + coord.LocalCoord.from_geodetic(1) + + with np.testing.assert_raises(TypeError): + coord.LocalCoord.from_geodetic(['a', 'b', 'c']) diff --git a/common/transformations/transformations.pyx b/common/transformations/transformations.pyx deleted file mode 100644 index ae045c369d7506..00000000000000 --- a/common/transformations/transformations.pyx +++ /dev/null @@ -1,173 +0,0 @@ -# distutils: language = c++ -# cython: language_level = 3 -from openpilot.common.transformations.transformations cimport Matrix3, Vector3, Quaternion -from openpilot.common.transformations.transformations cimport ECEF, NED, Geodetic - -from openpilot.common.transformations.transformations cimport euler2quat as euler2quat_c -from openpilot.common.transformations.transformations cimport quat2euler as quat2euler_c -from openpilot.common.transformations.transformations cimport quat2rot as quat2rot_c -from openpilot.common.transformations.transformations cimport rot2quat as rot2quat_c -from openpilot.common.transformations.transformations cimport euler2rot as euler2rot_c -from openpilot.common.transformations.transformations cimport rot2euler as rot2euler_c -from openpilot.common.transformations.transformations cimport rot_matrix as rot_matrix_c -from openpilot.common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c -from openpilot.common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c -from openpilot.common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c -from openpilot.common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c -from openpilot.common.transformations.transformations cimport LocalCoord_c - - -import numpy as np -cimport numpy as np - -cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m): - return np.array([ - [m(0, 0), m(0, 1), m(0, 2)], - [m(1, 0), m(1, 1), m(1, 2)], - [m(2, 0), m(2, 1), m(2, 2)], - ]) - -cdef Matrix3 numpy2matrix(np.ndarray[double, ndim=2, mode="fortran"] m): - assert m.shape[0] == 3 - assert m.shape[1] == 3 - return Matrix3(m.data) - -cdef ECEF list2ecef(ecef): - cdef ECEF e - e.x = ecef[0] - e.y = ecef[1] - e.z = ecef[2] - return e - -cdef NED list2ned(ned): - cdef NED n - n.n = ned[0] - n.e = ned[1] - n.d = ned[2] - return n - -cdef Geodetic list2geodetic(geodetic): - cdef Geodetic g - g.lat = geodetic[0] - g.lon = geodetic[1] - g.alt = geodetic[2] - return g - -def euler2quat_single(euler): - cdef Vector3 e = Vector3(euler[0], euler[1], euler[2]) - cdef Quaternion q = euler2quat_c(e) - return [q.w(), q.x(), q.y(), q.z()] - -def quat2euler_single(quat): - cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3]) - cdef Vector3 e = quat2euler_c(q) - return [e(0), e(1), e(2)] - -def quat2rot_single(quat): - cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3]) - cdef Matrix3 r = quat2rot_c(q) - return matrix2numpy(r) - -def rot2quat_single(rot): - cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double)) - cdef Quaternion q = rot2quat_c(r) - return [q.w(), q.x(), q.y(), q.z()] - -def euler2rot_single(euler): - cdef Vector3 e = Vector3(euler[0], euler[1], euler[2]) - cdef Matrix3 r = euler2rot_c(e) - return matrix2numpy(r) - -def rot2euler_single(rot): - cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double)) - cdef Vector3 e = rot2euler_c(r) - return [e(0), e(1), e(2)] - -def rot_matrix(roll, pitch, yaw): - return matrix2numpy(rot_matrix_c(roll, pitch, yaw)) - -def ecef_euler_from_ned_single(ecef_init, ned_pose): - cdef ECEF init = list2ecef(ecef_init) - cdef Vector3 pose = Vector3(ned_pose[0], ned_pose[1], ned_pose[2]) - - cdef Vector3 e = ecef_euler_from_ned_c(init, pose) - return [e(0), e(1), e(2)] - -def ned_euler_from_ecef_single(ecef_init, ecef_pose): - cdef ECEF init = list2ecef(ecef_init) - cdef Vector3 pose = Vector3(ecef_pose[0], ecef_pose[1], ecef_pose[2]) - - cdef Vector3 e = ned_euler_from_ecef_c(init, pose) - return [e(0), e(1), e(2)] - -def geodetic2ecef_single(geodetic): - cdef Geodetic g = list2geodetic(geodetic) - cdef ECEF e = geodetic2ecef_c(g) - return [e.x, e.y, e.z] - -def ecef2geodetic_single(ecef): - cdef ECEF e = list2ecef(ecef) - cdef Geodetic g = ecef2geodetic_c(e) - return [g.lat, g.lon, g.alt] - - -cdef class LocalCoord: - cdef LocalCoord_c * lc - - def __init__(self, geodetic=None, ecef=None): - assert (geodetic is not None) or (ecef is not None) - if geodetic is not None: - self.lc = new LocalCoord_c(list2geodetic(geodetic)) - elif ecef is not None: - self.lc = new LocalCoord_c(list2ecef(ecef)) - - @property - def ned2ecef_matrix(self): - return matrix2numpy(self.lc.ned2ecef_matrix) - - @property - def ecef2ned_matrix(self): - return matrix2numpy(self.lc.ecef2ned_matrix) - - @property - def ned_from_ecef_matrix(self): - return self.ecef2ned_matrix - - @property - def ecef_from_ned_matrix(self): - return self.ned2ecef_matrix - - @classmethod - def from_geodetic(cls, geodetic): - return cls(geodetic=geodetic) - - @classmethod - def from_ecef(cls, ecef): - return cls(ecef=ecef) - - def ecef2ned_single(self, ecef): - assert self.lc - cdef ECEF e = list2ecef(ecef) - cdef NED n = self.lc.ecef2ned(e) - return [n.n, n.e, n.d] - - def ned2ecef_single(self, ned): - assert self.lc - cdef NED n = list2ned(ned) - cdef ECEF e = self.lc.ned2ecef(n) - return [e.x, e.y, e.z] - - def geodetic2ned_single(self, geodetic): - assert self.lc - cdef Geodetic g = list2geodetic(geodetic) - cdef NED n = self.lc.geodetic2ned(g) - return [n.n, n.e, n.d] - - def ned2geodetic_single(self, ned): - assert self.lc - cdef NED n = list2ned(ned) - cdef Geodetic g = self.lc.ned2geodetic(n) - return [g.lat, g.lon, g.alt] - - def __dealloc__(self): - del self.lc diff --git a/common/transformations/transformations_module.cc b/common/transformations/transformations_module.cc new file mode 100644 index 00000000000000..6ed1bc629cacff --- /dev/null +++ b/common/transformations/transformations_module.cc @@ -0,0 +1,469 @@ +#define PY_SSIZE_T_CLEAN +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include + +#include +#include + +#include "common/transformations/coordinates.hpp" +#include "common/transformations/orientation.hpp" + + +static bool list2ecef(PyObject *obj, ECEF &out) { + if (!PySequence_Check(obj) || PySequence_Size(obj) != 3) { + PyErr_SetString(PyExc_ValueError, "ECEF must be a sequence of length 3"); + return false; + } + PyObject *item; + item = PySequence_GetItem(obj, 0); out.x = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(obj, 1); out.y = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(obj, 2); out.z = PyFloat_AsDouble(item); Py_DECREF(item); + if (PyErr_Occurred()) return false; + return true; +} + +static bool list2ned(PyObject *obj, NED &out) { + if (!PySequence_Check(obj) || PySequence_Size(obj) != 3) { + PyErr_SetString(PyExc_ValueError, "NED must be a sequence of length 3"); + return false; + } + PyObject *item; + item = PySequence_GetItem(obj, 0); out.n = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(obj, 1); out.e = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(obj, 2); out.d = PyFloat_AsDouble(item); Py_DECREF(item); + if (PyErr_Occurred()) return false; + return true; +} + +static bool list2geodetic(PyObject *obj, Geodetic &out) { + if (!PySequence_Check(obj) || PySequence_Size(obj) != 3) { + PyErr_SetString(PyExc_ValueError, "Geodetic must be a sequence of length 3"); + return false; + } + PyObject *item; + item = PySequence_GetItem(obj, 0); out.lat = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(obj, 1); out.lon = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(obj, 2); out.alt = PyFloat_AsDouble(item); Py_DECREF(item); + out.radians = false; + if (PyErr_Occurred()) return false; + return true; +} + +static PyObject* ecef2list(const ECEF &e) { + PyObject *lst = PyList_New(3); + PyList_SetItem(lst, 0, PyFloat_FromDouble(e.x)); + PyList_SetItem(lst, 1, PyFloat_FromDouble(e.y)); + PyList_SetItem(lst, 2, PyFloat_FromDouble(e.z)); + return lst; +} + +static PyObject* ned2list(const NED &n) { + PyObject *lst = PyList_New(3); + PyList_SetItem(lst, 0, PyFloat_FromDouble(n.n)); + PyList_SetItem(lst, 1, PyFloat_FromDouble(n.e)); + PyList_SetItem(lst, 2, PyFloat_FromDouble(n.d)); + return lst; +} + +static PyObject* geodetic2list(const Geodetic &g) { + PyObject *lst = PyList_New(3); + PyList_SetItem(lst, 0, PyFloat_FromDouble(g.lat)); + PyList_SetItem(lst, 1, PyFloat_FromDouble(g.lon)); + PyList_SetItem(lst, 2, PyFloat_FromDouble(g.alt)); + return lst; +} + +static PyObject* vector3_to_list(const Eigen::Vector3d &v) { + PyObject *lst = PyList_New(3); + PyList_SetItem(lst, 0, PyFloat_FromDouble(v(0))); + PyList_SetItem(lst, 1, PyFloat_FromDouble(v(1))); + PyList_SetItem(lst, 2, PyFloat_FromDouble(v(2))); + return lst; +} + +static PyObject* quat_to_list(const Eigen::Quaterniond &q) { + PyObject *lst = PyList_New(4); + PyList_SetItem(lst, 0, PyFloat_FromDouble(q.w())); + PyList_SetItem(lst, 1, PyFloat_FromDouble(q.x())); + PyList_SetItem(lst, 2, PyFloat_FromDouble(q.y())); + PyList_SetItem(lst, 3, PyFloat_FromDouble(q.z())); + return lst; +} + +static PyObject* matrix3_to_numpy(const Eigen::Matrix3d &m) { + npy_intp dims[2] = {3, 3}; + PyObject *arr = PyArray_SimpleNew(2, dims, NPY_DOUBLE); + if (!arr) return NULL; + + + double *data = (double*)PyArray_DATA((PyArrayObject*)arr); + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) { + data[i*3 + j] = m(i, j); + } + } + return arr; +} + +static bool numpy_to_matrix3(PyObject *arr, Eigen::Matrix3d &out) { + PyArrayObject *arr_np = (PyArrayObject*)PyArray_ContiguousFromAny(arr, NPY_DOUBLE, 2, 2); + if (!arr_np) return false; + + if (PyArray_DIM(arr_np, 0) != 3 || PyArray_DIM(arr_np, 1) != 3) { + PyErr_SetString(PyExc_ValueError, "Matrix must be 3x3"); + Py_DECREF(arr_np); + return false; + } + + double *data = (double*)PyArray_DATA(arr_np); + + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) { + out(i, j) = data[i*3 + j]; + } + } + Py_DECREF(arr_np); + return true; +} + +// --- Module Functions --- + +static PyObject* meth_euler2quat_single(PyObject *self, PyObject *args) { + PyObject *euler_obj; + if (!PyArg_ParseTuple(args, "O", &euler_obj)) return NULL; + + if (!PySequence_Check(euler_obj) || PySequence_Size(euler_obj) != 3) { + PyErr_SetString(PyExc_ValueError, "Euler sequence must be size 3"); + return NULL; + } + + Eigen::Vector3d e; + PyObject *item; + item = PySequence_GetItem(euler_obj, 0); e(0) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(euler_obj, 1); e(1) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(euler_obj, 2); e(2) = PyFloat_AsDouble(item); Py_DECREF(item); + + if (PyErr_Occurred()) return NULL; + + return quat_to_list(euler2quat(e)); +} + +static PyObject* meth_quat2euler_single(PyObject *self, PyObject *args) { + PyObject *quat_obj; + if (!PyArg_ParseTuple(args, "O", &quat_obj)) return NULL; + + if (!PySequence_Check(quat_obj) || PySequence_Size(quat_obj) != 4) { + PyErr_SetString(PyExc_ValueError, "Quaternion sequence must be size 4"); + return NULL; + } + + double q_vals[4]; + for(int i=0; i<4; ++i) { + PyObject *item = PySequence_GetItem(quat_obj, i); + q_vals[i] = PyFloat_AsDouble(item); + Py_DECREF(item); + } + if (PyErr_Occurred()) return NULL; + + Eigen::Quaterniond q(q_vals[0], q_vals[1], q_vals[2], q_vals[3]); + return vector3_to_list(quat2euler(q)); +} + +static PyObject* meth_quat2rot_single(PyObject *self, PyObject *args) { + PyObject *quat_obj; + if (!PyArg_ParseTuple(args, "O", &quat_obj)) return NULL; + + if (!PySequence_Check(quat_obj) || PySequence_Size(quat_obj) != 4) { + PyErr_SetString(PyExc_ValueError, "Quaternion sequence must be size 4"); + return NULL; + } + double q_vals[4]; + for(int i=0; i<4; ++i) { + PyObject *item = PySequence_GetItem(quat_obj, i); + q_vals[i] = PyFloat_AsDouble(item); + Py_DECREF(item); + } + if (PyErr_Occurred()) return NULL; + + Eigen::Quaterniond q(q_vals[0], q_vals[1], q_vals[2], q_vals[3]); + return matrix3_to_numpy(quat2rot(q)); +} + +static PyObject* meth_rot2quat_single(PyObject *self, PyObject *args) { + PyObject *rot_obj; + if (!PyArg_ParseTuple(args, "O", &rot_obj)) return NULL; + + Eigen::Matrix3d m; + if (!numpy_to_matrix3(rot_obj, m)) return NULL; + + return quat_to_list(rot2quat(m)); +} + +static PyObject* meth_euler2rot_single(PyObject *self, PyObject *args) { + PyObject *euler_obj; + if (!PyArg_ParseTuple(args, "O", &euler_obj)) return NULL; + + if (!PySequence_Check(euler_obj) || PySequence_Size(euler_obj) != 3) { + PyErr_SetString(PyExc_ValueError, "Euler sequence must be size 3"); + return NULL; + } + + Eigen::Vector3d e; + PyObject *item; + item = PySequence_GetItem(euler_obj, 0); e(0) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(euler_obj, 1); e(1) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(euler_obj, 2); e(2) = PyFloat_AsDouble(item); Py_DECREF(item); + if (PyErr_Occurred()) return NULL; + + return matrix3_to_numpy(euler2rot(e)); +} + +static PyObject* meth_rot2euler_single(PyObject *self, PyObject *args) { + PyObject *rot_obj; + if (!PyArg_ParseTuple(args, "O", &rot_obj)) return NULL; + + Eigen::Matrix3d m; + if (!numpy_to_matrix3(rot_obj, m)) return NULL; + + return vector3_to_list(rot2euler(m)); +} + +static PyObject* meth_rot_matrix(PyObject *self, PyObject *args) { + double r, p, y; + if (!PyArg_ParseTuple(args, "ddd", &r, &p, &y)) return NULL; + return matrix3_to_numpy(rot_matrix(r, p, y)); +} + +static PyObject* meth_ecef_euler_from_ned_single(PyObject *self, PyObject *args) { + PyObject *ecef_init_obj, *ned_pose_obj; + if (!PyArg_ParseTuple(args, "OO", &ecef_init_obj, &ned_pose_obj)) return NULL; + + ECEF init; + if (!list2ecef(ecef_init_obj, init)) return NULL; + + if (!PySequence_Check(ned_pose_obj) || PySequence_Size(ned_pose_obj) != 3) { + PyErr_SetString(PyExc_ValueError, "NED pose must be size 3"); + return NULL; + } + Eigen::Vector3d pose; + PyObject *item; + item = PySequence_GetItem(ned_pose_obj, 0); pose(0) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(ned_pose_obj, 1); pose(1) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(ned_pose_obj, 2); pose(2) = PyFloat_AsDouble(item); Py_DECREF(item); + if (PyErr_Occurred()) return NULL; + + return vector3_to_list(ecef_euler_from_ned(init, pose)); +} + +static PyObject* meth_ned_euler_from_ecef_single(PyObject *self, PyObject *args) { + PyObject *ecef_init_obj, *ecef_pose_obj; + if (!PyArg_ParseTuple(args, "OO", &ecef_init_obj, &ecef_pose_obj)) return NULL; + + ECEF init; + if (!list2ecef(ecef_init_obj, init)) return NULL; + + if (!PySequence_Check(ecef_pose_obj) || PySequence_Size(ecef_pose_obj) != 3) { + PyErr_SetString(PyExc_ValueError, "ECEF pose must be size 3"); + return NULL; + } + Eigen::Vector3d pose; + PyObject *item; + item = PySequence_GetItem(ecef_pose_obj, 0); pose(0) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(ecef_pose_obj, 1); pose(1) = PyFloat_AsDouble(item); Py_DECREF(item); + item = PySequence_GetItem(ecef_pose_obj, 2); pose(2) = PyFloat_AsDouble(item); Py_DECREF(item); + if (PyErr_Occurred()) return NULL; + + return vector3_to_list(ned_euler_from_ecef(init, pose)); +} + +static PyObject* meth_geodetic2ecef_single(PyObject *self, PyObject *args) { + PyObject *g_obj; + if (!PyArg_ParseTuple(args, "O", &g_obj)) return NULL; + Geodetic g; + if (!list2geodetic(g_obj, g)) return NULL; + return ecef2list(geodetic2ecef(g)); +} + +static PyObject* meth_ecef2geodetic_single(PyObject *self, PyObject *args) { + PyObject *e_obj; + if (!PyArg_ParseTuple(args, "O", &e_obj)) return NULL; + ECEF e; + if (!list2ecef(e_obj, e)) return NULL; + return geodetic2list(ecef2geodetic(e)); +} + +// --- LocalCoord Class --- + +typedef struct { + PyObject_HEAD + LocalCoord *lc; +} LocalCoordObject; + +static void LocalCoord_dealloc(LocalCoordObject *self) { + if (self->lc) delete self->lc; + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int LocalCoord_init(LocalCoordObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {(char *)"geodetic", (char *)"ecef", NULL}; + PyObject *geodetic_obj = NULL; + PyObject *ecef_obj = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, &geodetic_obj, &ecef_obj)) { + return -1; + } + + if (geodetic_obj && geodetic_obj != Py_None) { + Geodetic g; + if (!list2geodetic(geodetic_obj, g)) return -1; + self->lc = new LocalCoord(g); + } else if (ecef_obj && ecef_obj != Py_None) { + ECEF e; + if (!list2ecef(ecef_obj, e)) return -1; + self->lc = new LocalCoord(e); + } else { + PyErr_SetString(PyExc_ValueError, "Must provide geodetic or ecef"); + return -1; + } + return 0; +} + +static PyObject* LocalCoord_get_ned2ecef_matrix(LocalCoordObject *self, void *closure) { + if (!self->lc) { + PyErr_SetString(PyExc_RuntimeError, "LocalCoord not initialized"); + return NULL; + } + return matrix3_to_numpy(self->lc->ned2ecef_matrix); +} + +static PyObject* LocalCoord_get_ecef2ned_matrix(LocalCoordObject *self, void *closure) { + if (!self->lc) { + PyErr_SetString(PyExc_RuntimeError, "LocalCoord not initialized"); + return NULL; + } + return matrix3_to_numpy(self->lc->ecef2ned_matrix); +} + +static PyObject* LocalCoord_from_geodetic(PyObject *cls, PyObject *g_obj) { + PyObject *kwds = PyDict_New(); + PyDict_SetItemString(kwds, "geodetic", g_obj); + + PyObject *empty_args = PyTuple_New(0); + PyObject *result = PyObject_Call(cls, empty_args, kwds); + Py_DECREF(empty_args); + Py_DECREF(kwds); + return result; +} + +static PyObject* LocalCoord_from_ecef(PyObject *cls, PyObject *e_obj) { + PyObject *kwds = PyDict_New(); + PyDict_SetItemString(kwds, "ecef", e_obj); + + PyObject *empty_args = PyTuple_New(0); + PyObject *result = PyObject_Call(cls, empty_args, kwds); + Py_DECREF(empty_args); + Py_DECREF(kwds); + return result; +} + +static PyObject* LocalCoord_ecef2ned_single(LocalCoordObject *self, PyObject *e_obj) { + if (!self->lc) { PyErr_SetString(PyExc_RuntimeError, "Uninitialized"); return NULL; } + ECEF e; + if (!list2ecef(e_obj, e)) return NULL; + return ned2list(self->lc->ecef2ned(e)); +} + +static PyObject* LocalCoord_ned2ecef_single(LocalCoordObject *self, PyObject *n_obj) { + if (!self->lc) { PyErr_SetString(PyExc_RuntimeError, "Uninitialized"); return NULL; } + NED n; + if (!list2ned(n_obj, n)) return NULL; + return ecef2list(self->lc->ned2ecef(n)); +} + +static PyObject* LocalCoord_geodetic2ned_single(LocalCoordObject *self, PyObject *g_obj) { + if (!self->lc) { PyErr_SetString(PyExc_RuntimeError, "Uninitialized"); return NULL; } + Geodetic g; + if (!list2geodetic(g_obj, g)) return NULL; + return ned2list(self->lc->geodetic2ned(g)); +} + +static PyObject* LocalCoord_ned2geodetic_single(LocalCoordObject *self, PyObject *n_obj) { + if (!self->lc) { PyErr_SetString(PyExc_RuntimeError, "Uninitialized"); return NULL; } + NED n; + if (!list2ned(n_obj, n)) return NULL; + return geodetic2list(self->lc->ned2geodetic(n)); +} + +static PyMethodDef LocalCoord_methods[] = { + {"from_geodetic", (PyCFunction)LocalCoord_from_geodetic, METH_O | METH_CLASS, "Create from geodetic"}, + {"from_ecef", (PyCFunction)LocalCoord_from_ecef, METH_O | METH_CLASS, "Create from ecef"}, + {"ecef2ned_single", (PyCFunction)LocalCoord_ecef2ned_single, METH_O, "Convert ecef to ned"}, + {"ned2ecef_single", (PyCFunction)LocalCoord_ned2ecef_single, METH_O, "Convert ned to ecef"}, + {"geodetic2ned_single", (PyCFunction)LocalCoord_geodetic2ned_single, METH_O, "Convert geodetic to ned"}, + {"ned2geodetic_single", (PyCFunction)LocalCoord_ned2geodetic_single, METH_O, "Convert ned to geodetic"}, + {NULL} +}; + +static PyGetSetDef LocalCoord_getset[] = { + {"ned2ecef_matrix", (getter)LocalCoord_get_ned2ecef_matrix, NULL, "NED to ECEF matrix", NULL}, + {"ecef2ned_matrix", (getter)LocalCoord_get_ecef2ned_matrix, NULL, "ECEF to NED matrix", NULL}, + {"ned_from_ecef_matrix", (getter)LocalCoord_get_ecef2ned_matrix, NULL, "Alias for ecef2ned_matrix", NULL}, + {"ecef_from_ned_matrix", (getter)LocalCoord_get_ned2ecef_matrix, NULL, "Alias for ned2ecef_matrix", NULL}, + {NULL} +}; + +static PyTypeObject LocalCoordType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "transformations.LocalCoord", + .tp_basicsize = sizeof(LocalCoordObject), + .tp_dealloc = (destructor)LocalCoord_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_methods = LocalCoord_methods, + .tp_getset = LocalCoord_getset, + .tp_init = (initproc)LocalCoord_init, + .tp_new = PyType_GenericNew, +}; + +static PyMethodDef transformations_methods[] = { + {"euler2quat_single", (PyCFunction)meth_euler2quat_single, METH_VARARGS, ""}, + {"quat2euler_single", (PyCFunction)meth_quat2euler_single, METH_VARARGS, ""}, + {"quat2rot_single", (PyCFunction)meth_quat2rot_single, METH_VARARGS, ""}, + {"rot2quat_single", (PyCFunction)meth_rot2quat_single, METH_VARARGS, ""}, + {"euler2rot_single", (PyCFunction)meth_euler2rot_single, METH_VARARGS, ""}, + {"rot2euler_single", (PyCFunction)meth_rot2euler_single, METH_VARARGS, ""}, + {"rot_matrix", (PyCFunction)meth_rot_matrix, METH_VARARGS, ""}, + {"ecef_euler_from_ned_single", (PyCFunction)meth_ecef_euler_from_ned_single, METH_VARARGS, ""}, + {"ned_euler_from_ecef_single", (PyCFunction)meth_ned_euler_from_ecef_single, METH_VARARGS, ""}, + {"geodetic2ecef_single", (PyCFunction)meth_geodetic2ecef_single, METH_VARARGS, ""}, + {"ecef2geodetic_single", (PyCFunction)meth_ecef2geodetic_single, METH_VARARGS, ""}, + {NULL} +}; + +static struct PyModuleDef transformations_module = { + PyModuleDef_HEAD_INIT, + "transformations", + NULL, + -1, + transformations_methods +}; + +PyMODINIT_FUNC PyInit_transformations(void) { + import_array(); + PyObject *m; + if (PyType_Ready(&LocalCoordType) < 0) + return NULL; + + m = PyModule_Create(&transformations_module); + if (m == NULL) + return NULL; + + Py_INCREF(&LocalCoordType); + if (PyModule_AddObject(m, "LocalCoord", (PyObject *)&LocalCoordType) < 0) { + Py_DECREF(&LocalCoordType); + Py_DECREF(m); + return NULL; + } + + return m; +} From 457842e1ab1f7dc30f898c7303de702a1e91ef82 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Tue, 9 Dec 2025 22:35:30 -0600 Subject: [PATCH 03/11] remove-pandad-cython --- selfdrive/pandad/SConscript | 2 +- selfdrive/pandad/__init__.py | 4 +- selfdrive/pandad/pandad_api_impl.cc | 184 +++++++++++++++++++++++++++ selfdrive/pandad/pandad_api_impl.pyx | 56 -------- selfdrive/pandad/tests/test_api.py | 35 +++++ 5 files changed, 222 insertions(+), 59 deletions(-) create mode 100644 selfdrive/pandad/pandad_api_impl.cc delete mode 100644 selfdrive/pandad/pandad_api_impl.pyx create mode 100644 selfdrive/pandad/tests/test_api.py diff --git a/selfdrive/pandad/SConscript b/selfdrive/pandad/SConscript index 58777cafe962eb..816413520557ec 100644 --- a/selfdrive/pandad/SConscript +++ b/selfdrive/pandad/SConscript @@ -6,7 +6,7 @@ panda = env.Library('panda', ['panda.cc', 'panda_comms.cc', 'spi.cc']) env.Program('pandad', ['main.cc', 'pandad.cc', 'panda_safety.cc'], LIBS=[panda] + libs) env.Library('libcan_list_to_can_capnp', ['can_list_to_can_capnp.cc']) -pandad_python = envCython.Program('pandad_api_impl.so', 'pandad_api_impl.pyx', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"]) +pandad_python = envCython.Program('_pandad_api_impl.so', 'pandad_api_impl.cc', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"]) Export('pandad_python') if GetOption('extras'): diff --git a/selfdrive/pandad/__init__.py b/selfdrive/pandad/__init__.py index cc680e16765f6b..7c09275c3a2f0b 100644 --- a/selfdrive/pandad/__init__.py +++ b/selfdrive/pandad/__init__.py @@ -1,4 +1,4 @@ -# Cython, now uses scons to build -from openpilot.selfdrive.pandad.pandad_api_impl import can_list_to_can_capnp, can_capnp_to_list +# C++ extension, now uses scons to build +from openpilot.selfdrive.pandad._pandad_api_impl import can_list_to_can_capnp, can_capnp_to_list assert can_list_to_can_capnp assert can_capnp_to_list diff --git a/selfdrive/pandad/pandad_api_impl.cc b/selfdrive/pandad/pandad_api_impl.cc new file mode 100644 index 00000000000000..86bf3837d4ac8c --- /dev/null +++ b/selfdrive/pandad/pandad_api_impl.cc @@ -0,0 +1,184 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include "selfdrive/pandad/can_types.h" + +void can_list_to_can_capnp_cpp(const std::vector &can_list, std::string &out, bool sendcan, bool valid); +void can_capnp_to_can_list_cpp(const std::vector &strings, std::vector &can_list, bool sendcan); + +static bool parse_can_msgs(PyObject *can_msgs, std::vector &frames) { + if (!PyList_Check(can_msgs)) { + PyErr_SetString(PyExc_TypeError, "can_msgs must be a list"); + return false; + } + + Py_ssize_t len = PyList_Size(can_msgs); + frames.reserve(len); + + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *item = PyList_GetItem(can_msgs, i); + if (!PySequence_Check(item) || PySequence_Size(item) < 3) { + PyErr_SetString(PyExc_ValueError, "Each CAN message must be a sequence of length at least 3 (address, data, src)"); + return false; + } + + PyObject *addr_obj = PySequence_GetItem(item, 0); + PyObject *dat_obj = PySequence_GetItem(item, 1); + PyObject *src_obj = PySequence_GetItem(item, 2); + + CanFrame frame; + frame.address = (uint32_t)PyLong_AsUnsignedLong(addr_obj); + + char *buffer = NULL; + Py_ssize_t length = 0; + if (PyBytes_Check(dat_obj)) { + if (PyBytes_AsStringAndSize(dat_obj, &buffer, &length) < 0) { + Py_DECREF(addr_obj); Py_DECREF(dat_obj); Py_DECREF(src_obj); + return false; + } + frame.dat.assign((uint8_t*)buffer, (uint8_t*)buffer + length); + } else { + Py_DECREF(addr_obj); Py_DECREF(dat_obj); Py_DECREF(src_obj); + PyErr_SetString(PyExc_TypeError, "CAN data must be bytes"); + return false; + } + + frame.src = PyLong_AsLong(src_obj); + + Py_DECREF(addr_obj); + Py_DECREF(dat_obj); + Py_DECREF(src_obj); + + if (PyErr_Occurred()) return false; + + frames.push_back(frame); + } + return true; +} + +static PyObject* method_can_list_to_can_capnp(PyObject *self, PyObject *args, PyObject *kwds) { + PyObject *can_msgs; + char *msgtype_str = (char*)"can"; + int valid = 1; + static char *kwlist[] = {(char*)"can_msgs", (char*)"msgtype", (char*)"valid", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sp", kwlist, &can_msgs, &msgtype_str, &valid)) { + return NULL; + } + + std::vector frames; + if (!parse_can_msgs(can_msgs, frames)) return NULL; + + bool sendcan = (strcmp(msgtype_str, "sendcan") == 0); + + std::string out; + { + Py_BEGIN_ALLOW_THREADS + can_list_to_can_capnp_cpp(frames, out, sendcan, valid); + Py_END_ALLOW_THREADS + } + + return PyBytes_FromStringAndSize(out.data(), out.size()); +} + +static PyObject* method_can_capnp_to_list(PyObject *self, PyObject *args, PyObject *kwds) { + PyObject *strings_obj; + char *msgtype_str = (char*)"can"; + static char *kwlist[] = {(char*)"strings", (char*)"msgtype", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s", kwlist, &strings_obj, &msgtype_str)) { + return NULL; + } + + std::vector strings; + if (!PySequence_Check(strings_obj)) { + PyErr_SetString(PyExc_TypeError, "strings must be a list or tuple"); + return NULL; + } + Py_ssize_t len = PySequence_Size(strings_obj); + strings.reserve(len); + for (Py_ssize_t i=0; i can_data; + + { + Py_BEGIN_ALLOW_THREADS + can_capnp_to_can_list_cpp(strings, can_data, sendcan); + Py_END_ALLOW_THREADS + } + + PyObject *result = PyList_New(can_data.size()); + if (!result) return NULL; + + for (size_t i=0; i < can_data.size(); ++i) { + const auto &cd = can_data[i]; + PyObject *frames_list = PyList_New(cd.frames.size()); + if (!frames_list) { + Py_DECREF(result); + return NULL; + } + + for (size_t j=0; j < cd.frames.size(); ++j) { + const auto &f = cd.frames[j]; + PyObject *frame_tuple = PyTuple_New(3); + if (!frame_tuple) { + Py_DECREF(frames_list); + Py_DECREF(result); + return NULL; + } + + PyTuple_SetItem(frame_tuple, 0, PyLong_FromUnsignedLong(f.address)); + PyTuple_SetItem(frame_tuple, 1, PyBytes_FromStringAndSize((char*)f.dat.data(), f.dat.size())); + PyTuple_SetItem(frame_tuple, 2, PyLong_FromLong(f.src)); + PyList_SetItem(frames_list, j, frame_tuple); + } + + PyObject *entry = PyTuple_New(2); + if (!entry) { + Py_DECREF(frames_list); + Py_DECREF(result); + return NULL; + } + PyTuple_SetItem(entry, 0, PyLong_FromUnsignedLongLong(cd.nanos)); + PyTuple_SetItem(entry, 1, frames_list); + + PyList_SetItem(result, i, entry); + } + return result; +} + +static PyMethodDef methods[] = { + {"can_list_to_can_capnp", (PyCFunction)method_can_list_to_can_capnp, METH_VARARGS | METH_KEYWORDS, "Convert list of can messages to capnp"}, + {"can_capnp_to_list", (PyCFunction)method_can_capnp_to_list, METH_VARARGS | METH_KEYWORDS, "Convert capnp messages to list"}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + "_pandad_api_impl", + NULL, + -1, + methods +}; + +PyMODINIT_FUNC PyInit__pandad_api_impl(void) { + return PyModule_Create(&module); +} diff --git a/selfdrive/pandad/pandad_api_impl.pyx b/selfdrive/pandad/pandad_api_impl.pyx deleted file mode 100644 index aaecb8a594eb30..00000000000000 --- a/selfdrive/pandad/pandad_api_impl.pyx +++ /dev/null @@ -1,56 +0,0 @@ -# distutils: language = c++ -# cython: language_level=3 -from cython.operator cimport dereference as deref, preincrement as preinc -from libcpp.vector cimport vector -from libcpp.string cimport string -from libcpp cimport bool -from libc.stdint cimport uint8_t, uint32_t, uint64_t - -cdef extern from "selfdrive/pandad/can_types.h": - cdef struct CanFrame: - long src - uint32_t address - vector[uint8_t] dat - - cdef struct CanData: - uint64_t nanos - vector[CanFrame] frames - -cdef extern from "can_list_to_can_capnp.cc": - void can_list_to_can_capnp_cpp(const vector[CanFrame] &can_list, string &out, bool sendcan, bool valid) nogil - void can_capnp_to_can_list_cpp(const vector[string] &strings, vector[CanData] &can_data, bool sendcan) - -def can_list_to_can_capnp(can_msgs, msgtype='can', valid=True): - cdef CanFrame *f - cdef vector[CanFrame] can_list - cdef uint32_t cpp_can_msgs_len = len(can_msgs) - - with nogil: - can_list.reserve(cpp_can_msgs_len) - - for can_msg in can_msgs: - f = &(can_list.emplace_back()) - f.address = can_msg[0] - f.dat = can_msg[1] - f.src = can_msg[2] - - cdef string out - cdef bool is_sendcan = (msgtype == 'sendcan') - cdef bool is_valid = valid - with nogil: - can_list_to_can_capnp_cpp(can_list, out, is_sendcan, is_valid) - return out - -def can_capnp_to_list(strings, msgtype='can'): - cdef vector[CanData] data - can_capnp_to_can_list_cpp(strings, data, msgtype == 'sendcan') - - result = [] - cdef CanData *d - cdef vector[CanData].iterator it = data.begin() - while it != data.end(): - d = &deref(it) - frames = [(f.address, (&f.dat[0])[:f.dat.size()], f.src) for f in d.frames] - result.append((d.nanos, frames)) - preinc(it) - return result diff --git a/selfdrive/pandad/tests/test_api.py b/selfdrive/pandad/tests/test_api.py new file mode 100644 index 00000000000000..4a688bb19ea779 --- /dev/null +++ b/selfdrive/pandad/tests/test_api.py @@ -0,0 +1,35 @@ + +import pytest +from openpilot.selfdrive.pandad import can_list_to_can_capnp, can_capnp_to_list + +def test_round_trip(): + # [(addr, data, src)] + msgs = [ + [123, b'data123', 0], + [456, b'data456', 1] + ] + + capnp_out = can_list_to_can_capnp(msgs, msgtype='can') + assert len(capnp_out) > 0 + + decoded = can_capnp_to_list([capnp_out], msgtype='can') + # Structure: [(nanos, [(addr, data, src)])] + assert len(decoded) == 1 + nanos, frames = decoded[0] + assert len(frames) == 2 + assert frames[0] == (123, b'data123', 0) + assert frames[1] == (456, b'data456', 1) + +def test_sendcan(): + msgs = [[0x200, b'mydata', 128]] + capnp_out = can_list_to_can_capnp(msgs, msgtype='sendcan') + decoded = can_capnp_to_list([capnp_out], msgtype='sendcan') + assert decoded[0][1][0] == (0x200, b'mydata', 128) + +def test_errors(): + with pytest.raises(TypeError): + can_list_to_can_capnp("not a list") + with pytest.raises(ValueError): + can_list_to_can_capnp([[123, b'd']]) + with pytest.raises(TypeError): + can_list_to_can_capnp([[123, "string_not_bytes", 0]]) From 2b0adc1b08a852136ee49786a5c35c3814298616 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 09:24:04 -0600 Subject: [PATCH 04/11] remove-commonmodel-cython --- selfdrive/modeld/SConscript | 5 +- selfdrive/modeld/dmonitoringmodeld.py | 2 +- selfdrive/modeld/modeld.py | 2 +- selfdrive/modeld/models/__init__.py | 7 + selfdrive/modeld/models/commonmodel_module.cc | 353 ++++++++++++++++++ selfdrive/modeld/models/commonmodel_pyx.pxd | 13 - selfdrive/modeld/models/commonmodel_pyx.pyx | 74 ---- 7 files changed, 364 insertions(+), 92 deletions(-) create mode 100644 selfdrive/modeld/models/commonmodel_module.cc delete mode 100644 selfdrive/modeld/models/commonmodel_pyx.pxd delete mode 100644 selfdrive/modeld/models/commonmodel_pyx.pyx diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 8b33a457f2088d..2cc075ae278d58 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -25,10 +25,9 @@ for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transfor for xenv in (lenv, lenvCython): xenv['CXXFLAGS'].append(f'-D{pathdef}_PATH=\\"{File(fn).abspath}\\"') -# Compile cython -cython_libs = envCython["LIBS"] + libs +# Compile C++ module commonmodel_lib = lenv.Library('commonmodel', common_src) -lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks) +lenvCython.Program('models/_commonmodel_module.so', ['models/commonmodel_module.cc'], LIBS=libs + [commonmodel_lib], FRAMEWORKS=frameworks) tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x] # Get model metadata diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index fca762c69bf504..5f19935100e0af 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -16,7 +16,7 @@ from openpilot.common.realtime import config_realtime_process from openpilot.common.transformations.model import dmonitoringmodel_intrinsics from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye -from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame +from openpilot.selfdrive.modeld.models import CLContext, MonitoringModelFrame from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid, safe_exp from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 006eeef6f5ed56..e147779c79ee50 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -28,7 +28,7 @@ from openpilot.selfdrive.modeld.parse_model_outputs import Parser from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState from openpilot.selfdrive.modeld.constants import ModelConstants, Plan -from openpilot.selfdrive.modeld.models.commonmodel_pyx import DrivingModelFrame, CLContext +from openpilot.selfdrive.modeld.models import DrivingModelFrame, CLContext from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address diff --git a/selfdrive/modeld/models/__init__.py b/selfdrive/modeld/models/__init__.py index e69de29bb2d1d6..518b1fb984a791 100644 --- a/selfdrive/modeld/models/__init__.py +++ b/selfdrive/modeld/models/__init__.py @@ -0,0 +1,7 @@ +from openpilot.selfdrive.modeld.models._commonmodel_module import ( + CLContext as CLContext, + CLMem as CLMem, + ModelFrame as ModelFrame, + DrivingModelFrame as DrivingModelFrame, + MonitoringModelFrame as MonitoringModelFrame, +) diff --git a/selfdrive/modeld/models/commonmodel_module.cc b/selfdrive/modeld/models/commonmodel_module.cc new file mode 100644 index 00000000000000..1a025880c6015d --- /dev/null +++ b/selfdrive/modeld/models/commonmodel_module.cc @@ -0,0 +1,353 @@ + +#define PY_SSIZE_T_CLEAN +#include +#include + +#include "selfdrive/modeld/models/commonmodel.h" + + +// Global Type Pointers +static PyTypeObject *CLContextType = NULL; +static PyTypeObject *CLMemType = NULL; +static PyTypeObject *ModelFrameType = NULL; +static PyTypeObject *DrivingModelFrameType = NULL; +static PyTypeObject *MonitoringModelFrameType = NULL; + +// --- CLContext --- +typedef struct { + PyObject_HEAD + cl_device_id device_id; + cl_context context; +} CLContext; + +static void CLContext_dealloc(CLContext *self) { + if (self->context) { + clReleaseContext(self->context); + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int CLContext_init(CLContext *self, PyObject *args, PyObject *kwds) { + self->device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT); + self->context = cl_create_context(self->device_id); + return 0; +} + +static PyObject *CLContext_get_device_id(CLContext *self, void *closure) { + return PyLong_FromUnsignedLongLong((unsigned long long)self->device_id); +} + +static PyObject *CLContext_get_context(CLContext *self, void *closure) { + return PyLong_FromUnsignedLongLong((unsigned long long)self->context); +} + +static PyGetSetDef CLContext_getset[] = { + {"device_id", (getter)CLContext_get_device_id, NULL, "OpenCL Device ID", NULL}, + {"context", (getter)CLContext_get_context, NULL, "OpenCL Context", NULL}, + {NULL} +}; + +static PyType_Slot CLContext_slots[] = { + {Py_tp_dealloc, (void*)CLContext_dealloc}, + {Py_tp_init, (void*)CLContext_init}, + {Py_tp_getset, CLContext_getset}, + {Py_tp_new, (void*)PyType_GenericNew}, + {0, 0} +}; + +static PyType_Spec CLContext_spec = { + "commonmodel_module.CLContext", + sizeof(CLContext), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + CLContext_slots +}; + +// --- CLMem --- +typedef struct { + PyObject_HEAD + cl_mem mem; +} CLMem; + +static void CLMem_dealloc(CLMem *self) { + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject *CLMem_get_mem_address(CLMem *self, void *closure) { + return PyLong_FromUnsignedLongLong((unsigned long long)self->mem); +} + +static PyGetSetDef CLMem_getset[] = { + {"mem_address", (getter)CLMem_get_mem_address, NULL, "CL Mem Address", NULL}, + {NULL} +}; + +static PyObject *CLMem_create(PyTypeObject *cls, PyObject *args) { + unsigned long long mem_ptr; + if (!PyArg_ParseTuple(args, "K", &mem_ptr)) return NULL; + + CLMem *obj = (CLMem *)cls->tp_alloc(cls, 0); + if (obj) { + obj->mem = (cl_mem)mem_ptr; + } + return (PyObject *)obj; +} + +static PyMethodDef CLMem_methods[] = { + {"create", (PyCFunction)CLMem_create, METH_VARARGS | METH_CLASS, "Create CLMem from pointer"}, + {NULL} +}; + +static PyType_Slot CLMem_slots[] = { + {Py_tp_dealloc, (void*)CLMem_dealloc}, + {Py_tp_getset, CLMem_getset}, + {Py_tp_methods, CLMem_methods}, + {Py_tp_new, (void*)PyType_GenericNew}, + {0, 0} +}; + +static PyType_Spec CLMem_spec = { + "commonmodel_module.CLMem", + sizeof(CLMem), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + CLMem_slots +}; + +static PyObject *cl_from_visionbuf(PyObject *self, PyObject *visionbuf) { + PyObject *buf_cl_obj = PyObject_GetAttrString(visionbuf, "buf_cl"); + if (!buf_cl_obj) { + PyErr_SetString(PyExc_TypeError, "VisionBuf object has no 'buf_cl' attribute"); + return NULL; + } + + unsigned long long buf_cl = PyLong_AsUnsignedLongLong(buf_cl_obj); + Py_DECREF(buf_cl_obj); + if (PyErr_Occurred()) return NULL; + + PyObject *args = PyTuple_Pack(1, PyLong_FromUnsignedLongLong(buf_cl)); + PyObject *clmem = PyObject_CallMethod((PyObject*)CLMemType, "create", "K", buf_cl); + Py_DECREF(args); + return clmem; +} + +// --- ModelFrame (Base) --- +typedef struct { + PyObject_HEAD + ModelFrame *frame; + int buf_size; +} PyModelFrame; + +static void ModelFrame_dealloc(PyModelFrame *self) { + if (self->frame) { + delete self->frame; + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + + +static PyObject *ModelFrame_prepare(PyModelFrame *self, PyObject *args) { + PyObject *visionbuf; + PyObject *projection_obj; // memoryview or buffer + + if (!PyArg_ParseTuple(args, "OO", &visionbuf, &projection_obj)) return NULL; + + auto get_int = [&](const char* name) -> size_t { + PyObject *attr = PyObject_GetAttrString(visionbuf, name); + if (!attr) return 0; // Error set + size_t val = PyLong_AsSize_t(attr); + Py_DECREF(attr); + return val; + }; + + cl_mem buf_cl = (cl_mem)get_int("buf_cl"); + if (PyErr_Occurred()) return NULL; + int width = (int)get_int("width"); + int height = (int)get_int("height"); + int stride = (int)get_int("stride"); + int uv_offset = (int)get_int("uv_offset"); + + // Extract projection matrix + Py_buffer view; + if (PyObject_GetBuffer(projection_obj, &view, PyBUF_SIMPLE) < 0) return NULL; + + mat3 cprojection; + if (view.len == 9 * sizeof(float)) { + memcpy(cprojection.v, view.buf, 9 * sizeof(float)); + } else { + PyBuffer_Release(&view); + PyErr_SetString(PyExc_ValueError, "Projection matrix must be 9 floats"); + return NULL; + } + PyBuffer_Release(&view); + + cl_mem *data_ptr = self->frame->prepare(buf_cl, width, height, stride, uv_offset, cprojection); + + if (!data_ptr) Py_RETURN_NONE; + + PyObject *res = PyObject_CallMethod((PyObject*)CLMemType, "create", "K", (unsigned long long)*data_ptr); + return res; +} + +static PyObject *numpy_module = NULL; + +static PyObject *ModelFrame_buffer_from_cl(PyModelFrame *self, PyObject *args) { + PyObject *clmem_obj; + if (!PyArg_ParseTuple(args, "O", &clmem_obj)) return NULL; + + if (!PyObject_TypeCheck(clmem_obj, CLMemType)) { + PyErr_SetString(PyExc_TypeError, "Argument must be CLMem"); + return NULL; + } + + CLMem* clmem = (CLMem*)clmem_obj; + unsigned char* data2 = self->frame->buffer_from_cl(&clmem->mem, self->buf_size); + + if (!numpy_module) { + numpy_module = PyImport_ImportModule("numpy"); + if (!numpy_module) return NULL; + } + + PyObject *memview = PyMemoryView_FromMemory((char*)data2, self->buf_size, PyBUF_READ); + if (!memview) return NULL; + + PyObject *dtype_str = PyUnicode_FromString("uint8"); + PyObject *args_np = PyTuple_Pack(2, memview, dtype_str); + PyObject *ret = PyObject_CallMethod(numpy_module, "array", "OO", memview, dtype_str); + + Py_DECREF(memview); + Py_DECREF(dtype_str); + Py_DECREF(args_np); + + return ret; +} + +static PyMethodDef ModelFrame_methods[] = { + {"prepare", (PyCFunction)ModelFrame_prepare, METH_VARARGS, ""}, + {"buffer_from_cl", (PyCFunction)ModelFrame_buffer_from_cl, METH_VARARGS, ""}, + {NULL} +}; + +static PyType_Slot ModelFrame_slots[] = { + {Py_tp_dealloc, (void*)ModelFrame_dealloc}, + {Py_tp_methods, ModelFrame_methods}, + {Py_tp_new, (void*)PyType_GenericNew}, + {0, 0} +}; + +static PyType_Spec ModelFrame_spec = { + "commonmodel_module.ModelFrame", + sizeof(PyModelFrame), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + ModelFrame_slots +}; + +// --- DrivingModelFrame --- +static int DrivingModelFrame_init(PyModelFrame *self, PyObject *args, PyObject *kwds) { + PyObject *ctx_obj; + int temporal_skip; + if (!PyArg_ParseTuple(args, "Oi", &ctx_obj, &temporal_skip)) return -1; + + if (!PyObject_TypeCheck(ctx_obj, CLContextType)) { + PyErr_SetString(PyExc_TypeError, "Context must be CLContext"); + return -1; + } + CLContext *ctx = (CLContext*)ctx_obj; + + self->frame = new DrivingModelFrame(ctx->device_id, ctx->context, temporal_skip); + self->buf_size = self->frame->buf_size; + return 0; +} + +static PyType_Slot DrivingModelFrame_slots[] = { + {Py_tp_init, (void*)DrivingModelFrame_init}, + {0, 0} +}; + +static PyType_Spec DrivingModelFrame_spec = { + "commonmodel_module.DrivingModelFrame", + sizeof(PyModelFrame), + 0, + Py_TPFLAGS_DEFAULT, + DrivingModelFrame_slots +}; + +// --- MonitoringModelFrame --- +static int MonitoringModelFrame_init(PyModelFrame *self, PyObject *args) { + PyObject *ctx_obj; + if (!PyArg_ParseTuple(args, "O", &ctx_obj)) return -1; + + if (!PyObject_TypeCheck(ctx_obj, CLContextType)) { + PyErr_SetString(PyExc_TypeError, "Context must be CLContext"); + return -1; + } + CLContext *ctx = (CLContext*)ctx_obj; + + self->frame = new MonitoringModelFrame(ctx->device_id, ctx->context); + self->buf_size = self->frame->buf_size; + return 0; +} + +static PyType_Slot MonitoringModelFrame_slots[] = { + {Py_tp_init, (void*)MonitoringModelFrame_init}, + {0, 0} +}; + +static PyType_Spec MonitoringModelFrame_spec = { + "commonmodel_module.MonitoringModelFrame", + sizeof(PyModelFrame), + 0, + Py_TPFLAGS_DEFAULT, + MonitoringModelFrame_slots +}; + +static PyMethodDef module_methods[] = { + {"cl_from_visionbuf", (PyCFunction)cl_from_visionbuf, METH_O, ""}, + {NULL} +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + "_commonmodel_module", + NULL, + -1, + module_methods +}; + +PyMODINIT_FUNC PyInit__commonmodel_module(void) { + PyObject *m = PyModule_Create(&module); + if (!m) return NULL; + + CLContextType = (PyTypeObject*)PyType_FromSpec(&CLContext_spec); + if (PyModule_AddObject(m, "CLContext", (PyObject *)CLContextType) < 0) return NULL; + Py_INCREF(CLContextType); + + CLMemType = (PyTypeObject*)PyType_FromSpec(&CLMem_spec); + if (PyModule_AddObject(m, "CLMem", (PyObject *)CLMemType) < 0) return NULL; + Py_INCREF(CLMemType); + + ModelFrameType = (PyTypeObject*)PyType_FromSpec(&ModelFrame_spec); + if (PyModule_AddObject(m, "ModelFrame", (PyObject *)ModelFrameType) < 0) return NULL; + Py_INCREF(ModelFrameType); + + PyObject *bases = PyTuple_Pack(1, ModelFrameType); + + DrivingModelFrameType = (PyTypeObject*)PyType_FromSpecWithBases(&DrivingModelFrame_spec, bases); + if (PyModule_AddObject(m, "DrivingModelFrame", (PyObject *)DrivingModelFrameType) < 0) { + Py_DECREF(bases); + return NULL; + } + Py_INCREF(DrivingModelFrameType); + + MonitoringModelFrameType = (PyTypeObject*)PyType_FromSpecWithBases(&MonitoringModelFrame_spec, bases); + if (PyModule_AddObject(m, "MonitoringModelFrame", (PyObject *)MonitoringModelFrameType) < 0) { + Py_DECREF(bases); + return NULL; + } + Py_INCREF(MonitoringModelFrameType); + + Py_DECREF(bases); + + return m; +} diff --git a/selfdrive/modeld/models/commonmodel_pyx.pxd b/selfdrive/modeld/models/commonmodel_pyx.pxd deleted file mode 100644 index 0bb798625be28d..00000000000000 --- a/selfdrive/modeld/models/commonmodel_pyx.pxd +++ /dev/null @@ -1,13 +0,0 @@ -# distutils: language = c++ - -from msgq.visionipc.visionipc cimport cl_mem -from msgq.visionipc.visionipc_pyx cimport CLContext as BaseCLContext - -cdef class CLContext(BaseCLContext): - pass - -cdef class CLMem: - cdef cl_mem * mem - - @staticmethod - cdef create(void*) diff --git a/selfdrive/modeld/models/commonmodel_pyx.pyx b/selfdrive/modeld/models/commonmodel_pyx.pyx deleted file mode 100644 index 5b7d11bc71aa66..00000000000000 --- a/selfdrive/modeld/models/commonmodel_pyx.pyx +++ /dev/null @@ -1,74 +0,0 @@ -# distutils: language = c++ -# cython: c_string_encoding=ascii, language_level=3 - -import numpy as np -cimport numpy as cnp -from libc.string cimport memcpy -from libc.stdint cimport uintptr_t - -from msgq.visionipc.visionipc cimport cl_mem -from msgq.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext -from .commonmodel cimport CL_DEVICE_TYPE_DEFAULT, cl_get_device_id, cl_create_context, cl_release_context -from .commonmodel cimport mat3, ModelFrame as cppModelFrame, DrivingModelFrame as cppDrivingModelFrame, MonitoringModelFrame as cppMonitoringModelFrame - - -cdef class CLContext(BaseCLContext): - def __cinit__(self): - self.device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT) - self.context = cl_create_context(self.device_id) - - def __dealloc__(self): - if self.context: - cl_release_context(self.context) - -cdef class CLMem: - @staticmethod - cdef create(void * cmem): - mem = CLMem() - mem.mem = cmem - return mem - - @property - def mem_address(self): - return (self.mem) - -def cl_from_visionbuf(VisionBuf buf): - return CLMem.create(&buf.buf.buf_cl) - - -cdef class ModelFrame: - cdef cppModelFrame * frame - cdef int buf_size - - def __dealloc__(self): - del self.frame - - def prepare(self, VisionBuf buf, float[:] projection): - cdef mat3 cprojection - memcpy(cprojection.v, &projection[0], 9*sizeof(float)) - cdef cl_mem * data - data = self.frame.prepare(buf.buf.buf_cl, buf.width, buf.height, buf.stride, buf.uv_offset, cprojection) - return CLMem.create(data) - - def buffer_from_cl(self, CLMem in_frames): - cdef unsigned char * data2 - data2 = self.frame.buffer_from_cl(in_frames.mem, self.buf_size) - return np.asarray( data2) - - -cdef class DrivingModelFrame(ModelFrame): - cdef cppDrivingModelFrame * _frame - - def __cinit__(self, CLContext context, int temporal_skip): - self._frame = new cppDrivingModelFrame(context.device_id, context.context, temporal_skip) - self.frame = (self._frame) - self.buf_size = self._frame.buf_size - -cdef class MonitoringModelFrame(ModelFrame): - cdef cppMonitoringModelFrame * _frame - - def __cinit__(self, CLContext context): - self._frame = new cppMonitoringModelFrame(context.device_id, context.context) - self.frame = (self._frame) - self.buf_size = self._frame.buf_size - From 273c729fd1a81e0e5ea831f09778df4f9f86485b Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 09:08:13 -0600 Subject: [PATCH 05/11] REMOVE LATER: Add modified msgq submodule --- .gitmodules | 3 ++- msgq_repo | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ad6530de9ac910..985ceaff162709 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,8 @@ url = ../../commaai/opendbc.git [submodule "msgq"] path = msgq_repo - url = ../../commaai/msgq.git + url = ../../mpurnell1/msgq.git + branch = remove-ipc-cython [submodule "rednose_repo"] path = rednose_repo url = ../../commaai/rednose.git diff --git a/msgq_repo b/msgq_repo index a16cf1f608538d..5a53ab8677cb41 160000 --- a/msgq_repo +++ b/msgq_repo @@ -1 +1 @@ -Subproject commit a16cf1f608538d14f66bd6142230d8728f2d0abc +Subproject commit 5a53ab8677cb41b296d5044e197e2a135c29f3cb From eeab03a23c5bf122db4710ee06e338b1eab5fcf3 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 04:46:36 -0600 Subject: [PATCH 06/11] update-cereal-import --- cereal/messaging/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cereal/messaging/__init__.py b/cereal/messaging/__init__.py index b03285f80a206d..e9b231e71332ee 100644 --- a/cereal/messaging/__init__.py +++ b/cereal/messaging/__init__.py @@ -1,7 +1,7 @@ # must be built with scons -from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ - set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event -from msgq.ipc_pyx import MultiplePublishersError, IpcError +from msgq import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ + set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event, \ + MultiplePublishersError, IpcError from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw import msgq From 88a4d109f6836855e4a61e3dc47a9e744076746b Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 09:14:08 -0600 Subject: [PATCH 07/11] REMOVE LATER: Add modified rednose submodule --- .gitmodules | 3 ++- rednose_repo | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 985ceaff162709..9aa33f77a1179a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,7 +10,8 @@ branch = remove-ipc-cython [submodule "rednose_repo"] path = rednose_repo - url = ../../commaai/rednose.git + url = ../../mpurnell1/rednose.git + branch = remove-ekf-cython [submodule "teleoprtc_repo"] path = teleoprtc_repo url = ../../commaai/teleoprtc diff --git a/rednose_repo b/rednose_repo index 7fddc8e6d49def..1a5d6e49c43b11 160000 --- a/rednose_repo +++ b/rednose_repo @@ -1 +1 @@ -Subproject commit 7fddc8e6d49def83c952a78673179bdc62789214 +Subproject commit 1a5d6e49c43b1182ff98bc2a2096258d77d5dd14 From 9be5f6a39b63c8a1d29bdd26f39984cd51a90f85 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 10:01:30 -0600 Subject: [PATCH 08/11] update-locationd-imports --- selfdrive/locationd/models/car_kf.py | 4 ++-- selfdrive/locationd/models/pose_kf.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/selfdrive/locationd/models/car_kf.py b/selfdrive/locationd/models/car_kf.py index 27cc4ef9c93fc5..0718e75f17d9ee 100755 --- a/selfdrive/locationd/models/car_kf.py +++ b/selfdrive/locationd/models/car_kf.py @@ -15,7 +15,7 @@ import sympy as sp from rednose.helpers.ekf_sym import gen_code else: - from rednose.helpers.ekf_sym_pyx import EKF_sym_pyx + from rednose.helpers import EKFSym i = 0 @@ -163,7 +163,7 @@ def generate_code(generated_dir): def __init__(self, generated_dir): dim_state, dim_state_err = CarKalman.initial_x.shape[0], CarKalman.P_initial.shape[0] - self.filter = EKF_sym_pyx(generated_dir, CarKalman.name, CarKalman.Q, CarKalman.initial_x, CarKalman.P_initial, + self.filter = EKFSym(generated_dir, CarKalman.name, CarKalman.Q, CarKalman.initial_x, CarKalman.P_initial, dim_state, dim_state_err, global_vars=CarKalman.global_vars, logger=cloudlog) def set_globals(self, mass, rotational_inertia, center_to_front, center_to_rear, stiffness_front, stiffness_rear): diff --git a/selfdrive/locationd/models/pose_kf.py b/selfdrive/locationd/models/pose_kf.py index 020e51ad6e50a4..ac4bf566eafef8 100755 --- a/selfdrive/locationd/models/pose_kf.py +++ b/selfdrive/locationd/models/pose_kf.py @@ -12,7 +12,7 @@ from rednose.helpers.ekf_sym import gen_code from rednose.helpers.sympy_helpers import euler_rotate, rot_to_euler else: - from rednose.helpers.ekf_sym_pyx import EKF_sym_pyx + from rednose.helpers import EKFSym EARTH_G = 9.81 @@ -102,7 +102,7 @@ def generate_code(generated_dir): def __init__(self, generated_dir, max_rewind_age): dim_state, dim_state_err = PoseKalman.initial_x.shape[0], PoseKalman.initial_P.shape[0] - self.filter = EKF_sym_pyx(generated_dir, self.name, PoseKalman.Q, PoseKalman.initial_x, PoseKalman.initial_P, + self.filter = EKFSym(generated_dir, self.name, PoseKalman.Q, PoseKalman.initial_x, PoseKalman.initial_P, dim_state, dim_state_err, max_rewind_age=max_rewind_age) From 9c69ca208d48269007a1eccac9660fc7fe796e2d Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 01:10:28 -0600 Subject: [PATCH 09/11] remove-acados-cython --- selfdrive/controls/lib/acados_setup.py | 63 ++ .../controls/lib/lateral_mpc_lib/SConscript | 22 +- .../controls/lib/lateral_mpc_lib/lat_mpc.py | 17 +- .../lib/longitudinal_mpc_lib/SConscript | 22 +- .../lib/longitudinal_mpc_lib/long_mpc.py | 16 +- selfdrive/controls/tests/test_acados_setup.py | 21 + .../acados_template/acados_ocp_solver_pyx.pyx | 795 ------------------ .../acados_sim_solver_common.pxd | 64 -- .../acados_template/acados_sim_solver_pyx.pyx | 256 ------ .../acados_template/acados_solver_common.pxd | 100 --- .../gnsf/check_reformulation.py | 2 +- 11 files changed, 107 insertions(+), 1271 deletions(-) create mode 100644 selfdrive/controls/lib/acados_setup.py create mode 100644 selfdrive/controls/tests/test_acados_setup.py delete mode 100644 third_party/acados/acados_template/acados_ocp_solver_pyx.pyx delete mode 100644 third_party/acados/acados_template/acados_sim_solver_common.pxd delete mode 100644 third_party/acados/acados_template/acados_sim_solver_pyx.pyx delete mode 100644 third_party/acados/acados_template/acados_solver_common.pxd diff --git a/selfdrive/controls/lib/acados_setup.py b/selfdrive/controls/lib/acados_setup.py new file mode 100644 index 00000000000000..236969a597956a --- /dev/null +++ b/selfdrive/controls/lib/acados_setup.py @@ -0,0 +1,63 @@ +import os +import sys +import ctypes +import platform + +def get_acados_dir(): + # current file: openpilot/selfdrive/controls/lib/acados_setup.py + # acados is at openpilot/third_party/acados + current_dir = os.path.dirname(os.path.abspath(__file__)) + op_dir = os.path.abspath(os.path.join(current_dir, '..', '..', '..')) + return os.path.join(op_dir, 'third_party', 'acados') + +def get_acados_lib_path(): + acados_dir = get_acados_dir() + if sys.platform.startswith('linux'): + machine = platform.machine() + if machine == 'x86_64': + return os.path.join(acados_dir, 'x86_64', 'lib') + elif machine == 'aarch64': + return os.path.join(acados_dir, 'larch64', 'lib') + elif sys.platform.startswith('darwin'): + return os.path.join(acados_dir, 'Darwin', 'lib') + + # Fallback + return os.path.join(acados_dir, 'x86_64', 'lib') + +def acados_preload(): + lib_path = get_acados_lib_path() + if not os.path.exists(lib_path): + return + + if sys.platform.startswith('linux'): + libs = ['libblasfeo.so', 'libhpipm.so', 'libqpOASES_e.so.3.1'] + mode = ctypes.RTLD_GLOBAL + elif sys.platform.startswith('darwin'): + libs = ['libblasfeo.dylib', 'libhpipm.dylib', 'libqpOASES_e.3.1.dylib'] + mode = ctypes.RTLD_GLOBAL + else: + libs = [] + mode = 0 + + for lib in libs: + full_path = os.path.join(lib_path, lib) + if os.path.exists(full_path): + try: + ctypes.CDLL(full_path, mode=mode) + except OSError: + pass + +def prepare_acados_ocp_json(json_file): + import json + import tempfile + + with open(json_file) as f: + data = json.load(f) + + data['acados_lib_path'] = get_acados_lib_path() + + fd, path = tempfile.mkstemp(suffix='.json', text=True) + with os.fdopen(fd, 'w') as f: + json.dump(data, f, indent=4) + + return path diff --git a/selfdrive/controls/lib/lateral_mpc_lib/SConscript b/selfdrive/controls/lib/lateral_mpc_lib/SConscript index c9ebf892079976..962756f20d4e5a 100644 --- a/selfdrive/controls/lib/lateral_mpc_lib/SConscript +++ b/selfdrive/controls/lib/lateral_mpc_lib/SConscript @@ -1,4 +1,4 @@ -Import('env', 'envCython', 'arch', 'msgq_python', 'common_python', 'np_version') +Import('env', 'arch', 'msgq_python', 'common_python', 'np_version') gen = "c_generated_code" @@ -75,23 +75,3 @@ else: lib_solver = lenv.SharedLibrary(f"{gen}/acados_ocp_solver_lat", build_files, LIBS=['m', 'acados', 'hpipm', 'blasfeo', 'qpOASES_e']) - -# generate cython stuff -acados_ocp_solver_pyx = File("#third_party/acados/acados_template/acados_ocp_solver_pyx.pyx") -acados_ocp_solver_common = File("#third_party/acados/acados_template/acados_solver_common.pxd") -libacados_ocp_solver_pxd = File(f'{gen}/acados_solver.pxd') -libacados_ocp_solver_c = File(f'{gen}/acados_ocp_solver_pyx.c') - -lenv2 = envCython.Clone() -lenv2["LIBPATH"] += [lib_solver[0].dir.abspath] -lenv2["RPATH"] += [lenv2.Literal('\\$$ORIGIN')] -lenv2.Command(libacados_ocp_solver_c, - [acados_ocp_solver_pyx, acados_ocp_solver_common, libacados_ocp_solver_pxd], - f'cython' + \ - f' -o {libacados_ocp_solver_c.get_labspath()}' + \ - f' -I {libacados_ocp_solver_pxd.get_dir().get_labspath()}' + \ - f' -I {acados_ocp_solver_common.get_dir().get_labspath()}' + \ - f' {acados_ocp_solver_pyx.get_labspath()}') -lib_cython = lenv2.Program(f'{gen}/acados_ocp_solver_pyx.so', [libacados_ocp_solver_c], LIBS=['acados_ocp_solver_lat']) -lenv2.Depends(lib_cython, lib_solver) -lenv2.Depends(libacados_ocp_solver_c, np_version) diff --git a/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py b/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py index ad60861088bec4..9006b1acddd112 100755 --- a/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py +++ b/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py @@ -7,10 +7,8 @@ # WARNING: imports outside of constants will not trigger a rebuild from openpilot.selfdrive.modeld.constants import ModelConstants -if __name__ == '__main__': # generating code - from openpilot.third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver -else: - from openpilot.selfdrive.controls.lib.lateral_mpc_lib.c_generated_code.acados_ocp_solver_pyx import AcadosOcpSolverCython +from openpilot.third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver +from openpilot.selfdrive.controls.lib.acados_setup import acados_preload, prepare_acados_ocp_json LAT_MPC_DIR = os.path.dirname(os.path.abspath(__file__)) EXPORT_DIR = os.path.join(LAT_MPC_DIR, "c_generated_code") @@ -132,7 +130,14 @@ class LateralMpc: def __init__(self, x0=None): if x0 is None: x0 = np.zeros(X_DIM) - self.solver = AcadosOcpSolverCython(MODEL_NAME, ACADOS_SOLVER_TYPE, N) + acados_preload() + # Fixup JSON for current architecture + json_path = prepare_acados_ocp_json(JSON_FILE) + try: + self.solver = AcadosOcpSolver(gen_lat_ocp(), json_file=json_path, generate=False, build=False) + finally: + if os.path.exists(json_path): + os.remove(json_path) self.reset(x0) def reset(self, x0=None): @@ -196,4 +201,4 @@ def run(self, x0, p, y_pts, heading_pts, yaw_rate_pts): if __name__ == "__main__": ocp = gen_lat_ocp() AcadosOcpSolver.generate(ocp, json_file=JSON_FILE) - # AcadosOcpSolver.build(ocp.code_export_directory, with_cython=True) + diff --git a/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript b/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript index 164b965142ce53..5d1f82b98eab0d 100644 --- a/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript +++ b/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript @@ -1,4 +1,4 @@ -Import('env', 'envCython', 'arch', 'msgq_python', 'common_python', 'pandad_python', 'np_version') +Import('env', 'arch', 'msgq_python', 'common_python', 'pandad_python', 'np_version') gen = "c_generated_code" @@ -80,23 +80,3 @@ else: lib_solver = lenv.SharedLibrary(f"{gen}/acados_ocp_solver_long", build_files, LIBS=['m', 'acados', 'hpipm', 'blasfeo', 'qpOASES_e']) - -# generate cython stuff -acados_ocp_solver_pyx = File("#third_party/acados/acados_template/acados_ocp_solver_pyx.pyx") -acados_ocp_solver_common = File("#third_party/acados/acados_template/acados_solver_common.pxd") -libacados_ocp_solver_pxd = File(f'{gen}/acados_solver.pxd') -libacados_ocp_solver_c = File(f'{gen}/acados_ocp_solver_pyx.c') - -lenv2 = envCython.Clone() -lenv2["LIBPATH"] += [lib_solver[0].dir.abspath] -lenv2["RPATH"] += [lenv2.Literal('\\$$ORIGIN')] -lenv2.Command(libacados_ocp_solver_c, - [acados_ocp_solver_pyx, acados_ocp_solver_common, libacados_ocp_solver_pxd], - f'cython' + \ - f' -o {libacados_ocp_solver_c.get_labspath()}' + \ - f' -I {libacados_ocp_solver_pxd.get_dir().get_labspath()}' + \ - f' -I {acados_ocp_solver_common.get_dir().get_labspath()}' + \ - f' {acados_ocp_solver_pyx.get_labspath()}') -lib_cython = lenv2.Program(f'{gen}/acados_ocp_solver_pyx.so', [libacados_ocp_solver_c], LIBS=['acados_ocp_solver_long']) -lenv2.Depends(lib_cython, lib_solver) -lenv2.Depends(libacados_ocp_solver_c, np_version) diff --git a/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py b/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py index 3f9d8245bd54ae..98d4e7c294a003 100755 --- a/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py +++ b/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py @@ -10,10 +10,8 @@ from openpilot.selfdrive.modeld.constants import index_function from openpilot.selfdrive.controls.radard import _LEAD_ACCEL_TAU -if __name__ == '__main__': # generating code - from openpilot.third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver -else: - from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.c_generated_code.acados_ocp_solver_pyx import AcadosOcpSolverCython +from openpilot.third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver +from openpilot.selfdrive.controls.lib.acados_setup import acados_preload, prepare_acados_ocp_json from casadi import SX, vertcat @@ -225,12 +223,17 @@ class LongitudinalMpc: def __init__(self, mode='acc', dt=DT_MDL): self.mode = mode self.dt = dt - self.solver = AcadosOcpSolverCython(MODEL_NAME, ACADOS_SOLVER_TYPE, N) + acados_preload() + json_path = prepare_acados_ocp_json(JSON_FILE) + try: + self.solver = AcadosOcpSolver(gen_long_ocp(), json_file=json_path, generate=False, build=False) + finally: + if os.path.exists(json_path): + os.remove(json_path) self.reset() self.source = SOURCES[2] def reset(self): - # self.solver = AcadosOcpSolverCython(MODEL_NAME, ACADOS_SOLVER_TYPE, N) self.solver.reset() # self.solver.options_set('print_level', 2) self.v_solution = np.zeros(N+1) @@ -454,4 +457,3 @@ def run(self): if __name__ == "__main__": ocp = gen_long_ocp() AcadosOcpSolver.generate(ocp, json_file=JSON_FILE) - # AcadosOcpSolver.build(ocp.code_export_directory, with_cython=True) diff --git a/selfdrive/controls/tests/test_acados_setup.py b/selfdrive/controls/tests/test_acados_setup.py new file mode 100644 index 00000000000000..4ae7dc9b5d6c3b --- /dev/null +++ b/selfdrive/controls/tests/test_acados_setup.py @@ -0,0 +1,21 @@ +import os +import sys +import platform +from openpilot.selfdrive.controls.lib.acados_setup import get_acados_lib_path, get_acados_dir + +def test_get_acados_lib_path_linux_x86_64(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(platform, 'machine', lambda: 'x86_64') + expected = os.path.join(get_acados_dir(), 'x86_64', 'lib') + assert get_acados_lib_path() == expected + +def test_get_acados_lib_path_linux_aarch64(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(platform, 'machine', lambda: 'aarch64') + expected = os.path.join(get_acados_dir(), 'larch64', 'lib') + assert get_acados_lib_path() == expected + +def test_get_acados_lib_path_darwin(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'darwin') + expected = os.path.join(get_acados_dir(), 'Darwin', 'lib') + assert get_acados_lib_path() == expected diff --git a/third_party/acados/acados_template/acados_ocp_solver_pyx.pyx b/third_party/acados/acados_template/acados_ocp_solver_pyx.pyx deleted file mode 100644 index acd7f02d0a81ad..00000000000000 --- a/third_party/acados/acados_template/acados_ocp_solver_pyx.pyx +++ /dev/null @@ -1,795 +0,0 @@ -# -*- coding: future_fstrings -*- -# -# Copyright (c) The acados authors. -# -# This file is part of acados. -# -# The 2-Clause BSD License -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE.; -# -# cython: language_level=3 -# cython: profile=False -# distutils: language=c - -cimport cython -from libc cimport string - -cimport acados_solver_common -# TODO: make this import more clear? it is not a general solver, but problem specific. -cimport acados_solver - -cimport numpy as cnp - -import os -from datetime import datetime -import numpy as np - - -cdef class AcadosOcpSolverCython: - """ - Class to interact with the acados ocp solver C object. - """ - - cdef acados_solver.nlp_solver_capsule *capsule - cdef void *nlp_opts - cdef acados_solver_common.ocp_nlp_dims *nlp_dims - cdef acados_solver_common.ocp_nlp_config *nlp_config - cdef acados_solver_common.ocp_nlp_out *nlp_out - cdef acados_solver_common.ocp_nlp_out *sens_out - cdef acados_solver_common.ocp_nlp_in *nlp_in - cdef acados_solver_common.ocp_nlp_solver *nlp_solver - - cdef bint solver_created - - cdef str model_name - cdef int N - - cdef str nlp_solver_type - - def __cinit__(self, model_name, nlp_solver_type, N): - - self.solver_created = False - - self.N = N - self.model_name = model_name - self.nlp_solver_type = nlp_solver_type - - # create capsule - self.capsule = acados_solver.acados_create_capsule() - - # create solver - assert acados_solver.acados_create(self.capsule) == 0 - self.solver_created = True - - # get pointers solver - self.__get_pointers_solver() - - - def __get_pointers_solver(self): - """ - Private function to get the pointers for solver - """ - # get pointers solver - self.nlp_opts = acados_solver.acados_get_nlp_opts(self.capsule) - self.nlp_dims = acados_solver.acados_get_nlp_dims(self.capsule) - self.nlp_config = acados_solver.acados_get_nlp_config(self.capsule) - self.nlp_out = acados_solver.acados_get_nlp_out(self.capsule) - self.sens_out = acados_solver.acados_get_sens_out(self.capsule) - self.nlp_in = acados_solver.acados_get_nlp_in(self.capsule) - self.nlp_solver = acados_solver.acados_get_nlp_solver(self.capsule) - - - def solve_for_x0(self, x0_bar): - """ - Wrapper around `solve()` which sets initial state constraint, solves the OCP, and returns u0. - """ - self.set(0, "lbx", x0_bar) - self.set(0, "ubx", x0_bar) - - status = self.solve() - - if status == 2: - print("Warning: acados_ocp_solver reached maximum iterations.") - elif status != 0: - raise Exception(f'acados acados_ocp_solver returned status {status}') - - u0 = self.get(0, "u") - return u0 - - - def solve(self): - """ - Solve the ocp with current input. - """ - return acados_solver.acados_solve(self.capsule) - - - def reset(self, reset_qp_solver_mem=1): - """ - Sets current iterate to all zeros. - """ - return acados_solver.acados_reset(self.capsule, reset_qp_solver_mem) - - - def custom_update(self, data_): - """ - A custom function that can be implemented by a user to be called between solver calls. - By default this does nothing. - The idea is to have a convenient wrapper to do complex updates of parameters and numerical data efficiently in C, - in a function that is compiled into the solver library and can be conveniently used in the Python environment. - """ - data_len = len(data_) - cdef cnp.ndarray[cnp.float64_t, ndim=1] data = np.ascontiguousarray(data_, dtype=np.float64) - - return acados_solver.acados_custom_update(self.capsule, data.data, data_len) - - - def set_new_time_steps(self, new_time_steps): - """ - Set new time steps. - Recreates the solver if N changes. - - :param new_time_steps: 1 dimensional np array of new time steps for the solver - - .. note:: This allows for different use-cases: either set a new size of time-steps or a new distribution of - the shooting nodes without changing the number, e.g., to reach a different final time. Both cases - do not require a new code export and compilation. - """ - - raise NotImplementedError("AcadosOcpSolverCython: does not support set_new_time_steps() since it is only a prototyping feature") - # # unlikely but still possible - # if not self.solver_created: - # raise Exception('Solver was not yet created!') - - # ## check if time steps really changed in value - # # get time steps - # cdef cnp.ndarray[cnp.float64_t, ndim=1] old_time_steps = np.ascontiguousarray(np.zeros((self.N,)), dtype=np.float64) - # assert acados_solver.acados_get_time_steps(self.capsule, self.N, old_time_steps.data) - - # if np.array_equal(old_time_steps, new_time_steps): - # return - - # N = new_time_steps.size - # cdef cnp.ndarray[cnp.float64_t, ndim=1] value = np.ascontiguousarray(new_time_steps, dtype=np.float64) - - # # check if recreation of acados is necessary (no need to recreate acados if sizes are identical) - # if len(old_time_steps) == N: - # assert acados_solver.acados_update_time_steps(self.capsule, N, value.data) == 0 - - # else: # recreate the solver with the new time steps - # self.solver_created = False - - # # delete old memory (analog to __del__) - # acados_solver.acados_free(self.capsule) - - # # create solver with new time steps - # assert acados_solver.acados_create_with_discretization(self.capsule, N, value.data) == 0 - - # self.solver_created = True - - # # get pointers solver - # self.__get_pointers_solver() - - # # store time_steps, N - # self.time_steps = new_time_steps - # self.N = N - - - def update_qp_solver_cond_N(self, qp_solver_cond_N: int): - """ - Recreate solver with new value `qp_solver_cond_N` with a partial condensing QP solver. - This function is relevant for code reuse, i.e., if either `set_new_time_steps(...)` is used or - the influence of a different `qp_solver_cond_N` is studied without code export and compilation. - :param qp_solver_cond_N: new number of condensing stages for the solver - - .. note:: This function can only be used in combination with a partial condensing QP solver. - - .. note:: After `set_new_time_steps(...)` is used and depending on the new number of time steps it might be - necessary to change `qp_solver_cond_N` as well (using this function), i.e., typically - `qp_solver_cond_N < N`. - """ - raise NotImplementedError("AcadosOcpSolverCython: does not support update_qp_solver_cond_N() since it is only a prototyping feature") - - # # unlikely but still possible - # if not self.solver_created: - # raise Exception('Solver was not yet created!') - # if self.N < qp_solver_cond_N: - # raise Exception('Setting qp_solver_cond_N to be larger than N does not work!') - # if self.qp_solver_cond_N != qp_solver_cond_N: - # self.solver_created = False - - # # recreate the solver - # acados_solver.acados_update_qp_solver_cond_N(self.capsule, qp_solver_cond_N) - - # # store the new value - # self.qp_solver_cond_N = qp_solver_cond_N - # self.solver_created = True - - # # get pointers solver - # self.__get_pointers_solver() - - - def eval_param_sens(self, index, stage=0, field="ex"): - """ - Calculate the sensitivity of the curent solution with respect to the initial state component of index - - :param index: integer corresponding to initial state index in range(nx) - """ - - field_ = field - field = field_.encode('utf-8') - - # checks - if not isinstance(index, int): - raise Exception('AcadosOcpSolverCython.eval_param_sens(): index must be Integer.') - - cdef int nx = acados_solver_common.ocp_nlp_dims_get_from_attr(self.nlp_config, self.nlp_dims, self.nlp_out, 0, "x".encode('utf-8')) - - if index < 0 or index > nx: - raise Exception(f'AcadosOcpSolverCython.eval_param_sens(): index must be in [0, nx-1], got: {index}.') - - # actual eval_param - acados_solver_common.ocp_nlp_eval_param_sens(self.nlp_solver, field, stage, index, self.sens_out) - - return - - - def get(self, int stage, str field_): - """ - Get the last solution of the solver: - - :param stage: integer corresponding to shooting node - :param field: string in ['x', 'u', 'z', 'pi', 'lam', 't', 'sl', 'su',] - - .. note:: regarding lam, t: \n - the inequalities are internally organized in the following order: \n - [ lbu lbx lg lh lphi ubu ubx ug uh uphi; \n - lsbu lsbx lsg lsh lsphi usbu usbx usg ush usphi] - - .. note:: pi: multipliers for dynamics equality constraints \n - lam: multipliers for inequalities \n - t: slack variables corresponding to evaluation of all inequalities (at the solution) \n - sl: slack variables of soft lower inequality constraints \n - su: slack variables of soft upper inequality constraints \n - """ - - out_fields = ['x', 'u', 'z', 'pi', 'lam', 't', 'sl', 'su'] - field = field_.encode('utf-8') - - if field_ not in out_fields: - raise Exception('AcadosOcpSolverCython.get(): {} is an invalid argument.\ - \n Possible values are {}.'.format(field_, out_fields)) - - if stage < 0 or stage > self.N: - raise Exception('AcadosOcpSolverCython.get(): stage index must be in [0, N], got: {}.'.format(self.N)) - - if stage == self.N and field_ == 'pi': - raise Exception('AcadosOcpSolverCython.get(): field {} does not exist at final stage {}.'\ - .format(field_, stage)) - - cdef int dims = acados_solver_common.ocp_nlp_dims_get_from_attr(self.nlp_config, - self.nlp_dims, self.nlp_out, stage, field) - - cdef cnp.ndarray[cnp.float64_t, ndim=1] out = np.zeros((dims,)) - acados_solver_common.ocp_nlp_out_get(self.nlp_config, \ - self.nlp_dims, self.nlp_out, stage, field, out.data) - - return out - - - def print_statistics(self): - """ - prints statistics of previous solver run as a table: - - iter: iteration number - - res_stat: stationarity residual - - res_eq: residual wrt equality constraints (dynamics) - - res_ineq: residual wrt inequality constraints (constraints) - - res_comp: residual wrt complementarity conditions - - qp_stat: status of QP solver - - qp_iter: number of QP iterations - - qp_res_stat: stationarity residual of the last QP solution - - qp_res_eq: residual wrt equality constraints (dynamics) of the last QP solution - - qp_res_ineq: residual wrt inequality constraints (constraints) of the last QP solution - - qp_res_comp: residual wrt complementarity conditions of the last QP solution - """ - acados_solver.acados_print_stats(self.capsule) - - - def store_iterate(self, filename='', overwrite=False): - """ - Stores the current iterate of the ocp solver in a json file. - - :param filename: if not set, use model_name + timestamp + '.json' - :param overwrite: if false and filename exists add timestamp to filename - """ - import json - if filename == '': - filename += self.model_name + '_' + 'iterate' + '.json' - - if not overwrite: - # append timestamp - if os.path.isfile(filename): - filename = filename[:-5] - filename += datetime.utcnow().strftime('%Y-%m-%d-%H:%M:%S.%f') + '.json' - - # get iterate: - solution = dict() - - lN = len(str(self.N+1)) - for i in range(self.N+1): - i_string = f'{i:0{lN}d}' - solution['x_'+i_string] = self.get(i,'x') - solution['u_'+i_string] = self.get(i,'u') - solution['z_'+i_string] = self.get(i,'z') - solution['lam_'+i_string] = self.get(i,'lam') - solution['t_'+i_string] = self.get(i, 't') - solution['sl_'+i_string] = self.get(i, 'sl') - solution['su_'+i_string] = self.get(i, 'su') - if i < self.N: - solution['pi_'+i_string] = self.get(i,'pi') - - for k in list(solution.keys()): - if len(solution[k]) == 0: - del solution[k] - - # save - with open(filename, 'w') as f: - json.dump(solution, f, default=lambda x: x.tolist(), indent=4, sort_keys=True) - print("stored current iterate in ", os.path.join(os.getcwd(), filename)) - - - def load_iterate(self, filename): - """ - Loads the iterate stored in json file with filename into the ocp solver. - """ - import json - if not os.path.isfile(filename): - raise Exception('load_iterate: failed, file does not exist: ' + os.path.join(os.getcwd(), filename)) - - with open(filename, 'r') as f: - solution = json.load(f) - - for key in solution.keys(): - (field, stage) = key.split('_') - self.set(int(stage), field, np.array(solution[key])) - - - def get_stats(self, field_): - """ - Get the information of the last solver call. - - :param field: string in ['statistics', 'time_tot', 'time_lin', 'time_sim', 'time_sim_ad', 'time_sim_la', 'time_qp', 'time_qp_solver_call', 'time_reg', 'sqp_iter'] - Available fileds: - - time_tot: total CPU time previous call - - time_lin: CPU time for linearization - - time_sim: CPU time for integrator - - time_sim_ad: CPU time for integrator contribution of external function calls - - time_sim_la: CPU time for integrator contribution of linear algebra - - time_qp: CPU time qp solution - - time_qp_solver_call: CPU time inside qp solver (without converting the QP) - - time_qp_xcond: time_glob: CPU time globalization - - time_solution_sensitivities: CPU time for previous call to eval_param_sens - - time_reg: CPU time regularization - - sqp_iter: number of SQP iterations - - qp_iter: vector of QP iterations for last SQP call - - statistics: table with info about last iteration - - stat_m: number of rows in statistics matrix - - stat_n: number of columns in statistics matrix - - residuals: residuals of last iterate - - alpha: step sizes of SQP iterations - """ - - double_fields = ['time_tot', - 'time_lin', - 'time_sim', - 'time_sim_ad', - 'time_sim_la', - 'time_qp', - 'time_qp_solver_call', - 'time_qp_xcond', - 'time_glob', - 'time_solution_sensitivities', - 'time_reg' - ] - fields = double_fields + [ - 'sqp_iter', - 'qp_iter', - 'statistics', - 'stat_m', - 'stat_n', - 'residuals', - 'alpha', - ] - field = field_.encode('utf-8') - - if field_ in ['sqp_iter', 'stat_m', 'stat_n']: - return self.__get_stat_int(field) - - elif field_ in double_fields: - return self.__get_stat_double(field) - - elif field_ == 'statistics': - sqp_iter = self.get_stats("sqp_iter") - stat_m = self.get_stats("stat_m") - stat_n = self.get_stats("stat_n") - min_size = min([stat_m, sqp_iter+1]) - return self.__get_stat_matrix(field, stat_n+1, min_size) - - elif field_ == 'qp_iter': - full_stats = self.get_stats('statistics') - if self.nlp_solver_type == 'SQP': - return full_stats[6, :] - elif self.nlp_solver_type == 'SQP_RTI': - return full_stats[2, :] - - elif field_ == 'alpha': - full_stats = self.get_stats('statistics') - if self.nlp_solver_type == 'SQP': - return full_stats[7, :] - else: # self.nlp_solver_type == 'SQP_RTI': - raise Exception("alpha values are not available for SQP_RTI") - - elif field_ == 'residuals': - return self.get_residuals() - - else: - raise NotImplementedError("TODO!") - - - def __get_stat_int(self, field): - cdef int out - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, &out) - return out - - def __get_stat_double(self, field): - cdef cnp.ndarray[cnp.float64_t, ndim=1] out = np.zeros((1,)) - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, out.data) - return out - - def __get_stat_matrix(self, field, n, m): - cdef cnp.ndarray[cnp.float64_t, ndim=2] out_mat = np.ascontiguousarray(np.zeros((n, m)), dtype=np.float64) - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, out_mat.data) - return out_mat - - - def get_cost(self): - """ - Returns the cost value of the current solution. - """ - # compute cost internally - acados_solver_common.ocp_nlp_eval_cost(self.nlp_solver, self.nlp_in, self.nlp_out) - - # create output - cdef double out - - # call getter - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, "cost_value", &out) - - return out - - - def get_residuals(self, recompute=False): - """ - Returns an array of the form [res_stat, res_eq, res_ineq, res_comp]. - """ - # compute residuals if RTI - if self.nlp_solver_type == 'SQP_RTI' or recompute: - acados_solver_common.ocp_nlp_eval_residuals(self.nlp_solver, self.nlp_in, self.nlp_out) - - # create output array - cdef cnp.ndarray[cnp.float64_t, ndim=1] out = np.ascontiguousarray(np.zeros((4,), dtype=np.float64)) - cdef double double_value - - field = "res_stat".encode('utf-8') - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, &double_value) - out[0] = double_value - - field = "res_eq".encode('utf-8') - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, &double_value) - out[1] = double_value - - field = "res_ineq".encode('utf-8') - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, &double_value) - out[2] = double_value - - field = "res_comp".encode('utf-8') - acados_solver_common.ocp_nlp_get(self.nlp_config, self.nlp_solver, field, &double_value) - out[3] = double_value - - return out - - - # Note: this function should not be used anymore, better use cost_set, constraints_set - def set(self, int stage, str field_, value_): - - """ - Set numerical data inside the solver. - - :param stage: integer corresponding to shooting node - :param field: string in ['x', 'u', 'pi', 'lam', 't', 'p'] - - .. note:: regarding lam, t: \n - the inequalities are internally organized in the following order: \n - [ lbu lbx lg lh lphi ubu ubx ug uh uphi; \n - lsbu lsbx lsg lsh lsphi usbu usbx usg ush usphi] - - .. note:: pi: multipliers for dynamics equality constraints \n - lam: multipliers for inequalities \n - t: slack variables corresponding to evaluation of all inequalities (at the solution) \n - sl: slack variables of soft lower inequality constraints \n - su: slack variables of soft upper inequality constraints \n - """ - if not isinstance(value_, np.ndarray): - raise Exception(f"set: value must be numpy array, got {type(value_)}.") - cost_fields = ['y_ref', 'yref'] - constraints_fields = ['lbx', 'ubx', 'lbu', 'ubu'] - out_fields = ['x', 'u', 'pi', 'lam', 't', 'z', 'sl', 'su'] - mem_fields = ['xdot_guess', 'z_guess'] - - field = field_.encode('utf-8') - - cdef cnp.ndarray[cnp.float64_t, ndim=1] value = np.ascontiguousarray(value_, dtype=np.float64) - - # treat parameters separately - if field_ == 'p': - assert acados_solver.acados_update_params(self.capsule, stage, value.data, value.shape[0]) == 0 - else: - if field_ not in constraints_fields + cost_fields + out_fields: - raise Exception("AcadosOcpSolverCython.set(): {} is not a valid argument.\ - \nPossible values are {}.".format(field, \ - constraints_fields + cost_fields + out_fields + ['p'])) - - dims = acados_solver_common.ocp_nlp_dims_get_from_attr(self.nlp_config, - self.nlp_dims, self.nlp_out, stage, field) - - if value_.shape[0] != dims: - msg = 'AcadosOcpSolverCython.set(): mismatching dimension for field "{}" '.format(field_) - msg += 'with dimension {} (you have {})'.format(dims, value_.shape[0]) - raise Exception(msg) - - if field_ in constraints_fields: - acados_solver_common.ocp_nlp_constraints_model_set(self.nlp_config, - self.nlp_dims, self.nlp_in, stage, field, value.data) - elif field_ in cost_fields: - acados_solver_common.ocp_nlp_cost_model_set(self.nlp_config, - self.nlp_dims, self.nlp_in, stage, field, value.data) - elif field_ in out_fields: - acados_solver_common.ocp_nlp_out_set(self.nlp_config, - self.nlp_dims, self.nlp_out, stage, field, value.data) - elif field_ in mem_fields: - acados_solver_common.ocp_nlp_set(self.nlp_config, \ - self.nlp_solver, stage, field, value.data) - - if field_ == 'z': - field = 'z_guess'.encode('utf-8') - acados_solver_common.ocp_nlp_set(self.nlp_config, \ - self.nlp_solver, stage, field, value.data) - return - - def cost_set(self, int stage, str field_, value_): - """ - Set numerical data in the cost module of the solver. - - :param stage: integer corresponding to shooting node - :param field: string, e.g. 'yref', 'W', 'ext_cost_num_hess' - :param value: of appropriate size - """ - if not isinstance(value_, np.ndarray): - raise Exception(f"cost_set: value must be numpy array, got {type(value_)}.") - field = field_.encode('utf-8') - - cdef int dims[2] - acados_solver_common.ocp_nlp_cost_dims_get_from_attr(self.nlp_config, \ - self.nlp_dims, self.nlp_out, stage, field, &dims[0]) - - cdef double[::1,:] value - - value_shape = value_.shape - if len(value_shape) == 1: - value_shape = (value_shape[0], 0) - value = np.asfortranarray(value_[None,:]) - - elif len(value_shape) == 2: - # Get elements in column major order - value = np.asfortranarray(value_) - - if value_shape[0] != dims[0] or value_shape[1] != dims[1]: - raise Exception('AcadosOcpSolverCython.cost_set(): mismatching dimension' + - f' for field "{field_}" at stage {stage} with dimension {tuple(dims)} (you have {value_shape})') - - acados_solver_common.ocp_nlp_cost_model_set(self.nlp_config, \ - self.nlp_dims, self.nlp_in, stage, field, &value[0][0]) - - - def constraints_set(self, int stage, str field_, value_): - """ - Set numerical data in the constraint module of the solver. - - :param stage: integer corresponding to shooting node - :param field: string in ['lbx', 'ubx', 'lbu', 'ubu', 'lg', 'ug', 'lh', 'uh', 'uphi', 'C', 'D'] - :param value: of appropriate size - """ - if not isinstance(value_, np.ndarray): - raise Exception(f"constraints_set: value must be numpy array, got {type(value_)}.") - - field = field_.encode('utf-8') - - cdef int dims[2] - acados_solver_common.ocp_nlp_constraint_dims_get_from_attr(self.nlp_config, \ - self.nlp_dims, self.nlp_out, stage, field, &dims[0]) - - cdef double[::1,:] value - - value_shape = value_.shape - if len(value_shape) == 1: - value_shape = (value_shape[0], 0) - value = np.asfortranarray(value_[None,:]) - - elif len(value_shape) == 2: - # Get elements in column major order - value = np.asfortranarray(value_) - - if value_shape != tuple(dims): - raise Exception(f'AcadosOcpSolverCython.constraints_set(): mismatching dimension' + - f' for field "{field_}" at stage {stage} with dimension {tuple(dims)} (you have {value_shape})') - - acados_solver_common.ocp_nlp_constraints_model_set(self.nlp_config, \ - self.nlp_dims, self.nlp_in, stage, field, &value[0][0]) - - return - - - def get_from_qp_in(self, int stage, str field_): - """ - Get numerical data from the dynamics module of the solver: - - :param stage: integer corresponding to shooting node - :param field: string, e.g. 'A' - """ - field = field_.encode('utf-8') - - # get dims - cdef int[2] dims - acados_solver_common.ocp_nlp_qp_dims_get_from_attr(self.nlp_config, self.nlp_dims, self.nlp_out, stage, field, &dims[0]) - - # create output data - cdef cnp.ndarray[cnp.float64_t, ndim=2] out = np.zeros((dims[0], dims[1]), order='F') - - # call getter - acados_solver_common.ocp_nlp_get_at_stage(self.nlp_config, self.nlp_dims, self.nlp_solver, stage, field, out.data) - - return out - - - def options_set(self, str field_, value_): - """ - Set options of the solver. - - :param field: string, e.g. 'print_level', 'rti_phase', 'initialize_t_slacks', 'step_length', 'alpha_min', 'alpha_reduction', 'qp_warm_start', 'line_search_use_sufficient_descent', 'full_step_dual', 'globalization_use_SOC', 'qp_tol_stat', 'qp_tol_eq', 'qp_tol_ineq', 'qp_tol_comp', 'qp_tau_min', 'qp_mu0' - - :param value: of type int, float, string - - - qp_tol_stat: QP solver tolerance stationarity - - qp_tol_eq: QP solver tolerance equalities - - qp_tol_ineq: QP solver tolerance inequalities - - qp_tol_comp: QP solver tolerance complementarity - - qp_tau_min: for HPIPM QP solvers: minimum value of barrier parameter in HPIPM - - qp_mu0: for HPIPM QP solvers: initial value for complementarity slackness - - warm_start_first_qp: indicates if first QP in SQP is warm_started - """ - int_fields = ['print_level', 'rti_phase', 'initialize_t_slacks', 'qp_warm_start', 'line_search_use_sufficient_descent', 'full_step_dual', 'globalization_use_SOC', 'warm_start_first_qp'] - double_fields = ['step_length', 'tol_eq', 'tol_stat', 'tol_ineq', 'tol_comp', 'alpha_min', 'alpha_reduction', 'eps_sufficient_descent', - 'qp_tol_stat', 'qp_tol_eq', 'qp_tol_ineq', 'qp_tol_comp', 'qp_tau_min', 'qp_mu0'] - string_fields = ['globalization'] - - # encode - field = field_.encode('utf-8') - - cdef int int_value - cdef double double_value - cdef unsigned char[::1] string_value - - # check field availability and type - if field_ in int_fields: - if not isinstance(value_, int): - raise Exception('solver option {} must be of type int. You have {}.'.format(field_, type(value_))) - - if field_ == 'rti_phase': - if value_ < 0 or value_ > 2: - raise Exception('AcadosOcpSolverCython.solve(): argument \'rti_phase\' can ' - 'take only values 0, 1, 2 for SQP-RTI-type solvers') - if self.nlp_solver_type != 'SQP_RTI' and value_ > 0: - raise Exception('AcadosOcpSolverCython.solve(): argument \'rti_phase\' can ' - 'take only value 0 for SQP-type solvers') - - int_value = value_ - acados_solver_common.ocp_nlp_solver_opts_set(self.nlp_config, self.nlp_opts, field, &int_value) - - elif field_ in double_fields: - if not isinstance(value_, float): - raise Exception('solver option {} must be of type float. You have {}.'.format(field_, type(value_))) - - double_value = value_ - acados_solver_common.ocp_nlp_solver_opts_set(self.nlp_config, self.nlp_opts, field, &double_value) - - elif field_ in string_fields: - if not isinstance(value_, bytes): - raise Exception('solver option {} must be of type str. You have {}.'.format(field_, type(value_))) - - string_value = value_.encode('utf-8') - acados_solver_common.ocp_nlp_solver_opts_set(self.nlp_config, self.nlp_opts, field, &string_value[0]) - - else: - raise Exception('AcadosOcpSolverCython.options_set() does not support field {}.'\ - '\n Possible values are {}.'.format(field_, ', '.join(int_fields + double_fields + string_fields))) - - - def set_params_sparse(self, int stage, idx_values_, param_values_): - """ - set parameters of the solvers external function partially: - Pseudo: solver.param[idx_values_] = param_values_; - Parameters: - :param stage_: integer corresponding to shooting node - :param idx_values_: 0 based integer array corresponding to parameter indices to be set - :param param_values_: new parameter values as numpy array - """ - - if not isinstance(param_values_, np.ndarray): - raise Exception('param_values_ must be np.array.') - - if param_values_.shape[0] != len(idx_values_): - raise Exception(f'param_values_ and idx_values_ must be of the same size.' + - f' Got sizes idx {param_values_.shape[0]}, param_values {len(idx_values_)}.') - - # n_update = c_int(len(param_values_)) - - # param_data = cast(param_values_.ctypes.data, POINTER(c_double)) - # c_idx_values = np.ascontiguousarray(idx_values_, dtype=np.intc) - # idx_data = cast(c_idx_values.ctypes.data, POINTER(c_int)) - - # getattr(self.shared_lib, f"{self.model_name}_acados_update_params_sparse").argtypes = \ - # [c_void_p, c_int, POINTER(c_int), POINTER(c_double), c_int] - # getattr(self.shared_lib, f"{self.model_name}_acados_update_params_sparse").restype = c_int - # getattr(self.shared_lib, f"{self.model_name}_acados_update_params_sparse") \ - # (self.capsule, stage, idx_data, param_data, n_update) - - cdef cnp.ndarray[cnp.float64_t, ndim=1] value = np.ascontiguousarray(param_values_, dtype=np.float64) - # cdef cnp.ndarray[cnp.intc, ndim=1] idx = np.ascontiguousarray(idx_values_, dtype=np.intc) - - # NOTE: this does throw an error somehow: - # ValueError: Buffer dtype mismatch, expected 'int object' but got 'int' - # cdef cnp.ndarray[cnp.int, ndim=1] idx = np.ascontiguousarray(idx_values_, dtype=np.intc) - - cdef cnp.ndarray[cnp.int32_t, ndim=1] idx = np.ascontiguousarray(idx_values_, dtype=np.int32) - cdef int n_update = value.shape[0] - # print(f"in set_params_sparse Cython n_update {n_update}") - - assert acados_solver.acados_update_params_sparse(self.capsule, stage, idx.data, value.data, n_update) == 0 - return - - - def __del__(self): - if self.solver_created: - acados_solver.acados_free(self.capsule) - acados_solver.acados_free_capsule(self.capsule) diff --git a/third_party/acados/acados_template/acados_sim_solver_common.pxd b/third_party/acados/acados_template/acados_sim_solver_common.pxd deleted file mode 100644 index cc6a58efd77bb4..00000000000000 --- a/third_party/acados/acados_template/acados_sim_solver_common.pxd +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: future_fstrings -*- -# -# Copyright (c) The acados authors. -# -# This file is part of acados. -# -# The 2-Clause BSD License -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE.; -# - - -cdef extern from "acados/sim/sim_common.h": - ctypedef struct sim_config: - pass - - ctypedef struct sim_opts: - pass - - ctypedef struct sim_in: - pass - - ctypedef struct sim_out: - pass - - -cdef extern from "acados_c/sim_interface.h": - - ctypedef struct sim_plan: - pass - - ctypedef struct sim_solver: - pass - - # out - void sim_out_get(sim_config *config, void *dims, sim_out *out, const char *field, void *value) - int sim_dims_get_from_attr(sim_config *config, void *dims, const char *field, void *dims_data) - - # opts - void sim_opts_set(sim_config *config, void *opts_, const char *field, void *value) - - # get/set - void sim_in_set(sim_config *config, void *dims, sim_in *sim_in, const char *field, void *value) - void sim_solver_set(sim_solver *solver, const char *field, void *value) \ No newline at end of file diff --git a/third_party/acados/acados_template/acados_sim_solver_pyx.pyx b/third_party/acados/acados_template/acados_sim_solver_pyx.pyx deleted file mode 100644 index be400addc7dd3a..00000000000000 --- a/third_party/acados/acados_template/acados_sim_solver_pyx.pyx +++ /dev/null @@ -1,256 +0,0 @@ -# -*- coding: future_fstrings -*- -# -# Copyright (c) The acados authors. -# -# This file is part of acados. -# -# The 2-Clause BSD License -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE.; -# -# cython: language_level=3 -# cython: profile=False -# distutils: language=c - -cimport cython -from libc cimport string -# from libc cimport bool as bool_t - -cimport acados_sim_solver_common -cimport acados_sim_solver - -cimport numpy as cnp - -import os -from datetime import datetime -import numpy as np - - -cdef class AcadosSimSolverCython: - """ - Class to interact with the acados sim solver C object. - """ - - cdef acados_sim_solver.sim_solver_capsule *capsule - cdef void *sim_dims - cdef acados_sim_solver_common.sim_opts *sim_opts - cdef acados_sim_solver_common.sim_config *sim_config - cdef acados_sim_solver_common.sim_out *sim_out - cdef acados_sim_solver_common.sim_in *sim_in - cdef acados_sim_solver_common.sim_solver *sim_solver - - cdef bint solver_created - - cdef str model_name - - cdef str sim_solver_type - - cdef list gettable_vectors - cdef list gettable_matrices - cdef list gettable_scalars - - def __cinit__(self, model_name): - - self.solver_created = False - - self.model_name = model_name - - # create capsule - self.capsule = acados_sim_solver.acados_sim_solver_create_capsule() - - # create solver - assert acados_sim_solver.acados_sim_create(self.capsule) == 0 - self.solver_created = True - - # get pointers solver - self.__get_pointers_solver() - - self.gettable_vectors = ['x', 'u', 'z', 'S_adj'] - self.gettable_matrices = ['S_forw', 'Sx', 'Su', 'S_hess', 'S_algebraic'] - self.gettable_scalars = ['CPUtime', 'time_tot', 'ADtime', 'time_ad', 'LAtime', 'time_la'] - - def __get_pointers_solver(self): - """ - Private function to get the pointers for solver - """ - # get pointers solver - self.sim_opts = acados_sim_solver.acados_get_sim_opts(self.capsule) - self.sim_dims = acados_sim_solver.acados_get_sim_dims(self.capsule) - self.sim_config = acados_sim_solver.acados_get_sim_config(self.capsule) - self.sim_out = acados_sim_solver.acados_get_sim_out(self.capsule) - self.sim_in = acados_sim_solver.acados_get_sim_in(self.capsule) - self.sim_solver = acados_sim_solver.acados_get_sim_solver(self.capsule) - - - def simulate(self, x=None, u=None, z=None, p=None): - """ - Simulate the system forward for the given x, u, z, p and return x_next. - Wrapper around `solve()` taking care of setting/getting inputs/outputs. - """ - if x is not None: - self.set('x', x) - if u is not None: - self.set('u', u) - if z is not None: - self.set('z', z) - if p is not None: - self.set('p', p) - - status = self.solve() - - if status == 2: - print("Warning: acados_sim_solver reached maximum iterations.") - elif status != 0: - raise Exception(f'acados_sim_solver for model {self.model_name} returned status {status}.') - - x_next = self.get('x') - return x_next - - - def solve(self): - """ - Solve the sim with current input. - """ - return acados_sim_solver.acados_sim_solve(self.capsule) - - - def get(self, field_): - """ - Get the last solution of the solver. - - :param str field: string in ['x', 'u', 'z', 'S_forw', 'Sx', 'Su', 'S_adj', 'S_hess', 'S_algebraic', 'CPUtime', 'time_tot', 'ADtime', 'time_ad', 'LAtime', 'time_la'] - """ - field = field_.encode('utf-8') - - if field_ in self.gettable_vectors: - return self.__get_vector(field) - elif field_ in self.gettable_matrices: - return self.__get_matrix(field) - elif field_ in self.gettable_scalars: - return self.__get_scalar(field) - else: - raise Exception(f'AcadosSimSolver.get(): Unknown field {field_},' \ - f' available fields are {", ".join(self.gettable.keys())}') - - - def __get_scalar(self, field): - cdef double scalar - acados_sim_solver_common.sim_out_get(self.sim_config, self.sim_dims, self.sim_out, field, &scalar) - return scalar - - - def __get_vector(self, field): - cdef int[2] dims - acados_sim_solver_common.sim_dims_get_from_attr(self.sim_config, self.sim_dims, field, &dims[0]) - # cdef cnp.ndarray[cnp.float64_t, ndim=1] out = np.ascontiguousarray(np.zeros((dims[0],), dtype=np.float64)) - cdef cnp.ndarray[cnp.float64_t, ndim=1] out = np.zeros((dims[0]),) - acados_sim_solver_common.sim_out_get(self.sim_config, self.sim_dims, self.sim_out, field, out.data) - return out - - - def __get_matrix(self, field): - cdef int[2] dims - acados_sim_solver_common.sim_dims_get_from_attr(self.sim_config, self.sim_dims, field, &dims[0]) - cdef cnp.ndarray[cnp.float64_t, ndim=2] out = np.zeros((dims[0], dims[1]), order='F', dtype=np.float64) - acados_sim_solver_common.sim_out_get(self.sim_config, self.sim_dims, self.sim_out, field, out.data) - return out - - - def set(self, field_: str, value_): - """ - Set numerical data inside the solver. - - :param field: string in ['p', 'seed_adj', 'T', 'x', 'u', 'xdot', 'z'] - :param value: the value with appropriate size. - """ - settable = ['seed_adj', 'T', 'x', 'u', 'xdot', 'z', 'p'] # S_forw - - # cast value_ to avoid conversion issues - if isinstance(value_, (float, int)): - value_ = np.array([value_]) - # if len(value_.shape) > 1: - # raise RuntimeError('AcadosSimSolverCython.set(): value_ should be 1 dimensional') - - cdef cnp.ndarray[cnp.float64_t, ndim=1] value = np.ascontiguousarray(value_, dtype=np.float64).flatten() - - field = field_.encode('utf-8') - cdef int[2] dims - - # treat parameters separately - if field_ == 'p': - assert acados_sim_solver.acados_sim_update_params(self.capsule, value.data, value.shape[0]) == 0 - return - else: - acados_sim_solver_common.sim_dims_get_from_attr(self.sim_config, self.sim_dims, field, &dims[0]) - - value_ = np.ravel(value_, order='F') - - value_shape = value_.shape - if len(value_shape) == 1: - value_shape = (value_shape[0], 0) - - if value_shape != tuple(dims): - raise Exception(f'AcadosSimSolverCython.set(): mismatching dimension' \ - f' for field "{field_}" with dimension {tuple(dims)} (you have {value_shape}).') - - # set - if field_ in ['xdot', 'z']: - acados_sim_solver_common.sim_solver_set(self.sim_solver, field, value.data) - elif field_ in settable: - acados_sim_solver_common.sim_in_set(self.sim_config, self.sim_dims, self.sim_in, field, value.data) - else: - raise Exception(f'AcadosSimSolverCython.set(): Unknown field {field_},' \ - f' available fields are {", ".join(settable)}') - - - def options_set(self, field_: str, value_: bool): - """ - Set solver options - - :param field: string in ['sens_forw', 'sens_adj', 'sens_hess'] - :param value: Boolean - """ - fields = ['sens_forw', 'sens_adj', 'sens_hess'] - if field_ not in fields: - raise Exception(f"field {field_} not supported. Supported values are {', '.join(fields)}.\n") - - field = field_.encode('utf-8') - - if not isinstance(value_, bool): - raise TypeError("options_set: expected boolean for value") - - cdef bint bool_value = value_ - acados_sim_solver_common.sim_opts_set(self.sim_config, self.sim_opts, field, &bool_value) - # TODO: only allow setting - # if getattr(self.acados_sim.solver_options, field_) or value_ == False: - # acados_sim_solver_common.sim_opts_set(self.sim_config, self.sim_opts, field, &bool_value) - # else: - # raise RuntimeError(f"Cannot set option {field_} to True, because it was False in original solver options.\n") - - return - - - def __del__(self): - if self.solver_created: - acados_sim_solver.acados_sim_free(self.capsule) - acados_sim_solver.acados_sim_solver_free_capsule(self.capsule) diff --git a/third_party/acados/acados_template/acados_solver_common.pxd b/third_party/acados/acados_template/acados_solver_common.pxd deleted file mode 100644 index c6d59d40a501ef..00000000000000 --- a/third_party/acados/acados_template/acados_solver_common.pxd +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: future_fstrings -*- -# -# Copyright (c) The acados authors. -# -# This file is part of acados. -# -# The 2-Clause BSD License -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE.; -# - - -cdef extern from "acados/ocp_nlp/ocp_nlp_common.h": - ctypedef struct ocp_nlp_config: - pass - - ctypedef struct ocp_nlp_dims: - pass - - ctypedef struct ocp_nlp_in: - pass - - ctypedef struct ocp_nlp_out: - pass - - -cdef extern from "acados_c/ocp_nlp_interface.h": - ctypedef enum ocp_nlp_solver_t: - pass - - ctypedef enum ocp_nlp_cost_t: - pass - - ctypedef enum ocp_nlp_dynamics_t: - pass - - ctypedef enum ocp_nlp_constraints_t: - pass - - ctypedef enum ocp_nlp_reg_t: - pass - - ctypedef struct ocp_nlp_plan: - pass - - ctypedef struct ocp_nlp_solver: - pass - - int ocp_nlp_cost_model_set(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_in *in_, - int start_stage, const char *field, void *value) - int ocp_nlp_constraints_model_set(ocp_nlp_config *config, ocp_nlp_dims *dims, - ocp_nlp_in *in_, int stage, const char *field, void *value) - - # out - void ocp_nlp_out_set(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_out *out, - int stage, const char *field, void *value) - void ocp_nlp_out_get(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_out *out, - int stage, const char *field, void *value) - void ocp_nlp_get_at_stage(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_solver *solver, - int stage, const char *field, void *value) - int ocp_nlp_dims_get_from_attr(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_out *out, - int stage, const char *field) - void ocp_nlp_constraint_dims_get_from_attr(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_out *out, - int stage, const char *field, int *dims_out) - void ocp_nlp_cost_dims_get_from_attr(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_out *out, - int stage, const char *field, int *dims_out) - void ocp_nlp_qp_dims_get_from_attr(ocp_nlp_config *config, ocp_nlp_dims *dims, ocp_nlp_out *out, - int stage, const char *field, int *dims_out) - - # opts - void ocp_nlp_solver_opts_set(ocp_nlp_config *config, void *opts_, const char *field, void* value) - - # solver - void ocp_nlp_eval_residuals(ocp_nlp_solver *solver, ocp_nlp_in *nlp_in, ocp_nlp_out *nlp_out) - void ocp_nlp_eval_param_sens(ocp_nlp_solver *solver, char *field, int stage, int index, ocp_nlp_out *sens_nlp_out) - void ocp_nlp_eval_cost(ocp_nlp_solver *solver, ocp_nlp_in *nlp_in_, ocp_nlp_out *nlp_out) - - # get/set - void ocp_nlp_get(ocp_nlp_config *config, ocp_nlp_solver *solver, const char *field, void *return_value_) - void ocp_nlp_set(ocp_nlp_config *config, ocp_nlp_solver *solver, int stage, const char *field, void *value) diff --git a/third_party/acados/acados_template/gnsf/check_reformulation.py b/third_party/acados/acados_template/gnsf/check_reformulation.py index 2bdfbbc3363a61..291e2fa35667aa 100644 --- a/third_party/acados/acados_template/gnsf/check_reformulation.py +++ b/third_party/acados/acados_template/gnsf/check_reformulation.py @@ -28,7 +28,7 @@ # POSSIBILITY OF SUCH DAMAGE.; # -from acados_template.utils import casadi_length +from ..utils import casadi_length from casadi import * import numpy as np From 399192dba3ae91c5f15124a85b18d7c813ad4589 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 04:32:09 -0600 Subject: [PATCH 10/11] Remove remaining references to cython --- SConstruct | 6 +- common/transformations/transformations.pxd | 72 --------------------- selfdrive/modeld/models/commonmodel.pxd | 27 -------- site_scons/site_tools/cython.py | 75 ---------------------- 4 files changed, 3 insertions(+), 177 deletions(-) delete mode 100644 common/transformations/transformations.pxd delete mode 100644 selfdrive/modeld/models/commonmodel.pxd delete mode 100644 site_scons/site_tools/cython.py diff --git a/SConstruct b/SConstruct index 094503cfa7902a..9ff4a600472add 100644 --- a/SConstruct +++ b/SConstruct @@ -86,10 +86,10 @@ env = Environment( f"#third_party/acados/{arch}/lib", ], RPATH=[], - CYTHONCFILESUFFIX=".cpp", + COMPILATIONDB_USE_ABSPATH=True, REDNOSE_ROOT="#", - tools=["default", "cython", "compilation_db", "rednose_filter"], + tools=["default", "compilation_db", "rednose_filter"], toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"], ) @@ -149,7 +149,7 @@ def progress_function(node): if os.environ.get('SCONS_PROGRESS'): Progress(progress_function, interval=node_interval) -# ********** Cython build environment ********** +# ********** Python build environment ********** py_include = sysconfig.get_paths()['include'] envCython = env.Clone() envCython["CPPPATH"] += [py_include, np.get_include()] diff --git a/common/transformations/transformations.pxd b/common/transformations/transformations.pxd deleted file mode 100644 index fe32e18deac88b..00000000000000 --- a/common/transformations/transformations.pxd +++ /dev/null @@ -1,72 +0,0 @@ -# cython: language_level=3 -from libcpp cimport bool - -cdef extern from "orientation.cc": - pass - -cdef extern from "orientation.hpp": - cdef cppclass Quaternion "Eigen::Quaterniond": - Quaternion() - Quaternion(double, double, double, double) - double w() - double x() - double y() - double z() - - cdef cppclass Vector3 "Eigen::Vector3d": - Vector3() - Vector3(double, double, double) - double operator()(int) - - cdef cppclass Matrix3 "Eigen::Matrix3d": - Matrix3() - Matrix3(double*) - - double operator()(int, int) - - Quaternion euler2quat(const Vector3 &) - Vector3 quat2euler(const Quaternion &) - Matrix3 quat2rot(const Quaternion &) - Quaternion rot2quat(const Matrix3 &) - Vector3 rot2euler(const Matrix3 &) - Matrix3 euler2rot(const Vector3 &) - Matrix3 rot_matrix(double, double, double) - Vector3 ecef_euler_from_ned(const ECEF &, const Vector3 &) - Vector3 ned_euler_from_ecef(const ECEF &, const Vector3 &) - - -cdef extern from "coordinates.cc": - cdef struct ECEF: - double x - double y - double z - - cdef struct NED: - double n - double e - double d - - cdef struct Geodetic: - double lat - double lon - double alt - bool radians - - ECEF geodetic2ecef(const Geodetic &) - Geodetic ecef2geodetic(const ECEF &) - - cdef cppclass LocalCoord_c "LocalCoord": - Matrix3 ned2ecef_matrix - Matrix3 ecef2ned_matrix - - LocalCoord_c(const Geodetic &, const ECEF &) - LocalCoord_c(const Geodetic &) - LocalCoord_c(const ECEF &) - - NED ecef2ned(const ECEF &) - ECEF ned2ecef(const NED &) - NED geodetic2ned(const Geodetic &) - Geodetic ned2geodetic(const NED &) - -cdef extern from "coordinates.hpp": - pass diff --git a/selfdrive/modeld/models/commonmodel.pxd b/selfdrive/modeld/models/commonmodel.pxd deleted file mode 100644 index 4ac64d917205d3..00000000000000 --- a/selfdrive/modeld/models/commonmodel.pxd +++ /dev/null @@ -1,27 +0,0 @@ -# distutils: language = c++ - -from msgq.visionipc.visionipc cimport cl_device_id, cl_context, cl_mem - -cdef extern from "common/mat.h": - cdef struct mat3: - float v[9] - -cdef extern from "common/clutil.h": - cdef unsigned long CL_DEVICE_TYPE_DEFAULT - cl_device_id cl_get_device_id(unsigned long) - cl_context cl_create_context(cl_device_id) - void cl_release_context(cl_context) - -cdef extern from "selfdrive/modeld/models/commonmodel.h": - cppclass ModelFrame: - int buf_size - unsigned char * buffer_from_cl(cl_mem*, int); - cl_mem * prepare(cl_mem, int, int, int, int, mat3) - - cppclass DrivingModelFrame: - int buf_size - DrivingModelFrame(cl_device_id, cl_context, int) - - cppclass MonitoringModelFrame: - int buf_size - MonitoringModelFrame(cl_device_id, cl_context) diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py deleted file mode 100644 index f11db1d71bebca..00000000000000 --- a/site_scons/site_tools/cython.py +++ /dev/null @@ -1,75 +0,0 @@ -import re -import SCons -from SCons.Action import Action -from SCons.Scanner import Scanner -import numpy as np - -pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M) -pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M) -cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M) - -np_version = SCons.Script.Value(np.__version__) - -def pyx_scan(node, env, path, arg=None): - contents = node.get_text_contents() - env.Depends(str(node).split('.')[0] + env['CYTHONCFILESUFFIX'], np_version) - - # from cimport ... - matches = pyx_from_import_re.findall(contents) - # cimport - matches += pyx_import_re.findall(contents) - - # Modules can be either .pxd or .pyx files - files = [m.replace('.', '/') + '.pxd' for m in matches] - files += [m.replace('.', '/') + '.pyx' for m in matches] - - # cdef extern from - files += cdef_import_re.findall(contents) - - # Handle relative imports - cur_dir = str(node.get_dir()) - files = [cur_dir + f if f.startswith('/') else f for f in files] - - # Filter out non-existing files (probably system imports) - files = [f for f in files if env.File(f).exists()] - return env.File(files) - - -pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True) -cythonAction = Action("$CYTHONCOM") - - -def create_builder(env): - try: - cython = env['BUILDERS']['Cython'] - except KeyError: - cython = SCons.Builder.Builder( - action=cythonAction, - emitter={}, - suffix=cython_suffix_emitter, - single_source=1 - ) - env.Append(SCANNERS=pyxscanner) - env['BUILDERS']['Cython'] = cython - return cython - -def cython_suffix_emitter(env, source): - return "$CYTHONCFILESUFFIX" - -def generate(env): - env["CYTHON"] = "cythonize" - env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" - env["CYTHONCFILESUFFIX"] = ".cpp" - - c_file, _ = SCons.Tool.createCFileBuilders(env) - - c_file.suffix['.pyx'] = cython_suffix_emitter - c_file.add_action('.pyx', cythonAction) - - c_file.suffix['.py'] = cython_suffix_emitter - c_file.add_action('.py', cythonAction) - - create_builder(env) - -def exists(env): - return True From 0d76749ab3794135a1c9503e0256a5f4369dc510 Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 11 Dec 2025 10:01:09 -0600 Subject: [PATCH 11/11] Remove cython as a dependency --- pyproject.toml | 16 +--------------- uv.lock | 31 ------------------------------- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d5dc95e1b1a7b1..b8d6fcaeb2cb6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,18 +16,14 @@ dependencies = [ "sympy", # rednose + friends "crcmod", # cars + qcomgpsd "tqdm", # cars (fw_versions.py) on start + many one-off uses - # hardwared "smbus2", # configuring amp - # core "cffi", "scons", "pycapnp==2.1.0", - "Cython", "setuptools", "numpy >=2.0", - # body / webrtcd "aiohttp", "aiortc", @@ -35,42 +31,32 @@ dependencies = [ # with the latest release "pyopenssl < 24.3.0", "pyaudio", - # ubloxd (TODO: just use struct) "kaitaistruct", - # panda "libusb1", "spidev; platform_system == 'Linux'", - # modeld "onnx >= 1.14.0", - # logging "pyzmq", "sentry-sdk", "xattr", # used in place of 'os.getxattr' for macos compatibility - # athena "PyJWT", "json-rpc", "websocket_client", - # acados deps "casadi >=3.6.6", # 3.12 fixed in 3.6.6 "future-fstrings", - # joystickd "inputs", - # these should be removed "psutil", "pycryptodome", # used in updated/casync, panda, body, and a test "setproctitle", - # logreader "zstandard", - # ui "raylib < 5.5.0.3", # TODO: unpin when they fix https://github.com/electronstudio/raylib-python-cffi/issues/186 "qrcode", @@ -238,6 +224,7 @@ lint.ignore = [ "NPY002", # new numpy random syntax is worse "UP045", "UP007", # these don't play nice with raylib atm ] +lint.flake8-implicit-str-concat.allow-multiline = false line-length = 160 target-version ="py311" exclude = [ @@ -254,7 +241,6 @@ exclude = [ "*.ipynb", "generated", ] -lint.flake8-implicit-str-concat.allow-multiline = false [tool.ruff.lint.flake8-tidy-imports.banned-api] "selfdrive".msg = "Use openpilot.selfdrive" diff --git a/uv.lock b/uv.lock index b179517e0b26d1..b46165eb590220 100644 --- a/uv.lock +++ b/uv.lock @@ -450,35 +450,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] -[[package]] -name = "cython" -version = "3.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/f6/d762df1f436a0618455d37f4e4c4872a7cd0dcfc8dec3022ee99e4389c69/cython-3.1.4.tar.gz", hash = "sha256:9aefefe831331e2d66ab31799814eae4d0f8a2d246cbaaaa14d1be29ef777683", size = 3190778, upload-time = "2025-09-16T07:20:33.531Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/ab/0a568bac7c4c052db4ae27edf01e16f3093cdfef04a2dfd313ef1b3c478a/cython-3.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d1d7013dba5fb0506794d4ef8947ff5ed021370614950a8d8d04e57c8c84499e", size = 3026389, upload-time = "2025-09-16T07:22:02.212Z" }, - { url = "https://files.pythonhosted.org/packages/cb/b7/51f5566e1309215a7fef744975b2fabb56d3fdc5fa1922fd7e306c14f523/cython-3.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:eed989f5c139d6550ef2665b783d86fab99372590c97f10a3c26c4523c5fce9e", size = 2955954, upload-time = "2025-09-16T07:22:03.782Z" }, - { url = "https://files.pythonhosted.org/packages/28/fd/ad8314520000fe96292fb8208c640fa862baa3053d2f3453a2acb50cafb8/cython-3.1.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3df3beb8b024dfd73cfddb7f2f7456751cebf6e31655eed3189c209b634bc2f2", size = 3412005, upload-time = "2025-09-16T07:22:05.483Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3b/e570f8bcb392e7943fc9a25d1b2d1646ef0148ff017d3681511acf6bbfdc/cython-3.1.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8354703f1168e1aaa01348940f719734c1f11298be333bdb5b94101d49677c0", size = 3191100, upload-time = "2025-09-16T07:22:07.144Z" }, - { url = "https://files.pythonhosted.org/packages/78/81/f1ea09f563ebab732542cb11bf363710e53f3842458159ea2c160788bc8e/cython-3.1.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a928bd7d446247855f54f359057ab4a32c465219c8c1e299906a483393a59a9e", size = 3313786, upload-time = "2025-09-16T07:22:09.15Z" }, - { url = "https://files.pythonhosted.org/packages/ca/17/06575eb6175a926523bada7dac1cd05cc74add96cebbf2e8b492a2494291/cython-3.1.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c233bfff4cc7b9d629eecb7345f9b733437f76dc4441951ec393b0a6e29919fc", size = 3205775, upload-time = "2025-09-16T07:22:10.745Z" }, - { url = "https://files.pythonhosted.org/packages/10/ba/61a8cf56a76ab21ddf6476b70884feff2a2e56b6d9010e1e1b1e06c46f70/cython-3.1.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e9691a2cbc2faf0cd819108bceccf9bfc56c15a06d172eafe74157388c44a601", size = 3428423, upload-time = "2025-09-16T07:22:12.404Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c2/42cf9239088d6b4b62c1c017c36e0e839f64c8d68674ce4172d0e0168d3b/cython-3.1.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ada319207432ea7c6691c70b5c112d261637d79d21ba086ae3726fedde79bfbf", size = 3330489, upload-time = "2025-09-16T07:22:14.576Z" }, - { url = "https://files.pythonhosted.org/packages/b5/08/36a619d6b1fc671a11744998e5cdd31790589e3cb4542927c97f3f351043/cython-3.1.4-cp311-cp311-win32.whl", hash = "sha256:dae81313c28222bf7be695f85ae1d16625aac35a0973a3af1e001f63379440c5", size = 2482410, upload-time = "2025-09-16T07:22:17.373Z" }, - { url = "https://files.pythonhosted.org/packages/6d/58/7d9ae7944bcd32e6f02d1a8d5d0c3875125227d050e235584127f2c64ffd/cython-3.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:60d2f192059ac34c5c26527f2beac823d34aaa766ef06792a3b7f290c18ac5e2", size = 2713755, upload-time = "2025-09-16T07:22:18.949Z" }, - { url = "https://files.pythonhosted.org/packages/f0/51/2939c739cfdc67ab94935a2c4fcc75638afd15e1954552655503a4112e92/cython-3.1.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0d26af46505d0e54fe0f05e7ad089fd0eed8fa04f385f3ab88796f554467bcb9", size = 3062976, upload-time = "2025-09-16T07:22:20.517Z" }, - { url = "https://files.pythonhosted.org/packages/eb/bd/a84de57fd01017bf5dba84a49aeee826db21112282bf8d76ab97567ee15d/cython-3.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66ac8bb5068156c92359e3f0eefa138c177d59d1a2e8a89467881fa7d06aba3b", size = 2970701, upload-time = "2025-09-16T07:22:22.644Z" }, - { url = "https://files.pythonhosted.org/packages/71/79/a09004c8e42f5be188c7636b1be479cdb244a6d8837e1878d062e4e20139/cython-3.1.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2e42714faec723d2305607a04bafb49a48a8d8f25dd39368d884c058dbcfbc", size = 3387730, upload-time = "2025-09-16T07:22:24.271Z" }, - { url = "https://files.pythonhosted.org/packages/fb/bd/979f8c59e247f562642f3eb98a1b453530e1f7954ef071835c08ed2bf6ba/cython-3.1.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0fd655b27997a209a574873304ded9629de588f021154009e8f923475e2c677", size = 3167289, upload-time = "2025-09-16T07:22:26.35Z" }, - { url = "https://files.pythonhosted.org/packages/34/f8/0b98537f0b4e8c01f76d2a6cf75389987538e4d4ac9faf25836fd18c9689/cython-3.1.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9def7c41f4dc339003b1e6875f84edf059989b9c7f5e9a245d3ce12c190742d9", size = 3321099, upload-time = "2025-09-16T07:22:27.957Z" }, - { url = "https://files.pythonhosted.org/packages/f3/39/437968a2e7c7f57eb6e1144f6aca968aa15fbbf169b2d4da5d1ff6c21442/cython-3.1.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:196555584a8716bf7e017e23ca53e9f632ed493f9faa327d0718e7551588f55d", size = 3179897, upload-time = "2025-09-16T07:22:30.014Z" }, - { url = "https://files.pythonhosted.org/packages/2c/04/b3f42915f034d133f1a34e74a2270bc2def02786f9b40dc9028fbb968814/cython-3.1.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7fff0e739e07a20726484b8898b8628a7b87acb960d0fc5486013c6b77b7bb97", size = 3400936, upload-time = "2025-09-16T07:22:31.705Z" }, - { url = "https://files.pythonhosted.org/packages/21/eb/2ad9fa0896ab6cf29875a09a9f4aaea37c28b79b869a013bf9b58e4e652e/cython-3.1.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2754034fa10f95052949cd6b07eb2f61d654c1b9cfa0b17ea53a269389422e8", size = 3332131, upload-time = "2025-09-16T07:22:33.32Z" }, - { url = "https://files.pythonhosted.org/packages/3c/bf/f19283f8405e7e564c3353302a8665ea2c589be63a8e1be1b503043366a9/cython-3.1.4-cp312-cp312-win32.whl", hash = "sha256:2e0808ff3614a1dbfd1adfcbff9b2b8119292f1824b3535b4a173205109509f8", size = 2487672, upload-time = "2025-09-16T07:22:35.227Z" }, - { url = "https://files.pythonhosted.org/packages/30/bf/32150a2e6c7b50b81c5dc9e942d41969400223a9c49d04e2ed955709894c/cython-3.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f262b32327b6bce340cce5d45bbfe3972cb62543a4930460d8564a489f3aea12", size = 2705348, upload-time = "2025-09-16T07:22:37.922Z" }, - { url = "https://files.pythonhosted.org/packages/7c/24/f7351052cf9db771fe4f32fca47fd66e6d9b53d8613b17faf7d130a9d553/cython-3.1.4-py3-none-any.whl", hash = "sha256:d194d95e4fa029a3f6c7d46bdd16d973808c7ea4797586911fdb67cb98b1a2c6", size = 1227541, upload-time = "2025-09-16T07:20:29.595Z" }, -] - [[package]] name = "dbus-next" version = "0.2.3" @@ -1322,7 +1293,6 @@ dependencies = [ { name = "casadi" }, { name = "cffi" }, { name = "crcmod" }, - { name = "cython" }, { name = "future-fstrings" }, { name = "inputs" }, { name = "json-rpc" }, @@ -1416,7 +1386,6 @@ requires-dist = [ { name = "codespell", marker = "extra == 'testing'" }, { name = "coverage", marker = "extra == 'testing'" }, { name = "crcmod" }, - { name = "cython" }, { name = "dbus-next", marker = "extra == 'dev'" }, { name = "dearpygui", marker = "(platform_machine != 'aarch64' and extra == 'tools') or (sys_platform != 'linux' and extra == 'tools')", specifier = ">=2.1.0" }, { name = "dictdiffer", marker = "extra == 'dev'" },