From 59b7353c360c64095c967fa27800af77acecc0e0 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 15:45:35 -0700
Subject: [PATCH 01/25] python package

---
 MANIFEST.in    |  6 ++++++
 SConstruct     |  1 +
 pyproject.toml | 26 ++++++++++++++++++++++++++
 setup.py       | 21 +++++++++++++++++++++
 4 files changed, 54 insertions(+)
 create mode 100644 MANIFEST.in
 create mode 100644 setup.py

diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 000000000..f7d309432
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,6 @@
+exclude msgq/**/*.o
+recursive-exclude msgq/tests *
+recursive-exclude msgq/visionipc/tests *
+include SConscript
+include SConstruct
+recursive-include site_scons *
diff --git a/SConstruct b/SConstruct
index c1e7fc521..476672be5 100644
--- a/SConstruct
+++ b/SConstruct
@@ -61,6 +61,7 @@ env = Environment(
     "-Werror",
     "-Wshadow",
     "-Wno-vla-cxx-extension",
+    "-Wno-unknown-warning-option",
   ] + ccflags,
   LDFLAGS=ldflags,
   LINKFLAGS=ldflags,
diff --git a/pyproject.toml b/pyproject.toml
index c0d9b3f40..c275f7940 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,29 @@
+[project]
+name = "msgq"
+requires-python = ">= 3.8"
+version = "0.1.0"
+
+dependencies = [
+  "numpy",
+  "Cython",
+  "pycapnp",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pyyaml",
+  "scons",
+  "pre-commit",
+  "ruff",
+  "parameterized",
+  "coverage",
+  "pytest",
+]
+
+[build-system]
+requires = ["setuptools","scons","Cython","numpy"]
+build-backend = "setuptools.build_meta"
+
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
 lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"]
diff --git a/setup.py b/setup.py
new file mode 100644
index 000000000..9d3d88488
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,21 @@
+from setuptools import Command, setup
+from setuptools.command.build import build
+import subprocess
+
+class SconsBuild(Command):
+  def initialize_options(self) -> None:
+    pass
+
+  def finalize_options(self) -> None:
+    pass
+
+  def run(self) -> None:
+    subprocess.run(["scons --minimal -j$(nproc)"], shell=True)
+
+class CustomBuild(build):
+  sub_commands = [('scons_build', None)] + build.sub_commands
+
+setup(
+    package_data={'msgq': ['**/*.cc', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.so']},
+    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}
+    )

From e835fff96a3b0f3c7cb5ab0088ae36e94c1f30fc Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 16:53:57 -0700
Subject: [PATCH 02/25] packages

---
 Dockerfile | 5 ++---
 setup.py   | 6 +++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 77ef04c29..c93156d6b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,8 +35,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     zlib1g-dev \
   && rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy pytest
-
 WORKDIR /project/msgq/
 RUN cd /tmp/ && \
     git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \
@@ -51,4 +49,5 @@ ENV PYTHONPATH=/project
 
 COPY . .
 RUN ls && rm -rf .git && \
-    scons -c && scons -j$(nproc)
+    SCONS_EXTRAS=1 pip3 install --break-system-packages --no-cache-dir .[dev]
+
diff --git a/setup.py b/setup.py
index 9d3d88488..4dfcee787 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,7 @@
 from setuptools import Command, setup
 from setuptools.command.build import build
 import subprocess
+import os
 
 class SconsBuild(Command):
   def initialize_options(self) -> None:
@@ -10,12 +11,15 @@ def finalize_options(self) -> None:
     pass
 
   def run(self) -> None:
-    subprocess.run(["scons --minimal -j$(nproc)"], shell=True)
+    scons_flags = '--minimal' if 'SCONS_EXTRAS' not in  os.environ else ''
+    subprocess.run([f"scons {scons_flags} -j$(nproc)"], shell=True).check_returncode()
 
 class CustomBuild(build):
   sub_commands = [('scons_build', None)] + build.sub_commands
 
 setup(
+    packages = ["msgq", "msgq.visionipc"],
     package_data={'msgq': ['**/*.cc', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.so']},
+    include_package_data=True,
     cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}
     )

From 6bb46b9566b5569d039d7c91d860b08e6de1d8cc Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 17:48:54 -0700
Subject: [PATCH 03/25] test versions

---
 .github/workflows/tests.yml | 20 ++++++++-----
 Dockerfile                  |  1 -
 install_dependencies.sh     | 56 +++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+), 8 deletions(-)
 create mode 100755 install_dependencies.sh

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index cab495a46..d52092c97 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -32,18 +32,24 @@ jobs:
       matrix:
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v3
-    - name: Build docker image
-      run: eval "$BUILD"
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Installing build dependencies
+      run: ./install_dependencies.sh
+    - name: Installing msgq
+      run: SCONS_EXTRAS=1 pip3 install --break-system-packages --no-cache-dir .[dev]
     - name: C++ tests
       run: |
-        $RUN "export ${{ matrix.backend }}=1 && \
-              scons ${{ matrix.flags }} -j$(nproc) && \
-              msgq/test_runner && \
-              msgq/visionipc/test_runner"
+        export ${{ matrix.backend }}=1
+        scons ${{ matrix.flags }} -j$(nproc)
+        msgq/test_runner
+        msgq/visionipc/test_runner
     - name: python tests
-      run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m pytest"
+      run: ${{ matrix.backend }}=1 coverage run -m pytest
     - name: Upload coverage
       run: |
         docker commit msgq msgqci
diff --git a/Dockerfile b/Dockerfile
index c93156d6b..2e914ef33 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -50,4 +50,3 @@ ENV PYTHONPATH=/project
 COPY . .
 RUN ls && rm -rf .git && \
     SCONS_EXTRAS=1 pip3 install --break-system-packages --no-cache-dir .[dev]
-
diff --git a/install_dependencies.sh b/install_dependencies.sh
new file mode 100755
index 000000000..243f2e06d
--- /dev/null
+++ b/install_dependencies.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+cd $DIR
+
+SUDO=""
+
+# Use sudo if not root
+if [[ ! $(id -u) -eq 0 ]]; then
+  if [[ -z $(which sudo) ]]; then
+    echo "Please install sudo or run as root"
+    exit 1
+  fi
+  SUDO="sudo"
+fi
+
+$SUDO apt-get update
+$SUDO apt-get install -y --no-install-recommends \
+  autoconf \
+  build-essential \
+  ca-certificates \
+  capnproto \
+  clang \
+  cppcheck \
+  curl \
+  git \
+  libbz2-dev \
+  libcapnp-dev \
+  libffi-dev \
+  liblzma-dev \
+  libncurses5-dev \
+  libncursesw5-dev \
+  libreadline-dev \
+  libsqlite3-dev \
+  libssl-dev \
+  libtool \
+  libzmq3-dev \
+  llvm \
+  make \
+  cmake \
+  ocl-icd-opencl-dev \
+  opencl-headers  \
+  python3-dev \
+  python3-pip \
+  tk-dev \
+  wget \
+  xz-utils \
+  zlib1g-dev \
+  libclang-rt-dev
+
+git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
+cd Catch2
+mv single_include/* ../
+cd ..
+rm -rf Catch2

From d57ae055247cde05666dfb13b175a9eb175dc0be Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 17:53:44 -0700
Subject: [PATCH 04/25] pip

---
 .github/workflows/tests.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d52092c97..2966e5742 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -38,10 +38,12 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Installing pip
+      run: pip install pip==24.0
     - name: Installing build dependencies
       run: ./install_dependencies.sh
     - name: Installing msgq
-      run: SCONS_EXTRAS=1 pip3 install --break-system-packages --no-cache-dir .[dev]
+      run: SCONS_EXTRAS=1 pip install --break-system-packages --no-cache-dir .[dev]
     - name: C++ tests
       run: |
         export ${{ matrix.backend }}=1

From fb2b0b4d7126508d0bb4ab83d6e48b1f6c0f73c9 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 17:57:44 -0700
Subject: [PATCH 05/25] env

---
 .github/workflows/tests.yml | 26 +-------------------------
 1 file changed, 1 insertion(+), 25 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2966e5742..f09b7f168 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -2,29 +2,7 @@ name: tests
 
 on: [push, pull_request]
 
-env:
-  DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name msgq msgq /bin/sh -c
-  RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm msgq /bin/sh -c
-  CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm msgqci /bin/bash -c
-  BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/msgq:latest -t msgq -f Dockerfile .
-  PYTHONWARNINGS: error
-
 jobs:
-  build:
-    name: build
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Build docker image
-      run: eval "$BUILD"
-    - name: Push to dockerhub
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/msgq'
-      run: |
-        docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-        docker tag msgq $DOCKER_REGISTRY/msgq:latest
-        docker push $DOCKER_REGISTRY/msgq:latest
-
   unit_tests:
     name: unit tests
     runs-on: ubuntu-latest
@@ -34,12 +12,10 @@ jobs:
         backend: ['MSGQ', 'ZMQ']
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Installing pip
-      run: pip install pip==24.0
     - name: Installing build dependencies
       run: ./install_dependencies.sh
     - name: Installing msgq

From 52e4ad1fd2e9babfb2cf613f5c899460d0c6f6e8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 17:59:54 -0700
Subject: [PATCH 06/25] test

---
 install_dependencies.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/install_dependencies.sh b/install_dependencies.sh
index 243f2e06d..5b0c074d0 100755
--- a/install_dependencies.sh
+++ b/install_dependencies.sh
@@ -46,8 +46,7 @@ $SUDO apt-get install -y --no-install-recommends \
   tk-dev \
   wget \
   xz-utils \
-  zlib1g-dev \
-  libclang-rt-dev
+  zlib1g-dev
 
 git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
 cd Catch2

From e4737bfbaad061f404fb7adfc8293dac702ec04b Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 19:03:45 -0700
Subject: [PATCH 07/25] cov

---
 .github/workflows/tests.yml | 12 ++++++------
 pyproject.toml              |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f09b7f168..bc2f774a7 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -27,19 +27,19 @@ jobs:
         msgq/test_runner
         msgq/visionipc/test_runner
     - name: python tests
-      run: ${{ matrix.backend }}=1 coverage run -m pytest
+      run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: Upload coverage
-      run: |
-        docker commit msgq msgqci
-        $CI_RUN "cd /project/msgq && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
+      run: "bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
 
   static_analysis:
     name: static analysis
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Build docker image
-      run: eval "$BUILD"
+    - name: Installing build dependencies
+      run: ./install_dependencies.sh
+    - name: Installing msgq
+      run: pip install --break-system-packages --no-cache-dir .[dev]
     - name: Static analysis
       # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
       run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all"
diff --git a/pyproject.toml b/pyproject.toml
index c275f7940..d3c64e6e1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,6 +18,7 @@ dev = [
   "parameterized",
   "coverage",
   "pytest",
+  "pytest-cov",
 ]
 
 [build-system]

From 8beed006d150a3fed04c32943a2c3d9a25d7be50 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 19:49:51 -0700
Subject: [PATCH 08/25] runner

---
 .github/workflows/tests.yml | 4 ++--
 install_dependencies.sh     | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index bc2f774a7..946e7f85d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -5,7 +5,7 @@ on: [push, pull_request]
 jobs:
   unit_tests:
     name: unit tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         flags: ['', '--asan', '--ubsan']
@@ -42,4 +42,4 @@ jobs:
       run: pip install --break-system-packages --no-cache-dir .[dev]
     - name: Static analysis
       # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
-      run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all"
+      run: unset PYTHONWARNINGS && pre-commit run --all
diff --git a/install_dependencies.sh b/install_dependencies.sh
index 5b0c074d0..ec0d00843 100755
--- a/install_dependencies.sh
+++ b/install_dependencies.sh
@@ -27,6 +27,7 @@ $SUDO apt-get install -y --no-install-recommends \
   git \
   libbz2-dev \
   libcapnp-dev \
+  libclang-rt-dev \
   libffi-dev \
   liblzma-dev \
   libncurses5-dev \

From e91fbe0adb6ba700877163e13421e7f3d187269e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 20:11:53 -0700
Subject: [PATCH 09/25] not docker anymore...

---
 .github/workflows/tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 946e7f85d..a8de041c5 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -20,26 +20,26 @@ jobs:
       run: ./install_dependencies.sh
     - name: Installing msgq
       run: SCONS_EXTRAS=1 pip install --break-system-packages --no-cache-dir .[dev]
+    - name: python tests
+      run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: C++ tests
       run: |
         export ${{ matrix.backend }}=1
         scons ${{ matrix.flags }} -j$(nproc)
         msgq/test_runner
         msgq/visionipc/test_runner
-    - name: python tests
-      run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: Upload coverage
       run: "bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
 
   static_analysis:
     name: static analysis
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v3
     - name: Installing build dependencies
       run: ./install_dependencies.sh
     - name: Installing msgq
-      run: pip install --break-system-packages --no-cache-dir .[dev]
+      run: pip3 install --break-system-packages --no-cache-dir .[dev]
     - name: Static analysis
       # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
       run: unset PYTHONWARNINGS && pre-commit run --all

From f4ac5c7ad717efc97bc93a088b9a5bd740bec0a0 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 20:24:39 -0700
Subject: [PATCH 10/25] cleanup

---
 .github/workflows/tests.yml |  4 +--
 Dockerfile                  | 52 -------------------------------------
 2 files changed, 2 insertions(+), 54 deletions(-)
 delete mode 100644 Dockerfile

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a8de041c5..271da3327 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Installing build dependencies
       run: ./install_dependencies.sh
     - name: Installing msgq
-      run: SCONS_EXTRAS=1 pip install --break-system-packages --no-cache-dir .[dev]
+      run: SCONS_EXTRAS=1 pip3 install --break-system-packages .[dev]
     - name: python tests
       run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: C++ tests
@@ -39,7 +39,7 @@ jobs:
     - name: Installing build dependencies
       run: ./install_dependencies.sh
     - name: Installing msgq
-      run: pip3 install --break-system-packages --no-cache-dir .[dev]
+      run: pip3 install --break-system-packages .[dev]
     - name: Static analysis
       # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
       run: unset PYTHONWARNINGS && pre-commit run --all
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 2e914ef33..000000000
--- a/Dockerfile
+++ /dev/null
@@ -1,52 +0,0 @@
-FROM ubuntu:24.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    autoconf \
-    build-essential \
-    ca-certificates \
-    capnproto \
-    clang \
-    cppcheck \
-    curl \
-    git \
-    libbz2-dev \
-    libcapnp-dev \
-    libclang-rt-dev \
-    libffi-dev \
-    liblzma-dev \
-    libncurses5-dev \
-    libncursesw5-dev \
-    libreadline-dev \
-    libsqlite3-dev \
-    libssl-dev \
-    libtool \
-    libzmq3-dev \
-    llvm \
-    make \
-    cmake \
-    ocl-icd-opencl-dev \
-    opencl-headers  \
-    python3-dev \
-    python3-pip \
-    tk-dev \
-    wget \
-    xz-utils \
-    zlib1g-dev \
-  && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /project/msgq/
-RUN cd /tmp/ && \
-    git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \
-    cd Catch2 && \
-    mv single_include/* /project/msgq/ && \
-    cd .. \
-    rm -rf Catch2
-
-WORKDIR /project/msgq
-
-ENV PYTHONPATH=/project
-
-COPY . .
-RUN ls && rm -rf .git && \
-    SCONS_EXTRAS=1 pip3 install --break-system-packages --no-cache-dir .[dev]

From be794889a48da0fc59868079464ad1f569ae338e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 11 Jul 2024 20:43:43 -0700
Subject: [PATCH 11/25] ignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 6db3c7cd4..05501ec52 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,5 @@ services.h
 .sconsign.dblite
 libcereal_shared.*
 .mypy_cache/
+msgq.egg-info
+build

From df54d62d71641aae86bfeae2ef4d26636ee4c137 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 12 Jul 2024 15:06:58 -0700
Subject: [PATCH 12/25] cleanup

---
 .github/workflows/tests.yml                        | 11 ++++++-----
 .gitignore                                         |  4 +++-
 MANIFEST.in                                        |  2 --
 scripts/build_wheel.sh                             |  9 +++++++++
 .../install_dependencies.sh                        | 14 ++++++++------
 setup.py                                           |  2 +-
 6 files changed, 27 insertions(+), 15 deletions(-)
 create mode 100755 scripts/build_wheel.sh
 rename install_dependencies.sh => scripts/install_dependencies.sh (81%)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 271da3327..babdffdf2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -16,11 +16,12 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Installing build dependencies
-      run: ./install_dependencies.sh
-    - name: Installing msgq
-      run: SCONS_EXTRAS=1 pip3 install --break-system-packages .[dev]
-    - name: python tests
+    - name: Building msgq
+      run: |
+        export BUILD_TESTS=1
+        scripts/install_dependencies.sh
+        pip3 install --break-system-packages .[dev]
+    - name: Python tests
       run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: C++ tests
       run: |
diff --git a/.gitignore b/.gitignore
index 05501ec52..d7e38f211 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,6 @@ services.h
 libcereal_shared.*
 .mypy_cache/
 msgq.egg-info
-build
+build/
+dist/
+catch2/
diff --git a/MANIFEST.in b/MANIFEST.in
index f7d309432..a65dcef48 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,4 @@
 exclude msgq/**/*.o
-recursive-exclude msgq/tests *
-recursive-exclude msgq/visionipc/tests *
 include SConscript
 include SConstruct
 recursive-include site_scons *
diff --git a/scripts/build_wheel.sh b/scripts/build_wheel.sh
new file mode 100755
index 000000000..846ae3695
--- /dev/null
+++ b/scripts/build_wheel.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+cd $DIR
+
+./install_dependencies.sh
+cd ..
+python3 -m build
diff --git a/install_dependencies.sh b/scripts/install_dependencies.sh
similarity index 81%
rename from install_dependencies.sh
rename to scripts/install_dependencies.sh
index ec0d00843..2acdda92b 100755
--- a/install_dependencies.sh
+++ b/scripts/install_dependencies.sh
@@ -2,7 +2,7 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
-cd $DIR
+cd $DIR/../
 
 SUDO=""
 
@@ -49,8 +49,10 @@ $SUDO apt-get install -y --no-install-recommends \
   xz-utils \
   zlib1g-dev
 
-git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
-cd Catch2
-mv single_include/* ../
-cd ..
-rm -rf Catch2
+if [[ -n "$BUILD_TESTS" ]]; then
+  git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
+  cd Catch2
+  mv single_include/* ../
+  cd ..
+  rm -rf Catch2
+fi
diff --git a/setup.py b/setup.py
index 4dfcee787..0741723cd 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ def finalize_options(self) -> None:
     pass
 
   def run(self) -> None:
-    scons_flags = '--minimal' if 'SCONS_EXTRAS' not in  os.environ else ''
+    scons_flags = '' if 'BUILD_TESTS' in os.environ else '--minimal'
     subprocess.run([f"scons {scons_flags} -j$(nproc)"], shell=True).check_returncode()
 
 class CustomBuild(build):

From 5dd95c9959a5c7bb05eb98465d209b776e2ed301 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 12 Jul 2024 15:10:10 -0700
Subject: [PATCH 13/25] static

---
 .github/workflows/tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index babdffdf2..876a17e46 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -37,10 +37,10 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v3
-    - name: Installing build dependencies
-      run: ./install_dependencies.sh
-    - name: Installing msgq
-      run: pip3 install --break-system-packages .[dev]
+    - name: Building msgq
+      run: |
+        scripts/install_dependencies.sh
+        pip3 install --break-system-packages .[dev]
     - name: Static analysis
       # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
       run: unset PYTHONWARNINGS && pre-commit run --all

From 42864201c4a68335aee316a4ff0742faaeac48db Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 21:05:14 -0700
Subject: [PATCH 14/25] better

---
 .github/workflows/tests.yml       |  2 +-
 .github/workflows/wheels.yml      | 51 +++++++++++++++++++++++++++
 pyproject.toml                    |  4 +--
 requirements.txt                  | 50 ++++++++++++++++++++++++++
 scripts/build_wheel.sh            |  9 -----
 scripts/install_dependencies.sh   | 58 -------------------------------
 scripts/macos_dependencies.sh     |  6 ++++
 scripts/manylinux_dependencies.sh | 10 ++++++
 scripts/ubuntu_dependencies.sh    | 12 +++++++
 setup.py                          | 15 +++++---
 10 files changed, 143 insertions(+), 74 deletions(-)
 create mode 100644 .github/workflows/wheels.yml
 create mode 100644 requirements.txt
 delete mode 100755 scripts/build_wheel.sh
 delete mode 100755 scripts/install_dependencies.sh
 create mode 100755 scripts/macos_dependencies.sh
 create mode 100755 scripts/manylinux_dependencies.sh
 create mode 100755 scripts/ubuntu_dependencies.sh

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 876a17e46..d684c0489 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
new file mode 100644
index 000000000..b96a2e514
--- /dev/null
+++ b/.github/workflows/wheels.yml
@@ -0,0 +1,51 @@
+name: wheels
+
+on: [push, pull_request]
+
+jobs:
+  wheels:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64",  arch: "x86_64"},
+                    {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"},
+                    {os: "macos-14",      target: "macosx_arm64",      arch: "arm64"} ]
+        python:   [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+
+    - name: Building wheel
+      uses: pypa/cibuildwheel@v2.19.2
+      env:
+        CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
+        CIBW_ARCHS: "${{ matrix.platform.arch }}"
+        CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
+        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+        CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+        CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+        CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package}
+
+
+    - uses: actions/setup-python@v5
+      if: ${{ matrix.platform.arch }} != 'aarch64'
+      with:
+        python-version: ${{ matrix.python.py }}
+
+    - name: Installing the wheel
+      if: ${{ matrix.platform.arch }} != 'aarch64'
+      run: |
+        pip install --break-system-packages ./wheelhouse/*.whl
+
+    - name: Saving wheel
+      uses: actions/upload-artifact@v4
+      with:
+        name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }}
+        path: ./wheelhouse/*.whl
diff --git a/pyproject.toml b/pyproject.toml
index d3c64e6e1..bd7b187c6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,17 +1,17 @@
 [project]
 name = "msgq"
-requires-python = ">= 3.8"
+requires-python = ">= 3.11"
 version = "0.1.0"
 
 dependencies = [
   "numpy",
   "Cython",
   "pycapnp",
+  "setuptools",
 ]
 
 [project.optional-dependencies]
 dev = [
-  "pyyaml",
   "scons",
   "pre-commit",
   "ruff",
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 000000000..e179397e2
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,50 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --all-extras -o requirements.txt pyproject.toml
+cfgv==3.4.0
+    # via pre-commit
+coverage==7.6.0
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+cython==3.0.10
+    # via msgq (pyproject.toml)
+distlib==0.3.8
+    # via virtualenv
+filelock==3.15.4
+    # via virtualenv
+identify==2.6.0
+    # via pre-commit
+iniconfig==2.0.0
+    # via pytest
+nodeenv==1.9.1
+    # via pre-commit
+numpy==2.0.0
+    # via msgq (pyproject.toml)
+packaging==24.1
+    # via pytest
+parameterized==0.9.0
+    # via msgq (pyproject.toml)
+platformdirs==4.2.2
+    # via virtualenv
+pluggy==1.5.0
+    # via pytest
+pre-commit==3.7.1
+    # via msgq (pyproject.toml)
+pycapnp==2.0.0
+    # via msgq (pyproject.toml)
+pytest==8.2.2
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+pytest-cov==5.0.0
+    # via msgq (pyproject.toml)
+pyyaml==6.0.1
+    # via pre-commit
+ruff==0.5.3
+    # via msgq (pyproject.toml)
+scons==4.8.0
+    # via msgq (pyproject.toml)
+setuptools==71.0.4
+    # via msgq (pyproject.toml)
+virtualenv==20.26.3
+    # via pre-commit
diff --git a/scripts/build_wheel.sh b/scripts/build_wheel.sh
deleted file mode 100755
index 846ae3695..000000000
--- a/scripts/build_wheel.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-cd $DIR
-
-./install_dependencies.sh
-cd ..
-python3 -m build
diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh
deleted file mode 100755
index 2acdda92b..000000000
--- a/scripts/install_dependencies.sh
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/bin/bash
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-cd $DIR/../
-
-SUDO=""
-
-# Use sudo if not root
-if [[ ! $(id -u) -eq 0 ]]; then
-  if [[ -z $(which sudo) ]]; then
-    echo "Please install sudo or run as root"
-    exit 1
-  fi
-  SUDO="sudo"
-fi
-
-$SUDO apt-get update
-$SUDO apt-get install -y --no-install-recommends \
-  autoconf \
-  build-essential \
-  ca-certificates \
-  capnproto \
-  clang \
-  cppcheck \
-  curl \
-  git \
-  libbz2-dev \
-  libcapnp-dev \
-  libclang-rt-dev \
-  libffi-dev \
-  liblzma-dev \
-  libncurses5-dev \
-  libncursesw5-dev \
-  libreadline-dev \
-  libsqlite3-dev \
-  libssl-dev \
-  libtool \
-  libzmq3-dev \
-  llvm \
-  make \
-  cmake \
-  ocl-icd-opencl-dev \
-  opencl-headers  \
-  python3-dev \
-  python3-pip \
-  tk-dev \
-  wget \
-  xz-utils \
-  zlib1g-dev
-
-if [[ -n "$BUILD_TESTS" ]]; then
-  git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
-  cd Catch2
-  mv single_include/* ../
-  cd ..
-  rm -rf Catch2
-fi
diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh
new file mode 100755
index 000000000..f0e9dc597
--- /dev/null
+++ b/scripts/macos_dependencies.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+brew bundle --file=- <<-EOS
+brew "zeromq"
+cask "gcc-arm-embedded"
+brew "gcc@13"
+EOS
diff --git a/scripts/manylinux_dependencies.sh b/scripts/manylinux_dependencies.sh
new file mode 100755
index 000000000..9bd69863f
--- /dev/null
+++ b/scripts/manylinux_dependencies.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget
+
+wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz
+tar -xvf zeromq-4.3.5.tar.gz
+cd zeromq-4.3.5
+./configure
+make -j$(nproc)
+make install
diff --git a/scripts/ubuntu_dependencies.sh b/scripts/ubuntu_dependencies.sh
new file mode 100755
index 000000000..88314f593
--- /dev/null
+++ b/scripts/ubuntu_dependencies.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ ! $(id -u) -eq 0 ]]; then
+  if [[ -z $(which sudo) ]]; then
+    echo "Please install sudo or run as root"
+    exit 1
+  fi
+  SUDO="sudo"
+fi
+
+$SUDO apt-get update
+$SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck
diff --git a/setup.py b/setup.py
index 0741723cd..d3198fb50 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,13 @@
-from setuptools import Command, setup
+from setuptools import Command, setup, Distribution
 from setuptools.command.build import build
 import subprocess
 import os
 
+class BinaryDistribution(Distribution):
+  def has_ext_modules(self):
+    return True
+
+
 class SconsBuild(Command):
   def initialize_options(self) -> None:
     pass
@@ -12,14 +17,16 @@ def finalize_options(self) -> None:
 
   def run(self) -> None:
     scons_flags = '' if 'BUILD_TESTS' in os.environ else '--minimal'
-    subprocess.run([f"scons {scons_flags} -j$(nproc)"], shell=True).check_returncode()
+    subprocess.run([f"scons {scons_flags} -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode()
+
 
 class CustomBuild(build):
   sub_commands = [('scons_build', None)] + build.sub_commands
 
 setup(
     packages = ["msgq", "msgq.visionipc"],
-    package_data={'msgq': ['**/*.cc', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.so']},
+    package_data={'': ['**/*.cc', '**/*.c', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.py', '**/*.so', '**/*.npy']},
     include_package_data=True,
-    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}
+    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild},
+    distclass=BinaryDistribution,
     )

From 70983a5dc3220679a1e77a554c37f04d6922e5d1 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 21:17:30 -0700
Subject: [PATCH 15/25] no big sus

---
 .github/workflows/wheels.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index b96a2e514..ce5700af2 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -31,7 +31,8 @@ jobs:
         CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
         CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
         CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
-        CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package}
+        CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
+        MACOSX_DEPLOYMENT_TARGET: "14.0"
 
 
     - uses: actions/setup-python@v5

From 3c07c5a364344c230510ade894c7476fb4ffe4c5 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 21:23:02 -0700
Subject: [PATCH 16/25] test

---
 .github/workflows/wheels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index ce5700af2..353737c65 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -31,7 +31,7 @@ jobs:
         CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
         CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
         CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
-        CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
+        #CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
         MACOSX_DEPLOYMENT_TARGET: "14.0"
 
 

From aa6c09d888d090bcb5d591aa194d11189344b7cc Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:06:24 -0700
Subject: [PATCH 17/25] better

---
 .github/workflows/tests.yml    | 78 +++++++++++++++++++++++++---------
 .github/workflows/wheels.yml   |  4 +-
 scripts/macos_dependencies.sh  | 11 +++++
 scripts/ubuntu_dependencies.sh | 10 +++++
 4 files changed, 81 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d684c0489..4af04977f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -3,24 +3,75 @@ name: tests
 on: [push, pull_request]
 
 jobs:
-  unit_tests:
-    name: unit tests
+
+  static_analysis:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Installing requirements
+      run:
+        python -m pip install -r requirements.txt
+    - name: Installing ubuntu requirements
+      run: "sudo apt-get install -y cppcheck"
+    - name: Static analysis
+      run: pre-commit run --all
+
+  scons_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+        python: ["3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Installing python requirements
+      run:
+        python -m pip install -r requirements.txt
+    - name: Installing ubuntu requirements
+      if: ${{startsWith(matrix.os, 'ubuntu')}}
+      run:
+        scripts/ubuntu_dependencies.sh
+    - name: Installing macos requirements
+      if: ${{startsWith(matrix.os, 'macos')}}
+      run:
+        scripts/macos_dependencies.sh
+    - name: Building
+      run: scons -j$(nproc || sysctl -n hw.logicalcpu)
+
+
+  full_tests:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
+        os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+        python: ["3.11", "3.12"]
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
-        python-version: ["3.11", "3.12"]
+        exclude:
+          - os: "macos-14"
+            backend: "MSGQ"
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python }}
+    - name: Installing ubuntu requirements
+      if: ${{startsWith(matrix.os, 'ubuntu')}}
+      run:
+        scripts/ubuntu_dependencies.sh
+    - name: Installing macos requirements
+      if: ${{startsWith(matrix.os, 'macos')}}
+      run:
+        scripts/macos_dependencies.sh
     - name: Building msgq
       run: |
-        export BUILD_TESTS=1
-        scripts/install_dependencies.sh
-        pip3 install --break-system-packages .[dev]
+        pip3 install -e .[dev]
     - name: Python tests
       run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: C++ tests
@@ -31,16 +82,3 @@ jobs:
         msgq/visionipc/test_runner
     - name: Upload coverage
       run: "bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
-
-  static_analysis:
-    name: static analysis
-    runs-on: ubuntu-24.04
-    steps:
-    - uses: actions/checkout@v3
-    - name: Building msgq
-      run: |
-        scripts/install_dependencies.sh
-        pip3 install --break-system-packages .[dev]
-    - name: Static analysis
-      # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
-      run: unset PYTHONWARNINGS && pre-commit run --all
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 353737c65..12e26abc8 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -27,8 +27,8 @@ jobs:
       env:
         CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
         CIBW_ARCHS: "${{ matrix.platform.arch }}"
-        CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
-        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+        CIBW_BEFORE_ALL_LINUX: "bash NO_CATCH2=1 {project}/scripts/manylinux_dependencies.sh"
+        CIBW_BEFORE_BUILD_MACOS: "bash NO_CATCH2=1 {project}/scripts/macos_dependencies.sh"
         CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
         CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
         #CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh
index f0e9dc597..d5e5416c5 100755
--- a/scripts/macos_dependencies.sh
+++ b/scripts/macos_dependencies.sh
@@ -1,6 +1,17 @@
 #!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+ROOT=$DIR/../
+
 brew bundle --file=- <<-EOS
 brew "zeromq"
 cask "gcc-arm-embedded"
 brew "gcc@13"
 EOS
+
+if [[ -n "$NO_CATCH2" ]]; then
+  cd /tmp
+  git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
+  cd Catch2
+  mv single_include/* "$ROOT"
+  rm -rf Catch2
+fi
diff --git a/scripts/ubuntu_dependencies.sh b/scripts/ubuntu_dependencies.sh
index 88314f593..8b994c9e9 100755
--- a/scripts/ubuntu_dependencies.sh
+++ b/scripts/ubuntu_dependencies.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+ROOT=$DIR/../
 
 if [[ ! $(id -u) -eq 0 ]]; then
   if [[ -z $(which sudo) ]]; then
@@ -10,3 +12,11 @@ fi
 
 $SUDO apt-get update
 $SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck
+
+if [[ -n "$NO_CATCH2" ]]; then
+  cd /tmp
+  git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
+  cd Catch2
+  mv single_include/* "$ROOT"
+  rm -rf Catch2
+fi

From 00c41c6c2e9a7133223555659bbe365bbc7ca5b8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:09:14 -0700
Subject: [PATCH 18/25] better

---
 scripts/macos_dependencies.sh  | 2 +-
 scripts/ubuntu_dependencies.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh
index d5e5416c5..66a0c2bd8 100755
--- a/scripts/macos_dependencies.sh
+++ b/scripts/macos_dependencies.sh
@@ -8,7 +8,7 @@ cask "gcc-arm-embedded"
 brew "gcc@13"
 EOS
 
-if [[ -n "$NO_CATCH2" ]]; then
+if [[ -z "$NO_CATCH2" ]]; then
   cd /tmp
   git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
   cd Catch2
diff --git a/scripts/ubuntu_dependencies.sh b/scripts/ubuntu_dependencies.sh
index 8b994c9e9..855a937fe 100755
--- a/scripts/ubuntu_dependencies.sh
+++ b/scripts/ubuntu_dependencies.sh
@@ -13,7 +13,7 @@ fi
 $SUDO apt-get update
 $SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck
 
-if [[ -n "$NO_CATCH2" ]]; then
+if [[ -z "$NO_CATCH2" ]]; then
   cd /tmp
   git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
   cd Catch2

From d6d4e9ecb83986f9fd65c6275511f3d644ed1581 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:21:20 -0700
Subject: [PATCH 19/25] fix

---
 .github/workflows/wheels.yml | 4 ++--
 setup.py                     | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 12e26abc8..353737c65 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -27,8 +27,8 @@ jobs:
       env:
         CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
         CIBW_ARCHS: "${{ matrix.platform.arch }}"
-        CIBW_BEFORE_ALL_LINUX: "bash NO_CATCH2=1 {project}/scripts/manylinux_dependencies.sh"
-        CIBW_BEFORE_BUILD_MACOS: "bash NO_CATCH2=1 {project}/scripts/macos_dependencies.sh"
+        CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
+        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
         CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
         CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
         #CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
diff --git a/setup.py b/setup.py
index d3198fb50..fb0eafd08 100644
--- a/setup.py
+++ b/setup.py
@@ -16,8 +16,7 @@ def finalize_options(self) -> None:
     pass
 
   def run(self) -> None:
-    scons_flags = '' if 'BUILD_TESTS' in os.environ else '--minimal'
-    subprocess.run([f"scons {scons_flags} -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode()
+    subprocess.run([f"scons --minimal -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode()
 
 
 class CustomBuild(build):

From 52360ab8fabc6da235059709d2f43dbab88191fd Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:24:26 -0700
Subject: [PATCH 20/25] brew not in GH???

---
 scripts/macos_dependencies.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh
index 66a0c2bd8..950d2ffb3 100755
--- a/scripts/macos_dependencies.sh
+++ b/scripts/macos_dependencies.sh
@@ -2,6 +2,18 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 ROOT=$DIR/../
 
+if [[ $(command -v brew) == "" ]]; then
+  /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+
+  if [[ $ARCH == "x86_64" ]]; then
+      echo 'eval "$(/usr/local/homebrew/bin/brew shellenv)"' >> $RC_FILE
+      eval "$(/usr/local/homebrew/bin/brew shellenv)"
+  else
+      echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> $RC_FILE
+      eval "$(/opt/homebrew/bin/brew shellenv)"
+  fi
+fi
+
 brew bundle --file=- <<-EOS
 brew "zeromq"
 cask "gcc-arm-embedded"

From ee221fd96b3c6e20592c11df332c12fae8dc5c4e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:27:16 -0700
Subject: [PATCH 21/25] fix mac

---
 scripts/macos_dependencies.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh
index 950d2ffb3..746370ad1 100755
--- a/scripts/macos_dependencies.sh
+++ b/scripts/macos_dependencies.sh
@@ -1,10 +1,22 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-ROOT=$DIR/../
+ROOT="$(cd $DIR/../ && pwd)"
+ARCH=$(uname -m)
+
+if [[ $SHELL == "/bin/zsh" ]]; then
+  RC_FILE="$HOME/.zshrc"
+elif [[ $SHELL == "/bin/bash" ]]; then
+  RC_FILE="$HOME/.bash_profile"
+fi
 
+# Install brew if required
 if [[ $(command -v brew) == "" ]]; then
+  echo "Installing Homebrew"
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+  echo "[ ] installed brew t=$SECONDS"
 
+  # make brew available now
   if [[ $ARCH == "x86_64" ]]; then
       echo 'eval "$(/usr/local/homebrew/bin/brew shellenv)"' >> $RC_FILE
       eval "$(/usr/local/homebrew/bin/brew shellenv)"

From 56b29bae2eefd85f639441b0d0e0f7ae674b4f3f Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:29:05 -0700
Subject: [PATCH 22/25] ...

---
 .github/workflows/tests.yml   | 2 +-
 scripts/macos_dependencies.sh | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4af04977f..9bfccdacd 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,7 +46,7 @@ jobs:
 
 
   full_tests:
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh
index 746370ad1..bac87f705 100755
--- a/scripts/macos_dependencies.sh
+++ b/scripts/macos_dependencies.sh
@@ -12,11 +12,8 @@ fi
 
 # Install brew if required
 if [[ $(command -v brew) == "" ]]; then
-  echo "Installing Homebrew"
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
-  echo "[ ] installed brew t=$SECONDS"
 
-  # make brew available now
   if [[ $ARCH == "x86_64" ]]; then
       echo 'eval "$(/usr/local/homebrew/bin/brew shellenv)"' >> $RC_FILE
       eval "$(/usr/local/homebrew/bin/brew shellenv)"

From b75c9e7c1b6e66ee78d3df318a8a3593c89b0a5a Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:37:26 -0700
Subject: [PATCH 23/25] fix mac

---
 .github/workflows/tests.yml  | 9 +--------
 .github/workflows/wheels.yml | 3 ++-
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 9bfccdacd..2c0c61553 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -49,26 +49,19 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+        os: ["ubuntu-20.04", "ubuntu-24.04"]
         python: ["3.11", "3.12"]
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
-        exclude:
-          - os: "macos-14"
-            backend: "MSGQ"
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
     - name: Installing ubuntu requirements
-      if: ${{startsWith(matrix.os, 'ubuntu')}}
       run:
         scripts/ubuntu_dependencies.sh
     - name: Installing macos requirements
-      if: ${{startsWith(matrix.os, 'macos')}}
-      run:
-        scripts/macos_dependencies.sh
     - name: Building msgq
       run: |
         pip3 install -e .[dev]
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 353737c65..8184d4614 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -31,7 +31,8 @@ jobs:
         CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
         CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
         CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
-        #CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
+        CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
+        CIBW_TEST_SKIP: "*-macosx_arm64"
         MACOSX_DEPLOYMENT_TARGET: "14.0"
 
 

From 995608b8524692b0ed6915c60f96e788f8dc7eb0 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:39:24 -0700
Subject: [PATCH 24/25] fix the fix

---
 .github/workflows/tests.yml  | 1 -
 .github/workflows/wheels.yml | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2c0c61553..199153d1d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -61,7 +61,6 @@ jobs:
     - name: Installing ubuntu requirements
       run:
         scripts/ubuntu_dependencies.sh
-    - name: Installing macos requirements
     - name: Building msgq
       run: |
         pip3 install -e .[dev]
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 8184d4614..1a7feaba1 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -37,7 +37,7 @@ jobs:
 
 
     - uses: actions/setup-python@v5
-      if: ${{ matrix.platform.arch }} != 'aarch64'
+      if: ${{ matrix.platform.arch != 'aarch64' }}
       with:
         python-version: ${{ matrix.python.py }}
 

From 9b8f55674d5913bf3a829099e53255647fd229ab Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 19 Jul 2024 22:45:10 -0700
Subject: [PATCH 25/25] clean

---
 .github/workflows/wheels.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 1a7feaba1..57ef0fa3d 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -31,18 +31,15 @@ jobs:
         CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
         CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
         CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
-        CIBW_TEST_COMMAND: "pip install -r {project}/requirements.txt && pytest {package}"
-        CIBW_TEST_SKIP: "*-macosx_arm64"
         MACOSX_DEPLOYMENT_TARGET: "14.0"
 
-
     - uses: actions/setup-python@v5
       if: ${{ matrix.platform.arch != 'aarch64' }}
       with:
         python-version: ${{ matrix.python.py }}
 
     - name: Installing the wheel
-      if: ${{ matrix.platform.arch }} != 'aarch64'
+      if: ${{ matrix.platform.arch != 'aarch64' }}
       run: |
         pip install --break-system-packages ./wheelhouse/*.whl