diff --git a/.github/actions/install-colint/action.yaml b/.github/actions/install-colint/action.yaml deleted file mode 100644 index 6a9d9f5..0000000 --- a/.github/actions/install-colint/action.yaml +++ /dev/null @@ -1,16 +0,0 @@ -runs: - using: 'composite' - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10.12' - - - name: Install colint from github repository - shell: bash - run: | - pip install --upgrade pip - pip install git+https://github.com/secomind/colint.git diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index fc5e39d..3458b7a 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,60 +1,18 @@ name: Lint Checks - on: [pull_request, push] - jobs: - sort_libraries: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Install Colint - uses: ./.github/actions/install-colint - - - name: Run Sort Libraries Check - run: | - colint sort-libraries . --check --config-file ./lint - - code_format: + lint: runs-on: ubuntu-latest - steps: - name: Checkout Code uses: actions/checkout@v3 - - - name: Install Colint - uses: ./.github/actions/install-colint - - name: Run Code Format Check - run: | - colint code-format . --check --config-file ./lint - - grammar_check: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Install Colint - uses: ./.github/actions/install-colint - - - name: Run Grammar Check - run: | - colint grammar-check . --check --config-file ./lint - - newline_fix: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Install Colint - uses: ./.github/actions/install-colint + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10.12" - - name: Run Newline Fix Check + - name: Run linting run: | - colint newline-fix . --check --config-file ./lint + make setup-env + make lint CHECK=1 diff --git a/.github/workflows/push-to-registry.yaml b/.github/workflows/push-to-registry.yaml index 510357f..5729403 100644 --- a/.github/workflows/push-to-registry.yaml +++ b/.github/workflows/push-to-registry.yaml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.10.12' + python-version: "3.10.12" - name: Install Build Dependencies run: | diff --git a/.gitignore b/.gitignore index 02df1b9..bb1e32f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build *.jpg .DS_Store .venv/ +.venv-lint/ .vscode *.eim .act-secrets diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a67710e --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +VENV_DIR := $(abspath .venv-lint) +PIP := $(VENV_DIR)/bin/pip +PYTHON := $(VENV_DIR)/bin/python +RUFF := $(VENV_DIR)/bin/ruff +DPRINT := $(VENV_DIR)/bin/dprint +TARGET_DIRS ?= . +CHECK ?= 0 + +.PHONY: clean +clean: + git clean -Xdf --exclude "!.env" + +.PHONY: setup-venv +setup-venv: + @if [ ! -d "$(VENV_DIR)" ]; then \ + echo "Creating virtual environment for linters..."; \ + python3 -m venv $(VENV_DIR); \ + $(PIP) install --upgrade pip; \ + $(PIP) install ruff; \ + fi + +.PHONY: setup-dprint +setup-dprint: setup-venv + @if [ ! -f "$(DPRINT)" ]; then \ + echo "Installing dprint to $(VENV_DIR)/bin..."; \ + mkdir -p $(VENV_DIR)/bin; \ + curl -fsSL https://dprint.dev/install.sh | DPRINT_INSTALL=$(VENV_DIR) sh; \ + fi + +.PHONY: setup-env +setup-env: setup-venv setup-dprint + +.PHONY: ruff +ruff: setup-venv + @if [ "$(CHECK)" = "1" ]; then \ + echo "Running Ruff in check mode..."; \ + $(RUFF) check $(TARGET_DIRS); \ + LINT_STATUS=$$?; \ + $(RUFF) format $(TARGET_DIRS) --check; \ + FMT_STATUS=$$?; \ + if [ $$LINT_STATUS -ne 0 ] || [ $$FMT_STATUS -ne 0 ]; then exit 1; fi; \ + else \ + echo "Running Ruff and fixing issues..."; \ + $(RUFF) check $(TARGET_DIRS) --fix; \ + LINT_STATUS=$$?; \ + $(RUFF) format $(TARGET_DIRS); \ + FMT_STATUS=$$?; \ + if [ $$LINT_STATUS -ne 0 ] || [ $$FMT_STATUS -ne 0 ]; then exit 1; fi; \ + fi + +.PHONY: dprint +dprint: setup-dprint + @if [ "$(CHECK)" = "1" ]; then \ + echo "Running dprint in check mode..."; \ + $(DPRINT) check $(TARGET_DIRS); \ + else \ + echo "Running dprint and formatting..."; \ + $(DPRINT) fmt $(TARGET_DIRS); \ + fi + +.PHONY: lint +lint: ruff dprint diff --git a/README.md b/README.md index 5164c15..f91abec 100644 --- a/README.md +++ b/README.md @@ -20,34 +20,34 @@ This library lets you run machine learning models and collect sensor data on Lin 1. Install a recent version of [Python 3](https://www.python.org/downloads/) and `pip` tools. 1. Install the SDK: - **Raspberry Pi** + **Raspberry Pi** - ``` - $ sudo apt-get install libatlas-base-dev libportaudio0 libportaudio2 libportaudiocpp0 portaudio19-dev libopenjp2-7 libgtk-3-0 libswscale-dev libavformat58 libavcodec58 - $ pip3 install edge_impulse_linux -i https://pypi.python.org/simple - ``` + ``` + $ sudo apt-get install libatlas-base-dev libportaudio0 libportaudio2 libportaudiocpp0 portaudio19-dev libopenjp2-7 libgtk-3-0 libswscale-dev libavformat58 libavcodec58 + $ pip3 install edge_impulse_linux -i https://pypi.python.org/simple + ``` - **Other platforms** + **Other platforms** - ``` - $ pip3 install edge_impulse_linux - ``` + ``` + $ pip3 install edge_impulse_linux + ``` 1. Clone this repository to get the examples: - ``` - $ git clone https://github.com/edgeimpulse/linux-sdk-python - ``` + ``` + $ git clone https://github.com/edgeimpulse/linux-sdk-python + ``` -4. Install pip dependencies: +1. Install pip dependencies: - ``` - $ pip3 install -r requirements.txt - ``` + ``` + $ pip3 install -r requirements.txt + ``` - For the computer vision examples you'll want `opencv-python>=4.5.1.48,<5` - Note on macOS on apple silicon, you will need to use a later version, - 4.10.0.84 tested and installs cleanly + For the computer vision examples you'll want `opencv-python>=4.5.1.48,<5` + Note on macOS on apple silicon, you will need to use a later version, + 4.10.0.84 tested and installs cleanly ## Collecting data @@ -69,31 +69,32 @@ To classify data (whether this is from the camera, the microphone, or a custom s 1. Install the [Edge Impulse for Linux CLI](https://docs.edgeimpulse.com/docs/edge-impulse-for-linux). 1. Download the model file via: - ``` - $ edge-impulse-linux-runner --download modelfile.eim - ``` + ``` + $ edge-impulse-linux-runner --download modelfile.eim + ``` - This downloads the file into `modelfile.eim`. (Want to switch projects? Add `--clean`) + This downloads the file into `modelfile.eim`. (Want to switch projects? Add `--clean`) Then you can start classifying realtime sensor data. We have examples for: -* [Audio](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/audio/classify.py) - grabs data from the microphone and classifies it in realtime. -* [Camera](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify.py) - grabs data from a webcam and classifies it in realtime. -* [Camera (full frame)](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify-full-frame.py) - grabs data from a webcam and classifies it twice (once cut from the left, once cut from the right). This is useful if you have a wide-angle lense and don't want to miss any events. -* [Still image](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify-image.py) - classifies a still image from your hard drive. -* [Video](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify-video.py) - grabs frames from a video source from your hard drive and classifies it. -* [Custom data](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/custom/classify.py) - classifies custom sensor data. +- [Audio](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/audio/classify.py) - grabs data from the microphone and classifies it in realtime. +- [Camera](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify.py) - grabs data from a webcam and classifies it in realtime. +- [Camera (full frame)](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify-full-frame.py) - grabs data from a webcam and classifies it twice (once cut from the left, once cut from the right). This is useful if you have a wide-angle lense and don't want to miss any events. +- [Still image](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify-image.py) - classifies a still image from your hard drive. +- [Video](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/image/classify-video.py) - grabs frames from a video source from your hard drive and classifies it. +- [Custom data](https://github.com/edgeimpulse/linux-sdk-python/blob/master/examples/custom/classify.py) - classifies custom sensor data. ## Troubleshooting ### Collecting print out from the model To display the logging messages (ie, you may be used to in other deployments), init the runner like so + ``` # model_info = runner.init(debug=True) # to get debug print out ``` -This will pipe stdout and stderr into the same of your own process +This will pipe stdout and stderr into the same of your own process ### [Errno -9986] Internal PortAudio error (macOS) diff --git a/dprint.json b/dprint.json new file mode 100644 index 0000000..7f94e5e --- /dev/null +++ b/dprint.json @@ -0,0 +1,33 @@ +{ + "yaml": { + "formatComments": true, + "braceSpacing": false, + "preferSingleLine": true + }, + "json": { + "lineWidth": 80 + }, + "toml": { + "lineWidth": 80 + }, + "markdown": { + "emphasisKind": "asterisks" + }, + "markup": { + "formatComments": true + }, + "includes": [ + "**/*.{yaml,yml}", + "**/*.json", + "**/*.toml", + "**/*.md", + "**/*.html" + ], + "plugins": [ + "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm", + "https://plugins.dprint.dev/toml-0.7.0.wasm", + "https://plugins.dprint.dev/json-0.21.3.wasm", + "https://plugins.dprint.dev/markdown-0.22.1.wasm", + "https://plugins.dprint.dev/g-plane/markup_fmt-v0.27.3.wasm" + ] +} diff --git a/examples/custom/collect.py b/examples/custom/collect.py index 2bf06ec..317510f 100644 --- a/examples/custom/collect.py +++ b/examples/custom/collect.py @@ -31,13 +31,11 @@ freq = 1000 / INTERVAL_MS values_list = [] for i in range(2 * int(round(freq, 0))): - values_list.append( - [ - math.sin(i * 0.1) * 10, - math.cos(i * 0.1) * 10, - (math.sin(i * 0.1) + math.cos(i * 0.1)) * 10, - ] - ) + values_list.append([ + math.sin(i * 0.1) * 10, + math.cos(i * 0.1) * 10, + (math.sin(i * 0.1) + math.cos(i * 0.1)) * 10, + ]) data = { "protected": { diff --git a/examples/image/classify-video.py b/examples/image/classify-video.py index c7861eb..6702853 100644 --- a/examples/image/classify-video.py +++ b/examples/image/classify-video.py @@ -76,7 +76,6 @@ def getFrame(sec): img = getFrame(sec) while img.size != 0: - # imread returns images in BGR format, so we need to convert to RGB img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) diff --git a/examples/image/set-thresholds.py b/examples/image/set-thresholds.py index b26b995..0f53417 100755 --- a/examples/image/set-thresholds.py +++ b/examples/image/set-thresholds.py @@ -108,12 +108,10 @@ def main(argv): print("Invalid input, please enter a number") # dynamically override the thresold from 0.2 -> 0.8 - runner.set_threshold( - { - "id": threshold_id, - "min_score": new_threshold, - } - ) + runner.set_threshold({ + "id": threshold_id, + "min_score": new_threshold, + }) res = runner.classify(features) print("classify response", json.dumps(res, indent=4)) diff --git a/lint.toml b/lint.toml deleted file mode 100644 index fc64287..0000000 --- a/lint.toml +++ /dev/null @@ -1,46 +0,0 @@ -[tool.flake8] - -extend-ignore = [ - # Styling rules already handled by black - "E1", - "W1", - "E2", - "W2", - "E3", - "W3", - "E5", - "W503", - "W504", - # No import at top of file: conflict with certain Data Science Packages - "E402", - # Do not use bare except - "E722", - #Docstring - "D100", - "D101", - "D102", - "D103", - "D104", - "D105", - "D106", - "D107", - "D202", - "D212" -] - -per-file-ignores = ''' - __init__.py:F401 -''' - -max-complexity = 20 - -docstring-convention = "google" - -max-doc-length = 160 - -[tool.isort] -profile = "black" - -[tool.black] -line-length = 88 -preview = true diff --git a/pyproject.toml b/pyproject.toml index 2e3e03a..6fb72e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,30 +9,30 @@ dynamic = ["readme"] requires-python = ">=3.10" description = "Python runner for real-time ML classification" authors = [ - { name = "EdgeImpulse Inc.", email = "hello@edgeimpulse.com" }, + { name = "EdgeImpulse Inc.", email = "hello@edgeimpulse.com" }, ] license = { file = "LICENSE" } keywords = ["edge-impulse", "machine-learning", "embedded", "linux"] classifiers = [ - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", ] dependencies = [] [project.optional-dependencies] audio = [ - "numpy>=1.19.5,<3", - "pyaudio>=0.2.11,<0.3", + "numpy>=1.19.5,<3", + "pyaudio>=0.2.11,<0.3", ] image = [ - "opencv-python-headless>=4.5.1", - "psutil>=5.8.0" + "opencv-python-headless>=4.5.1", + "psutil>=5.8.0", ] all = [ - "edge_impulse_linux[audio]", - "edge_impulse_linux[image]", + "edge_impulse_linux[audio]", + "edge_impulse_linux[image]", ] [project.urls] @@ -46,3 +46,38 @@ package-dir = { "" = "src" } [tool.setuptools.dynamic] readme = { file = ["README.md"], content-type = "text/markdown" } +[tool.ruff] +line-length = 88 +target-version = "py310" + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +preview = true + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "D", "C90"] +ignore = [ + "E402", # Module level import not at top of file + "E722", # Do not use bare except + "D100", + "D101", + "D102", + "D103", + "D104", + "D105", + "D106", + "D107", # Missing docstrings + "D202", + "D212", + "E501", # Line too long +] + +[tool.ruff.lint.mccabe] +max-complexity = 22 + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] # Unused import + +[tool.ruff.lint.pydocstyle] +convention = "google" diff --git a/src/edge_impulse_linux/runner.py b/src/edge_impulse_linux/runner.py index ee17669..56a207f 100755 --- a/src/edge_impulse_linux/runner.py +++ b/src/edge_impulse_linux/runner.py @@ -61,6 +61,7 @@ def __init__(self, model_path: str, timeout: int = 30, allow_shm=True): Args: model_path (str): Path to the executable model file. timeout (int): timeout in seconds. + allow_shm (bool, optional): Allow shared memory usage. Defaults to True. """ self._model_path = model_path self._tempdir = None @@ -145,19 +146,17 @@ def init(self, debug=False): # python does not want the leading slash shm_name = shm_name.lstrip("/") shm = shared_memory.SharedMemory(name=shm_name) - self._freeform_output_shm.append( - { - "index": output_shm["index"], - "shm": shm, - "type": output_shm["type"], - "elements": output_shm["elements"], - "array": np.ndarray( - (output_shm["elements"],), - dtype=np.float32, - buffer=shm.buf, - ), - } - ) + self._freeform_output_shm.append({ + "index": output_shm["index"], + "shm": shm, + "type": output_shm["type"], + "elements": output_shm["elements"], + "array": np.ndarray( + (output_shm["elements"],), + dtype=np.float32, + buffer=shm.buf, + ), + }) return self._hello_resp @@ -245,16 +244,16 @@ def set_threshold(self, obj): return self.send_msg(msg) def send_msg(self, msg): - """Classifies the given data using the model. + """Sends a message to the runner process. Args: - data (any): The data to classify. + msg (dict): The message payload to send. Returns: - dict: The classification response. + dict: The response from the runner. Raises: - Exception: If classification fails. + Exception: If communication fails or returns an error. """ if not self._client: raise Exception("ImpulseRunner is not initialized (call init())")