diff --git a/README.md b/README.md index 260282a72902ebc701104d50f621cbc6c0b020db..56bdfc26e50a7f6c0a2940e74ca02f1f035e28b8 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,13 @@ - Jupyter-lab is served on `0.0.0.0:8888` with an empty password. This of course has security implications and should be discussed - Install the following python3 packages: - numpy - - sklearn + - scikit-learn (aka. sklearn) - matplotlib - pandas - Pillow -- On the desktop version of Raspberry Pi OS, `pil` and `numpy` are preinstalled from the APT repository and will be removed in favour the `pip3` versions. This behavior should be discussed, as there might be further problems caused by incompatible packages depending on the use of either APT or pip python packages + - seaborn + - plotly +- All python packages are installed into a virtual environment at `~/.ml-venv`. This also includes jupyter-lab which allows for using the default python3 kernel and still having access to all dependencies ### The project is structured as follows: - [local.yml](./local.yml): The main playbook file, can be seen as the entry point @@ -22,7 +24,6 @@ - [ml-python-packages](./roles/ml-python-packages/tasks/main.yml): Install the machine learning python packages - # Using the ansible playbook on Raspberry Pi OS Steps to run the ansible script on a fresh (or not fresh) installation of raspberry pi os: @@ -33,16 +34,15 @@ sudo apt update sudo apt install -y --no-install-recommends ansible git # Pull and run the ansible scripts from git -# NOTE: Since the repository is currently private, the gitlab login prompt will shown. When using -# 2FA for gitlab the password login is not possible and an access token is needed. This will be -# fixed in the future -sudo ansible-pull -U https://code.fbi.h-da.de/pse-ai-at-the-edge/raspberry-pi-setup.git +ansible-pull -U https://code.fbi.h-da.de/pse-ai-at-the-edge/raspberry-pi-setup.git ``` # Notes -## 64-Bit Raspberry Pi OS (Desktop) +## Raspberry Pi OS Version and Architecture +This current version of the playbook will install a prebuilt `tensorflow-2.7` wheel that only works on `Python3.9` and the `Aarch64` architecture. +That means for now the playbook **only works on Raspberry Pi OS 11 64-bit** -There were major problems when trying to install the software on 64-Bit Pi OS. Multiple python packages started to compile C/C++ library code on the device which caused extreme increases of installation times and full failures due to the 1GB RAM on model 3B beeing completely used up. Jupyter-lab also needed an additional dependency (`libffi-dev`) to compile its backend. This behavior occurred when installing packages using `pip3`, but not when using APT. Since some packages are not available, or too old on APT pip can't be fully avoided. +### Updating the playbook for 32-bit Raspberry Pi OS +Besides `tensorflow` all other dependencies will pretty much work regardless of architecture and python3 version. So to make this playbook work on 32-bit a decision must be made depending on the architecture. On 32-bit a different prebuilt `tensorflow` wheel must be used, as well as another `numpy` version. No prebuilt wheels were found yet for 32-bit in combination with `python3.9` (debian / raspberry pi os 11), only for `python3.7` which is not compatible with the current version of raspberry pi os. -This might be caused by those python libraries depending on C/C++ code which is either shipped precompile or build on device. In this case there might be a precompiled version available for armv7 (32-bit) but not for aarch64 (64-bit), causing the C/C++ code to be compiled on device with the 64-bit OS and not with the 32-bit OS. diff --git a/local.yml b/local.yml index 466be3f4870aa654eb094a3a7ba39099b2b035ed..ae4b6979bbf82dd0a082144ae34a39b13c22d888 100644 --- a/local.yml +++ b/local.yml @@ -1,33 +1,32 @@ ---- -- hosts: localhost - vars: - target_user: pi - jupyter_service: true - ignore_python_version: false - - pre_tasks: - - name: Check python version - fail: - msg: Python version is not supported. Set ignore_python_version to true to ignore this. - when: (not ignore_python_version) and not (ansible_python_version is version('3.7.0', '>=') and ansible_python_version is version('3.10.0', '<')) - - # - name: Upgrade packages - # become: true - # apt: - # upgrade: true - # update_cache: true - # force_apt_get: true - - # - name: Install pip3 - # become: true - # apt: - # update_cache: true - # name: - # - python3-pip - # - libffi-dev # This is needed for jupyter on aarch64 to compile its C backend - # state: present - - roles: - - jupyter-lab - - ml-python-packages - - misc-applications +--- +- hosts: localhost + vars: + target_user: pi + jupyter_service: true + ignore_python_version: false + ignore_architecture: faslse + + pre_tasks: + - name: Check python version + fail: + msg: "Python version is not supported. Expected 3.9.x . Set ignore_python_version to true to ignore this. Running on a different python version is not supported and will most likely not work." + when: (not ignore_python_version) and not (ansible_python_version is version('3.9.0', '>=') and ansible_python_version is version('3.10.0', '<')) + + - name: Check architecture + fail: + msg: "CPU Architecture not compatible. Expected aarch64 (arm 64-bit). Set ignore_architecture to true to ignore this. Running on a different architecture is not supported and will most likely not work." + when: not ignore_architecture and ansible_architecture != 'aarch64' + + - name: Update apt cache & install base requirements + become: true + apt: + update_cache: true + state: present + name: + - python3-pip + - python3-venv + + roles: + - jupyter-lab + - ml-python-packages + - misc-applications diff --git a/roles/jupyter-lab/files/jupyter.service b/roles/jupyter-lab/files/jupyter.service index bdbf4478cc954609169ed9a7aa7ed1ac12b5abee..462c8df0c68c4ad8c325ce3b3c0e5ff7ede0bc80 100644 --- a/roles/jupyter-lab/files/jupyter.service +++ b/roles/jupyter-lab/files/jupyter.service @@ -1,15 +1,15 @@ -[Unit] -Description=Jupyter Lab - -[Service] -Type=simple -PIDFile=/run/jupyter.pid -# Password: [empty string] -ExecStart=/home/{{ target_user }}/.ml-venv/bin/jupyter-lab --ip="0.0.0.0" --notebook-dir=/home/{{ target_user }}/notebooks --no-browser --NotebookApp.password='sha1:9a2d316959ac:843b251c27024afb46174ce40ce0ebebcf29217b' -User={{ target_user }} -Group={{ target_user }} -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target +[Unit] +Description=Jupyter Lab + +[Service] +Type=simple +PIDFile=/run/jupyter.pid +# Password: [empty string] +ExecStart=/home/{{ target_user }}/.ml-venv/bin/jupyter-lab --ip="0.0.0.0" --notebook-dir=/home/{{ target_user }}/notebooks --no-browser --NotebookApp.password='sha1:9a2d316959ac:843b251c27024afb46174ce40ce0ebebcf29217b' +User={{ target_user }} +Group={{ target_user }} +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/roles/jupyter-lab/tasks/main.yml b/roles/jupyter-lab/tasks/main.yml index f5b9df263bb95ed9ffc86819281131930634a788..c98db86ffef390ccc623852ad1eabc69c1c286cc 100644 --- a/roles/jupyter-lab/tasks/main.yml +++ b/roles/jupyter-lab/tasks/main.yml @@ -1,45 +1,35 @@ ---- -# roles/jupyter-lab - -- name: Notebooks dir - file: - path: "/home/{{ target_user }}/notebooks" - state: directory - owner: "{{ target_user }}" - group: "{{ target_user }}" - -# This should be optimized so that only one check / install is needed -# Still we should make sure that pip3 is actually installed before trying to use it -- name: Install dependecies - become: true - apt: - name: - - python3-pip - - python3-venv - state: present - update_cache: true - -- name: Install jupyter-lab - pip: - virtualenv_command: /usr/bin/python3 -m venv - virtualenv: "/home/{{ target_user }}/.ml-venv" - name: jupyterlab - state: present - -- name: Install systemd service - become: true - template: - src: files/jupyter.service - dest: /etc/systemd/system/jupyter.service - mode: '0644' - when: jupyter_service - -- name: Start Jupyter-Lab service - become: true - systemd: - name: jupyter - state: started - enabled: true - daemon_reload: true - when: jupyter_service - +--- +# roles/jupyter-lab + +- name: Notebooks dir + file: + path: "/home/{{ target_user }}/notebooks" + state: directory + owner: "{{ target_user }}" + group: "{{ target_user }}" + +- name: Install jupyter-lab + become: true + become_user: "{{ target_user }}" + pip: + virtualenv_command: /usr/bin/python3 -m venv + virtualenv: "/home/{{ target_user }}/.ml-venv" + name: jupyterlab==3.2.4 + state: present + +- name: Install systemd service + become: true + template: + src: files/jupyter.service + dest: /etc/systemd/system/jupyter.service + mode: '0644' + when: jupyter_service + +- name: Start Jupyter-Lab service + become: true + systemd: + name: jupyter + state: started + enabled: true + daemon_reload: true + when: jupyter_service diff --git a/roles/misc-applications/tasks/main.yml b/roles/misc-applications/tasks/main.yml index 13052db5da9823aad7b6966da79286ee02979e79..3eab41574767b21fbd141828714dff8e308b2eb8 100644 --- a/roles/misc-applications/tasks/main.yml +++ b/roles/misc-applications/tasks/main.yml @@ -1,11 +1,10 @@ ---- -# roles/misc-applications - -- name: Install useful cli applications - become: true - apt: - name: - - neovim - - tmux - state: present - +--- +# roles/misc-applications + +- name: Install useful cli applications + become: true + apt: + name: + - neovim + - tmux + state: present diff --git a/roles/ml-python-packages/files/requirements.txt b/roles/ml-python-packages/files/requirements.txt index 4a89ed45d5ecb42a6855f0f8a97963ee4d3cf356..290a36283f25e0d277ac804651f5d3a7adb7cf9a 100644 --- a/roles/ml-python-packages/files/requirements.txt +++ b/roles/ml-python-packages/files/requirements.txt @@ -3,4 +3,5 @@ matplotlib==3.4.* pandas==1.3.* Pillow==8.4.* scikit-learn==1.0.* - +seaborn==0.11.* +plotly==5.4.* diff --git a/roles/ml-python-packages/files/tensorflow-install.sh b/roles/ml-python-packages/files/tensorflow-install.sh new file mode 100644 index 0000000000000000000000000000000000000000..4158bb336e1753cbfc67678aaade72c646fdb8f9 --- /dev/null +++ b/roles/ml-python-packages/files/tensorflow-install.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +gdownload() { + URL=$1 + OUTFILE=$2 + COOKIES="/tmp/${RANDOM}.cookie" + + if [ -z "$URL" -o -z "$OUTFILE" ]; then + return 1 + fi + + # Get cookie with skip verification code + curl -sc "$COOKIES" "$URL" &> /dev/null + + # Extract skip verification code from cookie + CODE="$(awk '/_warning_/ {print $NF}' $COOKIES)" + + echo "Code: $CODE" + + # Download the actual file + echo "Using link: " "$URL&confirm=$CODE" + curl -Lb "$COOKIES" "$URL&confirm=$CODE" -o "$OUTFILE" + + # Cleanup cookie file + rm "$COOKIES" + + return $? +} + +install() { + DLFILE="/tmp/tensorflow-2.7.0-cp39-none-linux_aarch64.whl" + + # Try to download the wheel file from google drive + if gdownload "https://drive.google.com/uc?export=download&id=1D3R7bzkuFdY_gGTkSmnF9IbRNlj9wssB" "$DLFILE"; then + # Install the downloaded wheel + pip3 install "$DLFILE" + + # Save the pip3 install return code (0 => success) + PIPRC="$?" + + # Cleanup the downloaded wheel + rm "$DLFILE" + + return $PIPRC + fi + + return 1 +} + +install diff --git a/roles/ml-python-packages/tasks/main.yml b/roles/ml-python-packages/tasks/main.yml index ee1c3cf783e0c088e45077b4421e4ca39fe6ba75..84a9da7bf65dd084e31a36eeade276d0364ffdb4 100644 --- a/roles/ml-python-packages/tasks/main.yml +++ b/roles/ml-python-packages/tasks/main.yml @@ -1,34 +1,42 @@ ---- -# roles/ml-python-packages - -# This should be optimized so that only one check / install for pip3 is needed -# Still we should make sure that pip3 is actually installed before trying to use it -- name: Install dependecies - become: true - apt: - name: - - python3-pip - - libatlas-base-dev - state: present - update_cache: true - -- name: Copy requirements.txt to remote fs - copy: - src: files/requirements.txt - dest: /tmp/ml-requirements.txt - -- name: Install ml python packages - pip: - virtualenv_command: /usr/bin/python3 -m venv - requirements: /tmp/ml-requirements.txt - virtualenv: "/home/{{ target_user }}/.ml-venv" - # name: - # - numpy - # - sklearn - # - matplotlib - # - pandas - # - Pillow - # - torch - # - torchvision - # - fastai - # state: present +--- +# roles/ml-python-packages + +- name: Install APT dependecies + become: true + apt: + name: + - libatlas-base-dev + state: present + +- name: Copy requirements.txt to remote fs + copy: + src: files/requirements.txt + dest: /tmp/ml-requirements.txt + +- name: Install ml python packages + become: true + become_user: "{{ target_user }}" + pip: + virtualenv_command: /usr/bin/python3 -m venv + requirements: /tmp/ml-requirements.txt + virtualenv: "/home/{{ target_user }}/.ml-venv" + + + +- name: Check if tensorflow is installed (from wheel file) + shell: ". /home/{{ target_user }}/.ml-venv/bin/activate && pip3 freeze | grep -q 'tensorflow @ file'" + register: tfinstalled + changed_when: "tfinstalled.rc != 0" + failed_when: false + +- name: Copy tf install script to remote fs + copy: + src: files/tensorflow-install.sh + dest: /tmp/tensorflow-install.sh + when: "tfinstalled.rc != 0" + +- name: Download and install prebuilt tensorflow wheel + become: true + become_user: "{{ target_user }}" + shell: ". /home/{{ target_user }}/.ml-venv/bin/activate && bash /tmp/tensorflow-install.sh" + when: "tfinstalled.rc != 0"