diff --git a/.circleci/config.yml b/.circleci/_config.yml similarity index 100% rename from .circleci/config.yml rename to .circleci/_config.yml diff --git a/.github/workflows/build_backend.yml b/.github/workflows/build_backend.yml index 4eddddf0..568c391d 100644 --- a/.github/workflows/build_backend.yml +++ b/.github/workflows/build_backend.yml @@ -11,7 +11,7 @@ jobs: container: coderbot/coderbot-ci:3.9-bullseye-slim steps: - uses: actions/checkout@v3 # Checking out the repo - - run: pip install -r requirements_stub.txt + - run: pip install -r docker/stub/requirements.txt - run: | export PYTHONPATH=./coderbot:./stub:./test mkdir test-reports @@ -58,7 +58,7 @@ jobs: type=ref,event=pr # push event type=sha,enable=true,prefix=git-,format=short - - uses: actions/checkout@v2 # Checking out the repo + - uses: actions/checkout@v3 # Checking out the repo - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -74,9 +74,46 @@ jobs: uses: docker/build-push-action@v3 with: push: true + build-args: CODERBOT_VERSION=${{github.ref_name}}-${{github.sha}} platforms: linux/arm/v7 tags: ${{ steps.meta.outputs.tags }} context: . file: docker/Dockerfile cache-from: type=registry,ref=ghcr.io/coderbotorg/backend:latest cache-to: type=inline + + release-stub: + needs: [test] + runs-on: ubuntu-latest + steps: + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + # list of Docker images to use as base name for tags + images: ghcr.io/coderbotorg/backend + # generate Docker tags based on the following events/attributes + tags: | + # always latest + type=raw,value=stub-latest + - uses: actions/checkout@v3 # Checking out the repo + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v3 + with: + push: true + build-args: CODERBOT_VERSION=${{github.ref_name}}-${{github.sha}} + platforms: linux/amd64 + tags: ${{ steps.meta.outputs.tags }} + context: . + file: docker/stub/Dockerfile + cache-from: type=registry,ref=ghcr.io/coderbotorg/backend:stub-latest + cache-to: type=inline diff --git a/coderbot/activity.py b/coderbot/activity.py index f2864c07..19e3d102 100644 --- a/coderbot/activity.py +++ b/coderbot/activity.py @@ -1,5 +1,5 @@ from tinydb import TinyDB, Query - +from threading import Lock # Programs and Activities databases class Activities(): _instance = None @@ -13,34 +13,39 @@ def get_instance(cls): def __init__(self): self.activities = TinyDB("data/activities.json") self.query = Query() + self.lock = Lock() def load(self, name, default): - if name and default is None: - activities = self.activities.search(self.query.name == name) - if len(activities) > 0: - return activities[0] - elif default is not None: - if len(self.activities.search(self.query.default == True)) > 0: - return self.activities.search(self.query.default == True)[0] + with self.lock: + if name and default is None: + activities = self.activities.search(self.query.name == name) + if len(activities) > 0: + return activities[0] + elif default is not None: + if len(self.activities.search(self.query.default == True)) > 0: + return self.activities.search(self.query.default == True)[0] + return None return None - return None def save(self, name, activity): - # if saved activity is "default", reset existing default activity to "non-default" - if activity.get("default", False) is True: - self.activities.update({'default': False}) - if self.activities.search(self.query.name == name) == []: - self.activities.insert(activity) - else: - self.activities.update(activity, self.query.name == activity["name"]) + with self.lock: + # if saved activity is "default", reset existing default activity to "non-default" + if activity.get("default", False) is True: + self.activities.update({'default': False}) + if self.activities.search(self.query.name == name) == []: + self.activities.insert(activity) + else: + self.activities.update(activity, self.query.name == activity["name"]) def delete(self, name): - activities = self.activities.search(self.query.name == name) - if len(activities) > 0: - activity = activities[0] - if activity.get("default", False) is True: - self.activities.update({'default': True}, self.query.stock == True) - self.activities.remove(self.query.name == activity["name"]) + with self.lock: + activities = self.activities.search(self.query.name == name) + if len(activities) > 0: + activity = activities[0] + if activity.get("default", False) is True: + self.activities.update({'default': True}, self.query.stock == True) + self.activities.remove(self.query.name == activity["name"]) def list(self): - return self.activities.all() + with self.lock: + return self.activities.all() diff --git a/coderbot/api.py b/coderbot/api.py index 15d1e237..6b596e76 100644 --- a/coderbot/api.py +++ b/coderbot/api.py @@ -68,14 +68,14 @@ def get_status(): uptime = 0 try: uptime = subprocess.check_output(["uptime"]).decode('utf-8').replace('\n', '') - except: + except Exception: pass internet_status = False try: urllib.request.urlopen("https://coderbot.org") internet_status = True - except: + except Exception: pass return {'internet_status': internet_status, @@ -87,42 +87,20 @@ def get_info(): Expose informations about the CoderBot system. (Cached method) """ - backend_commit = "undefined" - coderbot_version = "undefined" - update_status = "ok" device = {} - motors = 'undefined' - - try: - # manifest.json is generated while building/copying the backend - with open('manifest.json', 'r') as f: - metadata = json.load(f) - backend_commit = metadata["backend_commit"][0:7] - coderbot_version = metadata["backend_version"][0:7] - except Exception: - pass - - try: - encoder = bool(Config.read().get('encoder')) - if(encoder): - motors = 'DC encoder motors' - else: - motors = 'DC motors' - except Exception: - pass - serial = get_serial() try: - device = Baleba.get_instance().device() + device = Balena.get_instance().device() + logging.info("device: %s", str(device)) except Exception: pass - return { 'backend_commit': device.get("commit"), - 'coderbot_version': coderbot_version, + + return { 'release_commit': device.get("commit"), + 'coderbot_version': os.getenv("CODERBOT_VERSION"), 'update_status': device.get("status"), 'kernel': device.get("os_version"), - 'serial': serial, - 'motors': motors } + 'serial': serial } prog = None prog_engine = ProgramEngine.get_instance() @@ -249,39 +227,6 @@ def deletePhoto(name): except FileNotFoundError: return 404 -## System - -def status(): - sts = get_status() - # getting reset log file - data = [] - try: - with open('/home/pi/coderbot/logs/reset_trigger_service.log', 'r') as log_file: - data = [x for x in log_file.read().split('\n') if x] - except Exception: # direct control case - pass # if file doesn't exist, no restore as ever been performed. return empty data - - - return { - "status": "ok", - "internetConnectivity": sts["internet_status"], - "temp": sts["temp"], - "uptime": sts["uptime"], - "log": data - } - -def info(): - inf = get_info() - return { - "model": 1, - "version": inf["coderbot_version"], - "backend commit build": inf["backend_commit"], - "kernel" : inf["kernel"], - "update status": inf["update_status"], - "serial": inf["serial"], - "motors": inf["motors"] - } - def restoreSettings(): Config.restore() return restart() diff --git a/coderbot/balena/__init__.py b/coderbot/balena/__init__.py index 5b5c9f9f..9df7c43f 100644 --- a/coderbot/balena/__init__.py +++ b/coderbot/balena/__init__.py @@ -21,24 +21,24 @@ def __init__(self): def purge(self): logging.debug("reset bot") req = Request(f'{self.supervisor_address}/v1/purge?apikey={self.supervisor_key}', data=self.app_id_data, headers=self.headers, method='POST') - return urlopen(req).read() + return json.load(urlopen(req)) def shutdown(self): logging.debug("shutdown bot") req = Request(f'{self.supervisor_address}/v1/shutdown?apikey={self.supervisor_key}', headers=self.headers, method='POST') - return urlopen(req).read() + return json.load(urlopen(req)) def restart(self): logging.debug("restarting bot") req = Request(f'{self.supervisor_address}/v1/restart?apikey={self.supervisor_key}', data=self.app_id_data, headers=self.headers, method='POST') - return urlopen(req).read() + return json.load(urlopen(req)) def reboot(self): logging.debug("reboot bot") req = Request(f'{self.supervisor_address}/v1/reboot?apikey={self.supervisor_key}', headers=self.headers, method='POST') - return urlopen(req).read() + return json.load(urlopen(req)) def device(self): logging.debug("reboot bot", f'{self.supervisor_address}get?apikey={self.supervisor_key}') - req = Request(f'{self.supervisor_address}/device?apikey={self.supervisor_key}', headers=self.headers, method='GET') + req = Request(f'{self.supervisor_address}/v1/device?apikey={self.supervisor_key}', headers=self.headers, method='GET') return json.load(urlopen(req)) diff --git a/coderbot/program.py b/coderbot/program.py index 67153ddf..d76395fd 100644 --- a/coderbot/program.py +++ b/coderbot/program.py @@ -25,6 +25,7 @@ import math from tinydb import TinyDB, Query +from threading import Lock import coderbot import camera @@ -80,6 +81,7 @@ def __init__(self): self._programs = TinyDB(PROGRAMS_DB) # initialise DB from default programs query = Query() + self.lock = Lock() for dirname, dirnames, filenames, in os.walk(PROGRAMS_PATH_DEFAULTS): dirnames for filename in filenames: @@ -102,28 +104,31 @@ def prog_list(self): return self._programs.all() def save(self, program): - query = Query() - self._program = program - program_db_entry = self._program.as_dict() - if self._programs.search(query.name == program.name) != []: - self._programs.update(program_db_entry, query.name == program.name) - else: - self._programs.insert(program_db_entry) + with self.lock: + query = Query() + self._program = program + program_db_entry = self._program.as_dict() + if self._programs.search(query.name == program.name) != []: + self._programs.update(program_db_entry, query.name == program.name) + else: + self._programs.insert(program_db_entry) def load(self, name): - query = Query() - program_db_entries = self._programs.search(query.name == name) - if len(program_db_entries) > 0: - prog_db_entry = program_db_entries[0] - logging.debug(prog_db_entry) - self._program = Program.from_dict(prog_db_entry) - return self._program - return None + with self.lock: + query = Query() + program_db_entries = self._programs.search(query.name == name) + if len(program_db_entries) > 0: + prog_db_entry = program_db_entries[0] + logging.debug(prog_db_entry) + self._program = Program.from_dict(prog_db_entry) + return self._program + return None def delete(self, name): - query = Query() - program_db_entries = self._programs.search(query.name == name) - self._programs.remove(query.name == name) + with self.lock: + query = Query() + program_db_entries = self._programs.search(query.name == name) + self._programs.remove(query.name == name) def create(self, name, code): self._program = Program(name, code) diff --git a/coderbot/v1.yml b/coderbot/v1.yml index f987579d..150c5bfa 100644 --- a/coderbot/v1.yml +++ b/coderbot/v1.yml @@ -397,7 +397,7 @@ paths: description: "upload failed" /system/status: get: - operationId: "api.status" + operationId: "api.get_status" summary: "Bot general informations, execution status and reset log file" tags: - System operations @@ -422,7 +422,7 @@ paths: description: Test ended. /system/info: get: - operationId: "api.info" + operationId: "api.get_info" summary: "Bot general informations and execution status" tags: - System operations diff --git a/docker/Dockerfile b/docker/Dockerfile index f2ee6362..6b88fce8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -50,7 +50,6 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt RUN mkdir -p /coderbot && \ mkdir -p /coderbot/data && \ mkdir -p /coderbot/logs && \ -mkdir -p /coderbot/updatePackages && \ mkdir -p /coderbot/cnn_modules && \ mkdir -p /coderbot/coderbot && \ mkdir -p /coderbot/defaults && \ @@ -63,5 +62,9 @@ ADD sounds /coderbot/sounds/. ADD docker/scripts/*.sh /tmp/. RUN /tmp/install_generic_cnn_models.sh RUN /tmp/install_lib_firmware.sh +ADD docker/start.sh /coderbot/. -ENTRYPOINT cd /coderbot && modprobe i2c-dev && python3 coderbot/main.py +ARG CODERBOT_VERSION +ENV CODERBOT_VERSION=${CODERBOT_VERSION} + +ENTRYPOINT /coderbot/start.sh diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 00000000..672297f1 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cd /coderbot && modprobe i2c-dev && python3 coderbot/main.py \ No newline at end of file diff --git a/docker/stub/Dockerfile b/docker/stub/Dockerfile new file mode 100644 index 00000000..aaf42d81 --- /dev/null +++ b/docker/stub/Dockerfile @@ -0,0 +1,65 @@ +FROM debian:bullseye-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && apt-get install -y \ + procps \ + sudo \ + wget \ + unzip \ + xz-utils \ + ffmpeg \ + portaudio19-dev \ + python3 \ + python3-pip \ + libopenjp2-7-dev \ + libtiff5 \ + libatlas-base-dev \ + libhdf5-dev \ + alsa-utils \ + espeak \ + libharfbuzz-bin \ + libwebp6 \ + libilmbase25 \ + libgstreamer1.0-0 \ + libavcodec-extra58 \ + libavformat58 \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + sox \ + libsox-fmt-all \ + avrdude \ + tesseract-ocr \ + tesseract-ocr-eng \ + tesseract-ocr-ita \ + tesseract-ocr-fra \ + tesseract-ocr-spa \ + tesseract-ocr-deu + +ADD docker/stub/requirements.txt /tmp/. +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +RUN mkdir -p /coderbot && \ +mkdir -p /coderbot/data && \ +mkdir -p /coderbot/logs && \ +mkdir -p /coderbot/updatePackages && \ +mkdir -p /coderbot/cnn_modules && \ +mkdir -p /coderbot/coderbot && \ +mkdir -p /coderbot/defaults && \ +mkdir -p /coderbot/sounds + +ADD coderbot /coderbot/coderbot/. +ADD stub /coderbot/stub/. +ADD test /coderbot/test/. +ADD defaults /coderbot/defaults/. +ADD sounds /coderbot/sounds/. + +ADD docker/scripts/*.sh /tmp/. +RUN /tmp/install_generic_cnn_models.sh +ADD docker/stub/start.sh /coderbot/. + +ARG CODERBOT_VERSION +ENV CODERBOT_VERSION=${CODERBOT_VERSION} + +ENTRYPOINT /coderbot/start.sh diff --git a/requirements_stub.txt b/docker/stub/requirements.txt similarity index 85% rename from requirements_stub.txt rename to docker/stub/requirements.txt index 7f5cd482..925819f3 100644 --- a/requirements_stub.txt +++ b/docker/stub/requirements.txt @@ -26,17 +26,9 @@ pybind11==2.10.0 inflection==0.5.1 event-channel==0.4.3 pytz==2022.2.1 -cachetools==5.2.0 - -# IO extensions -pigpio==1.78 -smbus2==0.4.2 -spidev==3.5 # Audio sox==1.4.1 -PyAudio==0.2.12 -pyalsaaudio==0.9.2 # Computer Vision grpcio==1.48.1 @@ -48,4 +40,5 @@ tflite-runtime==2.10.0 pytesseract==0.3.10 pyzbar==0.1.9 -#picamera==1.13 +#I/O +spidev==3.5 diff --git a/docker/stub/start.sh b/docker/stub/start.sh new file mode 100755 index 00000000..ff2ed337 --- /dev/null +++ b/docker/stub/start.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +export PYTHONPATH=./stub:./test:./coderbot +cd /coderbot +python3 coderbot/main.py \ No newline at end of file diff --git a/logs/.gitkeep b/logs/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/updatePackages/.gitkeep b/updatePackages/.gitkeep deleted file mode 100644 index e69de29b..00000000