diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4c42e68..d325329 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -3,10 +3,9 @@ name: Build modules on: [push, pull_request] + jobs: - test: - permissions: - contents: write + build-linux: runs-on: ubuntu-24.04 env: MPY_DIR: ./micropython @@ -24,7 +23,7 @@ jobs: with: repository: jonnor/micropython path: micropython - ref: v1.25.0 + ref: v1.25.0-armv6m-abs-relocations - name: Install Python dependencies run: pip install -r requirements.txt - name: Setup MicroPython X86 @@ -33,27 +32,94 @@ jobs: source tools/ci.sh && ci_unix_32bit_setup && ci_unix_standard_build mv ./ports/unix/build-standard/ ./ports/unix/build-nomodules/ - name: Build custom firmware with user modules, and tests. Unix/x64 - run: make check_unix V=1 + run: | + make check_unix V=1 + mkdir -p ./dist/ports/linux + mv ./dist/ports/unix/micropython ./dist/ports/linux/micropython - name: Build .mpy modules and run tests Unix/x64 run: make check ARCH=x64 V=1 + - name: Archive dist artifacts + uses: actions/upload-artifact@v4 + with: + name: emlearn-micropython-build-linux + path: dist + + build-arm: + runs-on: ubuntu-24.04 + env: + MPY_DIR: ./micropython + MICROPYTHON_BIN: ./micropython/ports/unix/build-nomodules/micropython + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install OS dependencies + run: sudo add-apt-repository universe + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: actions/checkout@v4 + with: + repository: jonnor/micropython + path: micropython + ref: v1.25.0-armv6m-abs-relocations + - name: Install Python dependencies + run: pip install -r requirements.txt - name: Setup MicroPython ARM working-directory: micropython - run: source tools/ci.sh && ci_rp2_setup + run: | + source tools/ci.sh && ci_rp2_setup + make -C mpy-cross - name: Setup MicroPython RP2 port working-directory: micropython/ports/rp2 - run: make submodules + run: | + make submodules + make clean - name: Build custom firmware with extmod, RP2 - run: make rp2 V=1 + run: make rp2 PORT=rp2 V=1 - name: Build module armv6m - run: echo make dist ARCH=armv6m V=1 + run: make dist ARCH=armv6m V=1 - name: Build module armv7m run: make dist ARCH=armv7m V=1 - name: Build module armv7emsp run: make dist ARCH=armv7emsp V=1 + - name: Archive dist artifacts + uses: actions/upload-artifact@v4 + with: + name: emlearn-micropython-build-arm + path: dist + + build-esp32: + runs-on: ubuntu-24.04 + env: + MPY_DIR: ./micropython + MICROPYTHON_BIN: ./micropython/ports/unix/build-nomodules/micropython + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install OS dependencies + run: sudo add-apt-repository universe + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: actions/checkout@v4 + with: + repository: jonnor/micropython + path: micropython + ref: v1.25.0-armv6m-abs-relocations + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Setup MicroPython ESP32 working-directory: micropython - run: source tools/ci.sh && ci_esp32_idf_setup - - name: Build custom firmware with extmod, RP2 + run: | + source tools/ci.sh && ci_esp32_idf_setup + make -C mpy-cross + - name: Setup submodules esp32 + working-directory: micropython/ports/esp32 + run: source ../../esp-idf/export.sh && make submodules + - name: Build custom firmware with extmod, ESP32 run: | source micropython/esp-idf/export.sh && pip install -r requirements.txt make extmod PORT=esp32 BOARD=ESP32_GENERIC_S3 @@ -63,15 +129,111 @@ jobs: - name: Archive dist artifacts uses: actions/upload-artifact@v4 with: - name: dist + name: emlearn-micropython-build-esp32 path: dist - - name: Prepare release - run: make release - - name: Archive release artifacts + + + build-riscv: + runs-on: ubuntu-24.04 + env: + MPY_DIR: ./micropython + MICROPYTHON_BIN: ./micropython/ports/unix/build-nomodules/micropython + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install OS dependencies + run: sudo add-apt-repository universe + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: actions/checkout@v4 + with: + repository: jonnor/micropython + path: micropython + ref: v1.25.0-armv6m-abs-relocations + - name: Install Python dependencies + run: pip install -r requirements.txt + + - name: Setup MicroPython ESP32 + working-directory: micropython + run: | + source tools/ci.sh && ci_esp32_idf_setup && ci_gcc_riscv_setup + make -C mpy-cross + - name: Setup submodules esp32 + working-directory: micropython/ports/esp32 + run: source ../../esp-idf/export.sh && make submodules + - name: Build custom firmware with extmod, ESP32 + run: | + source micropython/esp-idf/export.sh && pip install -r requirements.txt + echo make extmod PORT=esp32 BOARD=ESP32_GENERIC_C6 + - name: Build nadmod xtensawin + run: source micropython/esp-idf/export.sh && pip install -r requirements.txt && make dist ARCH=rv32imc V=1 + - name: Archive dist artifacts uses: actions/upload-artifact@v4 with: - name: release - path: emlearn-micropython-*.zip + name: emlearn-micropython-build-riscv + path: dist + + + build-macos: + runs-on: macos-latest + env: + MPY_DIR: ./micropython + MICROPYTHON_BIN: ./micropython/ports/unix/build-nomodules/micropython + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install OS dependencies + run: brew install pkg-config libffi + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: actions/checkout@v4 + with: + repository: jonnor/micropython + path: micropython + ref: v1.25.0-armv6m-abs-relocations + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Setup MicroPython X86 + working-directory: micropython + run: | + source tools/ci.sh && ci_unix_32bit_setup && ci_unix_standard_build + mv ./ports/unix/build-standard/ ./ports/unix/build-nomodules/ + - name: Build custom firmware with user modules, and tests. Unix/x64 + run: | + make check_unix V=1 + mkdir -p ./dist/ports/macos + mv ./dist/ports/unix/micropython ./dist/ports/macos/micropython + - name: Build .mpy modules and run tests Unix/x64 + # natmod build with linking seems broken on Mac OS, exception in mpy_ld.py, ELFError, magic number does not match + run: echo SKIP make check ARCH=x64 V=1 + + - name: Archive dist artifacts + uses: actions/upload-artifact@v4 + with: + name: emlearn-micropython-build-macos + path: dist + + deploy-pages: + permissions: + contents: write + needs: [build-linux, build-arm, build-esp32, build-macos, build-riscv] + runs-on: ubuntu-latest + + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + + - name: Prepare files for distribution + run: | + mkdir -p dist/ + cp -r artifacts/*/* dist/ - name: Deploy to Github Pages if: "github.ref_name == 'master' || github.ref_type == 'tag'" diff --git a/.github/workflows/draft-pdf.yaml b/.github/workflows/draft-pdf.yaml new file mode 100644 index 0000000..b7053c0 --- /dev/null +++ b/.github/workflows/draft-pdf.yaml @@ -0,0 +1,24 @@ +name: Draft PDF +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper/paper.md + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper/paper.pdf diff --git a/Makefile b/Makefile index 0b6ff72..436a486 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,9 @@ $(MODULES_PATH)/emlearn_iir_q15.mpy: $(MODULES_PATH)/emlearn_arrayutils.mpy: make -C src/emlearn_arrayutils/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) V=1 clean dist +$(MODULES_PATH)/emlearn_linreg.mpy: + make -C src/emlearn_linreg/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) V=1 clean dist + emlearn_trees.results: $(MODULES_PATH)/emlearn_trees.mpy MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_trees.py @@ -79,6 +82,9 @@ emlearn_iir_q15.results: $(MODULES_PATH)/emlearn_iir_q15.mpy emlearn_arrayutils.results: $(MODULES_PATH)/emlearn_arrayutils.mpy MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_arrayutils.py +emlearn_linreg.results: $(MODULES_PATH)/emlearn_linreg.mpy + MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_linreg.py + $(PORT_DIR): mkdir -p $@ @@ -124,8 +130,8 @@ release: zip -r $(RELEASE_NAME).zip $(RELEASE_NAME) #cp $(RELEASE_NAME).zip emlearn-micropython-latest.zip -check: emlearn_trees.results emlearn_neighbors.results emlearn_iir.results emlearn_iir_q15.results emlearn_fft.results emlearn_kmeans.results emlearn_arrayutils.results emlearn_cnn.results +check: emlearn_trees.results emlearn_neighbors.results emlearn_iir.results emlearn_iir_q15.results emlearn_fft.results emlearn_kmeans.results emlearn_arrayutils.results emlearn_cnn.results emlearn_linreg.results -dist: $(MODULES_PATH)/emlearn_trees.mpy $(MODULES_PATH)/emlearn_neighbors.mpy $(MODULES_PATH)/emlearn_iir.mpy $(MODULES_PATH)/emlearn_iir_q15.mpy $(MODULES_PATH)/emlearn_fft.mpy $(MODULES_PATH)/emlearn_kmeans.mpy $(MODULES_PATH)/emlearn_arrayutils.mpy $(MODULES_PATH)/emlearn_cnn_int8.mpy $(MODULES_PATH)/emlearn_cnn_fp32.mpy +dist: $(MODULES_PATH)/emlearn_trees.mpy $(MODULES_PATH)/emlearn_neighbors.mpy $(MODULES_PATH)/emlearn_iir.mpy $(MODULES_PATH)/emlearn_iir_q15.mpy $(MODULES_PATH)/emlearn_fft.mpy $(MODULES_PATH)/emlearn_kmeans.mpy $(MODULES_PATH)/emlearn_arrayutils.mpy $(MODULES_PATH)/emlearn_cnn_int8.mpy $(MODULES_PATH)/emlearn_cnn_fp32.mpy $(MODULES_PATH)/emlearn_linreg.mpy diff --git a/README.md b/README.md index 0237bd2..e809343 100644 --- a/README.md +++ b/README.md @@ -80,16 +80,34 @@ git clone https://github.com/emlearn/emlearn-micropython ``` #### Prerequisites -These come in addition to the prequisites described above. -Make sure you have the dependencies needed to build for your platform. -See [MicroPython: Building native modules](https://docs.micropython.org/en/latest/develop/natmod.html). +You will need to have **Python 3.10+ or later** already installed. -We assume that micropython is installed in the same place as this repository. +We assume that **micropython** git repository available. +It is assumed to be at the same level as this repository in the file system. If using another location, adjust `MPY_DIR` accordingly. - You should be using MicroPython 1.25 (or newer). +Make sure you have the **build toolchain** needed for your platform. +See [MicroPython: Building native modules](https://docs.micropython.org/en/latest/develop/natmod.html), +and the documentation for the MicroPython port/architecture of interest. + +#### Download dependencies + +Fetch git submodules + +``` +git submodule update --init +``` + +Recommend using a Python virtual environment. + +Install Python packages +``` +pip install -r requirements.txt +``` + + #### Build Build the .mpy native module diff --git a/examples/har_trees/README.md b/examples/har_trees/README.md index 40b9caa..cb570b8 100644 --- a/examples/har_trees/README.md +++ b/examples/har_trees/README.md @@ -123,7 +123,7 @@ mpremote cp windower.py : Run the classification ``` -mpremote har_live.py +mpremote run har_live.py ``` This will continiously output the results of the classification, diff --git a/examples/har_trees/basic.py b/examples/har_trees/basic.py new file mode 100644 index 0000000..6060c58 --- /dev/null +++ b/examples/har_trees/basic.py @@ -0,0 +1,251 @@ + +import array +import math + +DATA_TYPECODE = 'h' # int16 + + +def feature_names() -> list[str]: + cls = Feature + + features = {} # index -> name + for key, value in cls.__dict__.items(): + if key.startswith('__') or callable(value): + continue + if features.get(value): + raise ValueError(f"Duplicated value {value}") + features[value] = key + + names = [] + for idx, value in enumerate(sorted(features.keys())): + if idx != value: + raise ValueError(f'Holes in enum') + + names.append(features[value]) + + return names + +class Feature: + orient_w = 0 + orient_x = 1 + orient_y = 2 + orient_z = 3 + sma = 4 + mag_rms = 5 + x_rms = 6 + y_rms = 7 + z_rms = 8 + +N_FEATURES = 9 + + +def compute_sma(x_data : array.array, y_data : array.array, z_data : array.array) -> float: + """Signal Magnitude Area (SMA)""" + + n = len(x_data) + sma_sum = 0.0 + for x, y, z in zip(x_data, y_data, z_data): + sma_sum += abs(x) + abs(y) + abs(z) + + sma = sma_sum / n + return sma + +def compute_magnitude_rms(x_data : array.array, y_data : array.array, z_data : array.array) -> float: + + n = len(x_data) + agg = 0.0 + for x, y, z in zip(x_data, y_data, z_data): + agg += (x*x) + (y*y) + (z*z) + + rms = math.sqrt(agg / n) + return rms + +def compute_rms(data : array.array) -> tuple[float]: + + n = len(data) + agg = 0.0 + for v in data: + agg += (v*v) + + rms = math.sqrt(agg / n) + return rms + +def compute_pitch_roll(x : float, y : float, z : float) -> tuple[float]: + """In degrees""" + + roll = round(math.degrees(math.atan2(y, z)), 2) + denominator = math.sqrt(y * y + z * z) + pitch = round(math.degrees(math.atan2(-x, denominator)), 2) + + return pitch, roll + + +def calculate_features_xyz(xyz : tuple[array.array]) -> array.array: + + xo, yo, zo = xyz + + if not (len(xo) == len(yo) == len(zo)): + raise ValueError("Input data lists must have the same length.") + + window_length = len(xo) + + # Output array + feature_data = array.array('f', (0 for i in range(N_FEATURES))) + + + # Gravity separation + # FIXME: replace mean with a low-pass filter at 0.5 Hz. Say Chebychev 2 order + # and to subtract the continues values. + # !! need to be able to initialize IIR filter state to first sample + gravity_x = sum(xo) / window_length + gravity_y = sum(yo) / window_length + gravity_z = sum(zo) / window_length + + linear_x = array.array(DATA_TYPECODE, (0 for _ in range(window_length))) + linear_y = array.array(DATA_TYPECODE, (0 for _ in range(window_length))) + linear_z = array.array(DATA_TYPECODE, (0 for _ in range(window_length))) + + + for i in range(window_length): + i = int(i) # XXX ?? + linear_x[i] = int(xo[i] - gravity_x) + linear_y[i] = int(yo[i] - gravity_y) + linear_z[i] = int(zo[i] - gravity_z) + + # Orientation + # normalize gravity vector + #gravity_magnitude = math.sqrt(gravity_x*gravity_x + gravity_y*gravity_y + gravity_z*gravity_z) + #ox = gravity_x/gravity_magnitude + #oy = gravity_y/gravity_magnitude + #oz = gravity_z/gravity_magnitude + #print(ox, oy, oz) + + #roll, pitch = compute_pitch_roll(ox, oy, oz) + #feature_data[Feature.pitch] = pitch + #feature_data[Feature.roll] = roll + + # FIXME: use the normalized (unit-scaled) gravity vector instead of quaternion + orientation_q = tilt_quaternion_from_accel(gravity_x, gravity_y, gravity_z) + feature_data[Feature.orient_w] = orientation_q[0] + feature_data[Feature.orient_x] = orientation_q[1] + feature_data[Feature.orient_y] = orientation_q[2] + feature_data[Feature.orient_z] = orientation_q[3] + + + # Overall motion + #feature_data[Feature.sma] = compute_sma(linear_x, linear_y, linear_z) + #feature_data[Feature.mag_rms] = compute_magnitude_rms(linear_x, linear_y, linear_z) + + # Per-axis motion + #feature_data[Feature.x_rms] = compute_rms(linear_x) + #feature_data[Feature.y_rms] = compute_rms(linear_y) + #feature_data[Feature.z_rms] = compute_rms(linear_z) + + print(orientation_q) + + + return feature_data + + +def normalize(v): + mag = math.sqrt(sum(c*c for c in v)) + if mag == 0: + return (0.0, 0.0, 0.0) + return tuple(c / mag for c in v) + +def dot(v1, v2): + return sum(a*b for a, b in zip(v1, v2)) + +def cross(v1, v2): + return ( + v1[1]*v2[2] - v1[2]*v2[1], + v1[2]*v2[0] - v1[0]*v2[2], + v1[0]*v2[1] - v1[1]*v2[0] + ) + +def tilt_quaternion_from_accel(a_x, a_y, a_z): + # Gravity vector measured by accelerometer (assumed already low-pass filtered) + g = normalize((a_x, a_y, a_z)) + + # Reference "down" vector in world space + ref = (0.0, 0.0, 1.0) + + # Compute axis and angle between vectors + axis = cross(ref, g) + axis_mag = math.sqrt(sum(c*c for c in axis)) + + if axis_mag < 1e-6: + # Vectors are nearly aligned or opposite + if dot(ref, g) > 0.999: + # Identity rotation (device is flat, facing up) + return (1.0, 0.0, 0.0, 0.0) + else: + # 180° rotation around X or Y (choose arbitrary orthogonal axis) + return (0.0, 1.0, 0.0, 0.0) # Flip around X + + axis = normalize(axis) + angle = math.acos(max(-1.0, min(1.0, dot(ref, g)))) # Clamp dot product to avoid domain error + + # Convert axis-angle to quaternion + half_angle = angle / 2 + sin_half = math.sin(half_angle) + q_w = math.cos(half_angle) + q_x = axis[0] * sin_half + q_y = axis[1] * sin_half + q_z = axis[2] * sin_half + + return (q_w, q_x, q_y, q_z) + + +def test_compute(): + + window_length = 50 + + x = array.array(DATA_TYPECODE, (0 for _ in range(window_length))) + y = array.array(DATA_TYPECODE, (0 for _ in range(window_length))) + z = array.array(DATA_TYPECODE, (0 for _ in range(window_length))) + + features = calculate_features_xyz((x, y, z)) + + names = feature_names() + assert len(names) == len(features) + + +def test_pitch_rotation(): + + print("\n--- Simulating Pitch (X-axis) Rotation ---") + # Rotate around X-axis (pitch) + for angle_deg in range(-90, 91, 15): + rad = math.radians(angle_deg) + # Simulated gravity vector for X-axis rotation + a_x = math.sin(rad) + a_y = 0 + a_z = math.cos(rad) + + a_xn, a_yn, a_zn = normalize(a_x, a_y, a_z) + pitch, roll = compute_pitch_roll(a_xn, a_yn, a_zn) + + print(f"{angle_deg:6} | {a_xn:+.2f} {a_yn:+.2f} {a_zn:+.2f} | {roll:+6.1f}° {pitch:+6.1f}°") + +def test_roll_rotation(): + + print("\n--- Simulating Roll (Y-axis) Rotation ---") + for angle_deg in range(-90, 91, 15): + rad = math.radians(angle_deg) + a_x = 0 + a_y = math.sin(rad) + a_z = math.cos(rad) + + a_xn, a_yn, a_zn = normalize(a_x, a_y, a_z) + pitch, roll = compute_pitch_roll(a_xn, a_yn, a_zn) + + print(f"{angle_deg:6} | {a_xn:+.2f} {a_yn:+.2f} {a_zn:+.2f} | {roll:+6.1f}° {pitch:+6.1f}°") + + +if __name__ == '__main__': + + #test_compute() + test_pitch_rotation() + test_roll_rotation() + + print('PASS') diff --git a/examples/har_trees/color_setup.py b/examples/har_trees/color_setup.py index 9d74d0a..50f5155 100644 --- a/examples/har_trees/color_setup.py +++ b/examples/har_trees/color_setup.py @@ -1,23 +1,21 @@ -import gc -from machine import Pin, SPI +from machine import Pin, SPI, PWM -# M5Stack M5StickC PLUS 2 -# https://docs.m5stack.com/en/core/M5StickC%20PLUS2 -from drivers.st7789.st7789_4bit import * -# ESP32 GPIO15 GPIO13 GPIO14 GPIO12 GPIO5 GPIO27 -# TFT LCD TFT_MOSI TFT_CLK TFT_DC TFT_RST TFT_CS TFT_BL +from drivers.st7789.st7789_4bit import ST7789 as SSD, LANDSCAPE, TDISPLAY -pdc = Pin(14, Pin.OUT, value=0) -pcs = Pin(5, Pin.OUT, value=1) -prst = Pin(12, Pin.OUT, value=1) -pbl = Pin(27, Pin.OUT, value=1) +def init_screen(backlight_duty=400, backlight_freq=1000): -gc.collect() # Precaution before instantiating framebuf + # M5Stack M5StickC PLUS 2 + # https://docs.m5stack.com/en/core/M5StickC%20PLUS2 + # ESP32 GPIO15 GPIO13 GPIO14 GPIO12 GPIO5 GPIO27 + # TFT LCD TFT_MOSI TFT_CLK TFT_DC TFT_RST TFT_CS TFT_BL -SSD = ST7789 - -# Conservative low baudrate. Can go to 62.5MHz. -spi = SPI(1, 30_000_000, sck=Pin(13), mosi=Pin(15)) -ssd = ST7789(spi, height=135, width=240, dc=pdc, cs=pcs, rst=prst, disp_mode=LANDSCAPE, display=TDISPLAY) + pdc = Pin(14, Pin.OUT, value=0) + pcs = Pin(5, Pin.OUT, value=1) + prst = Pin(12, Pin.OUT, value=1) + backlight_pwm = PWM(Pin(27), freq=backlight_freq, duty=backlight_duty) + # Conservative low baudrate. Can go to 62.5MHz. + spi = SPI(1, 30_000_000, sck=Pin(13), mosi=Pin(15)) + ssd = SSD(spi, height=135, width=240, dc=pdc, cs=pcs, rst=prst, disp_mode=LANDSCAPE, display=TDISPLAY) + return ssd diff --git a/examples/har_trees/data/configurations/cgestures.yaml b/examples/har_trees/data/configurations/cgestures.yaml new file mode 100644 index 0000000..9684cdc --- /dev/null +++ b/examples/har_trees/data/configurations/cgestures.yaml @@ -0,0 +1,12 @@ +groups: + - file +data_columns: + - x + - y + - z +classes: + - idle + - snake + - updown + - wave + - other diff --git a/examples/har_trees/data/configurations/har_exercise1.yaml b/examples/har_trees/data/configurations/har_exercise1.yaml new file mode 100644 index 0000000..18b7e87 --- /dev/null +++ b/examples/har_trees/data/configurations/har_exercise1.yaml @@ -0,0 +1,12 @@ +groups: + - file +data_columns: + - x + - y + - z +classes: + # - mixed + - squat + - jumpingjack + - lunge + - other diff --git a/examples/har_trees/data/configurations/pamap2.yaml b/examples/har_trees/data/configurations/pamap2.yaml new file mode 100644 index 0000000..0c2af56 --- /dev/null +++ b/examples/har_trees/data/configurations/pamap2.yaml @@ -0,0 +1,20 @@ +groups: + - subject +data_columns: + - hand_acceleration_16g_x + - hand_acceleration_16g_y + - hand_acceleration_16g_z +classes: + # - transient + - walking + - ironing + - lying + - standing + - Nordic_walking + - sitting + - vacuum_cleaning + - cycling + - ascending_stairs + - descending_stairs + - running + - rope_jumping diff --git a/examples/har_trees/data/configurations/uci_har.yaml b/examples/har_trees/data/configurations/uci_har.yaml new file mode 100644 index 0000000..9d044d1 --- /dev/null +++ b/examples/har_trees/data/configurations/uci_har.yaml @@ -0,0 +1,20 @@ +groups: + - subject + - experiment +data_columns: + - acc_x + - acc_y + - acc_z +classes: + # - STAND_TO_LIE + # - SIT_TO_LIE + # - LIE_TO_SIT + # - STAND_TO_SIT + # - LIE_TO_STAND + # - SIT_TO_STAND + - STANDING + - LAYING + - SITTING + - WALKING + - WALKING_UPSTAIRS + - WALKING_DOWNSTAIRS diff --git a/examples/har_trees/data/processed/cgestures.parquet b/examples/har_trees/data/processed/cgestures.parquet new file mode 100644 index 0000000..8956b11 Binary files /dev/null and b/examples/har_trees/data/processed/cgestures.parquet differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192744_snake.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192744_snake.npy new file mode 100644 index 0000000..1946221 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192744_snake.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192748_snake.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192748_snake.npy new file mode 100644 index 0000000..93aa532 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192748_snake.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192752_snake.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192752_snake.npy new file mode 100644 index 0000000..255ebfc Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192752_snake.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192759_snake.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192759_snake.npy new file mode 100644 index 0000000..056c105 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192759_snake.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192803_snake.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192803_snake.npy new file mode 100644 index 0000000..cdc06b5 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192803_snake.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192807_snake.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192807_snake.npy new file mode 100644 index 0000000..5bb4357 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192807_snake.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192823_updown.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192823_updown.npy new file mode 100644 index 0000000..b54bc74 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192823_updown.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192827_updown.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192827_updown.npy new file mode 100644 index 0000000..f6f4904 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192827_updown.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192831_updown.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192831_updown.npy new file mode 100644 index 0000000..247f36c Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192831_updown.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192835_updown.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192835_updown.npy new file mode 100644 index 0000000..ba4efa4 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192835_updown.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192839_updown.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192839_updown.npy new file mode 100644 index 0000000..7786798 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192839_updown.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192843_updown.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192843_updown.npy new file mode 100644 index 0000000..5f1e80f Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192843_updown.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192906_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192906_wave.npy new file mode 100644 index 0000000..f8c68c5 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192906_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192910_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192910_wave.npy new file mode 100644 index 0000000..d89a486 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192910_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192914_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192914_wave.npy new file mode 100644 index 0000000..c85077d Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192914_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192918_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192918_wave.npy new file mode 100644 index 0000000..6ba10e0 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192918_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192922_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192922_wave.npy new file mode 100644 index 0000000..9bebc6c Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192922_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192927_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192927_wave.npy new file mode 100644 index 0000000..b61d93e Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192927_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192932_wave.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192932_wave.npy new file mode 100644 index 0000000..2edbd30 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T192932_wave.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193704_idle.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193704_idle.npy new file mode 100644 index 0000000..430f230 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193704_idle.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193708_idle.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193708_idle.npy new file mode 100644 index 0000000..897dc6f Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193708_idle.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193712_idle.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193712_idle.npy new file mode 100644 index 0000000..4ed4506 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193712_idle.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193716_idle.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193716_idle.npy new file mode 100644 index 0000000..23c5237 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193716_idle.npy differ diff --git a/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193720_idle.npy b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193720_idle.npy new file mode 100644 index 0000000..f0bc6f7 Binary files /dev/null and b/examples/har_trees/data/raw/cgestures/har_record/2025-06-15T193720_idle.npy differ diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:00:51_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000051_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:00:51_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000051_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:01:32_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000132_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:01:32_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000132_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:01:42_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000142_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:01:42_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000142_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:01:52_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000152_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:01:52_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000152_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:02_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000202_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:02_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000202_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:04_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000204_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:04_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000204_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:11_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000211_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:11_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000211_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:14_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000214_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:14_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000214_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:21_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000221_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:21_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000221_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:24_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000224_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:24_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000224_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:31_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000231_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:31_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000231_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:33_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000233_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:33_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000233_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:43_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000243_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:43_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000243_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:43_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000243_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:43_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000243_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:52_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000252_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:52_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000252_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:53_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000253_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:02:53_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000253_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:02_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000302_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:02_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000302_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:08_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000308_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:08_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000308_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:17_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000317_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:17_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000317_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:22_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000322_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:22_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000322_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:27_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000327_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:27_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000327_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:31_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000331_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:31_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000331_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:37_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000337_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:37_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000337_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:41_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000341_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:41_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000341_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:47_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000347_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:47_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000347_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:51_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000351_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:51_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000351_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:57_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000357_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:03:57_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000357_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:04:01_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000401_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:04:01_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000401_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:04:07_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000407_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T00:04:07_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2000-01-01T000407_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:07_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153607_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:07_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153607_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:17_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153617_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:17_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153617_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:27_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153627_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:27_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153627_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:37_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153637_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:37_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153637_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:47_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153647_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:47_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153647_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:57_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153657_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:36:57_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153657_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:06_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153706_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:06_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153706_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:19_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153719_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:19_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153719_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:29_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153729_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:29_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153729_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:38_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153738_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:38_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153738_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:48_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153748_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:48_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153748_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:58_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153758_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:37:58_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153758_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:08_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153808_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:08_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153808_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:18_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153818_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:18_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153818_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:28_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153828_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:28_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153828_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:38_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153838_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:38_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153838_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:48_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153848_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:48_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153848_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:58_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153858_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:38:58_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153858_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:08_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153908_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:08_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153908_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:18_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153918_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:18_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153918_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:27_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153927_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:27_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153927_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:37_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153937_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:37_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153937_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:59_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153959_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:39:59_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T153959_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:09_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154009_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:09_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154009_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:19_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154019_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:19_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154019_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:29_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154029_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:29_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154029_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:39_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154039_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:39_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154039_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:49_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154049_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:49_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154049_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:58_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154058_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:40:58_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154058_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:08_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154108_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:08_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154108_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:18_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154118_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:18_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154118_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:28_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154128_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:28_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154128_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:38_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154138_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:38_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154138_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:48_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154148_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:48_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154148_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:58_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154158_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:41:58_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154158_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:08_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154208_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:08_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154208_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:28_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154228_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:28_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154228_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:38_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154238_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:38_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154238_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:47_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154247_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:47_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154247_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:57_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154257_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:42:57_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154257_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:07_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154307_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:07_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154307_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:17_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154317_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:17_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154317_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:27_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154327_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:27_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154327_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:37_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154337_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:37_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154337_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:47_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154347_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:47_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154347_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:57_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154357_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:43:57_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154357_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:07_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154407_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:07_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154407_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:17_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154417_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:17_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154417_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:27_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154427_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:27_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154427_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:36_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154436_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:36_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154436_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:46_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154446_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T15:44:46_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T154446_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:05_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160405_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:05_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160405_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:15_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160415_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:15_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160415_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:25_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160425_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:25_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160425_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:35_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160435_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:35_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160435_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:45_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160445_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:45_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160445_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:55_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160455_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:04:55_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160455_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:04_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160504_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:04_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160504_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:14_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160514_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:14_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160514_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:35_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160535_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:35_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160535_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:45_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160545_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:45_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160545_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:54_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160554_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:05:54_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160554_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:04_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160604_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:04_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160604_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:14_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160614_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:14_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160614_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:24_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160624_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:24_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160624_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:56_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160656_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:06:56_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160656_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:06_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160706_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:06_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160706_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:16_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160716_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:16_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160716_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:26_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160726_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:26_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160726_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:37_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160737_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:37_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160737_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:47_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160747_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:47_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160747_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:57_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160757_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:07:57_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160757_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:08:07_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160807_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:08:07_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T160807_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:06_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162106_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:06_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162106_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:16_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162116_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:16_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162116_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:26_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162126_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:26_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162126_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:35_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162135_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:35_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162135_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:45_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162145_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:45_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162145_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:55_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162155_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:21:55_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162155_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:05_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162205_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:05_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162205_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:15_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162215_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:15_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162215_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:25_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162225_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:25_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162225_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:35_other.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162235_other.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:22:35_other.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162235_other.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:07_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162307_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:07_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162307_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:17_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162317_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:17_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162317_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:27_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162327_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:27_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162327_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:37_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162337_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:37_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162337_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:48_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162348_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:48_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162348_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:58_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162358_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:23:58_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162358_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:08_jumpingjack.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162408_jumpingjack.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:08_jumpingjack.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162408_jumpingjack.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:28_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162428_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:28_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162428_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:38_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162438_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:38_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162438_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:48_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162448_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:48_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162448_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:58_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162458_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:24:58_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162458_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:08_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162508_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:08_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162508_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:18_lunge.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162518_lunge.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:18_lunge.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162518_lunge.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:31_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162531_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:31_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162531_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:41_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162541_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:41_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162541_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:51_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162551_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:25:51_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162551_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:01_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162601_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:01_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162601_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:10_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162610_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:10_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162610_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:20_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162620_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:20_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162620_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:30_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162630_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:30_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162630_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:40_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162640_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:40_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162640_squat.npy diff --git a/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:50_squat.npy b/examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162650_squat.npy similarity index 100% rename from examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T16:26:50_squat.npy rename to examples/har_trees/data/raw/har_record_exercises_1/data/2024-12-01T162650_squat.npy diff --git a/examples/har_trees/har_data2labelstudio.py b/examples/har_trees/har_data2labelstudio.py index 52452ce..f819507 100644 --- a/examples/har_trees/har_data2labelstudio.py +++ b/examples/har_trees/har_data2labelstudio.py @@ -34,7 +34,7 @@ def load_har_record(path, df['time'] = t df = df.set_index('time') - classname = f.split('_')[1].rstrip(suffix) + classname = os.path.splitext(f)[0].split('_')[1] # Remove :, special character on Windows filename = f.replace(':', '') diff --git a/examples/har_trees/har_labelstudio2dataset.py b/examples/har_trees/har_labelstudio2dataset.py index b51a678..ac5add9 100644 --- a/examples/har_trees/har_labelstudio2dataset.py +++ b/examples/har_trees/har_labelstudio2dataset.py @@ -5,8 +5,8 @@ import pandas +from har_data2labelstudio import load_har_record -from read_data import load_har_record def extract_filename(url): diff --git a/examples/har_trees/har_live.py b/examples/har_trees/har_live.py index 7bb30c8..a3bcf6d 100644 --- a/examples/har_trees/har_live.py +++ b/examples/har_trees/har_live.py @@ -1,12 +1,18 @@ import machine -from machine import Pin, I2C +from machine import Pin, I2C, SPI + +# On M5StickC we need to set HOLD pin to stay alive when on battery +hold_pin = machine.Pin(4, machine.Pin.OUT) +hold_pin.value(1) + from mpu6886 import MPU6886 import bluetooth import time import struct import array +import json from windower import TriaxialWindower, empty_array import timebased @@ -14,16 +20,19 @@ # for display # mpremote mip install "github:peterhinch/micropython-nano-gui" -from color_setup import ssd from gui.core.writer import Writer from gui.core.nanogui import refresh from gui.widgets.meter import Meter from gui.widgets.label import Label, ALIGN_RIGHT -import gui.fonts.courier20 as fixed_font +#import gui.fonts.courier20 as fixed_font +import gui.fonts.font10 as fixed_font + +from color_setup import init_screen # Cleanup after import frees considerable memory gc.collect() + def mean(arr): m = sum(arr) / float(len(arr)) return m @@ -118,7 +127,7 @@ def send_bluetooth_le(sequence, probabilities, ble.active(False) -def render_display(durations): +def render_display(ssd, durations): start_time = time.ticks_ms() ssd.fill(0) @@ -130,6 +139,8 @@ def render_display(durations): y = 5 for classname, stats in durations.items(): + classname = classname[:8] # truncate to make sure it fits + key_text = classname text1 = Label(wri, y, 10, wri.stringlen(key_text)) text1.value(key_text) @@ -137,30 +148,38 @@ def render_display(durations): value_text = f'{stats:.0f}s' text2 = Label(wri, y, 140, 50, align=ALIGN_RIGHT) text2.value(value_text) - y += 22 + y += 20 refresh(ssd) duration = time.ticks_ms() - start_time - print('render-display', duration, 'ms') + if False: + print('render-display', duration, 'ms') def main(): - dataset = 'har_exercise_1' - - if dataset == 'uci_har': - classname_index = {"LAYING": 0, "SITTING": 1, "STANDING": 2, "WALKING": 3, "WALKING_DOWNSTAIRS": 4, "WALKING_UPSTAIRS": 5} - window_length = 128 - elif dataset == 'har_exercise_1': - classname_index = {"jacks": 0, "lunge": 1, "other": 2, "squat": 3} - window_length = 256 - else: - raise ValueError('Unknown dataset') - - model_path = f'{dataset}_trees.csv' - class_index_to_name = { v: k for k, v in classname_index.items() } + # Settings + DATASET = 'uci_har' + BLE_ENABLED = False + MIN_PROBABILITY = 0.4 # if no class has higher, consider as "other" + SAMPLERATE = 100 # TODO: load from model meta.json + hop_length = 64 + # Internal LED on M5StickC PLUS2 + led_pin = machine.Pin(19, machine.Pin.OUT) + led_pin.value(1) + + ssd = init_screen() + + # Load model metadata + with open(f'{DATASET}.meta.json', 'r') as f: + model_meta = json.loads(f.read()) + classname_index = model_meta['classes'] + window_length = model_meta['window_length'] + + model_path = f'{DATASET}.trees.csv' + class_index_to_name = { v: k for k, v in classname_index.items() } # Load a CSV file with the model model = emlearn_trees.new(10, 1000, 10) @@ -170,11 +189,9 @@ def main(): mpu = MPU6886(I2C(0, sda=21, scl=22, freq=100000)) # Enable FIFO at a fixed samplerate - SAMPLERATE = 100 mpu.fifo_enable(True) mpu.set_odr(SAMPLERATE) - hop_length = 64 chunk = bytearray(mpu.bytes_per_sample*hop_length) x_values = empty_array('h', hop_length) @@ -193,7 +210,7 @@ def main(): prediction_no = 0 durations = { classname: 0.0 for classname in classname_index.keys() } # how long each class has been active - MIN_PROBABILITY = 0.5 # if no class has higher, consider as "other" + while True: @@ -224,19 +241,24 @@ def main(): if max(out) < MIN_PROBABILITY: activity = 'other' - durations[activity] += (hop_length/SAMPLERATE) + if activity in durations.keys(): + durations[activity] += (hop_length/SAMPLERATE) # Print status d = time.ticks_diff(time.ticks_ms(), start) print('classify', activity, list(out), d, 'ms') for classname, duration in durations.items(): - print(f'{classname}:\t\t\t{duration:.0f} seconds') + print(f'{classname}:\t\t\t{duration:.0f} s') # Send predictions over BLE - send_bluetooth_le(prediction_no, out) + try: + if BLE_ENABLED: + send_bluetooth_le(prediction_no, out) + except OSError as e: + print('send-ble-failed', e) # Update display - render_display(durations) + render_display(ssd, durations) prediction_no += 1 @@ -246,3 +268,4 @@ def main(): if __name__ == '__main__': main() + diff --git a/examples/har_trees/har_quicklabel.py b/examples/har_trees/har_quicklabel.py new file mode 100644 index 0000000..9b1a417 --- /dev/null +++ b/examples/har_trees/har_quicklabel.py @@ -0,0 +1,77 @@ + +""" +Take the activity markers in har_record.py as labels +""" + +import os + +import pandas + +from har_data2labelstudio import load_har_record + +def parse(): + import argparse + parser = argparse.ArgumentParser(description='') + + parser.add_argument('--dataset', type=str, default='uci_har', + help='Which dataset to use') + parser.add_argument('--config', type=str, default='data/configurations/uci_har.yaml', + help='Which dataset/training config to use') + + parser.add_argument('--data-dir', metavar='DIRECTORY', type=str, default='./data/raw/uci_har', + help='Where the input data is stored') + parser.add_argument('--out-dir', metavar='DIRECTORY', type=str, default='./data/processed', + help='Where to store results') + + parser.add_argument('--features', type=str, default='timebased', + help='Which feature-set to use') + parser.add_argument('--window-length', type=int, default=128, + help='Length of each window to classify (in samples)') + parser.add_argument('--window-hop', type=int, default=64, + help='How far to hop for next window to classify (in samples)') + + args = parser.parse_args() + + return args + + +def main(): + args = parse() + + dataset = args.dataset + out_path = os.path.join(args.out_dir, f'{dataset}.parquet') + data_path = os.path.join(args.data_dir) + + # Lookup data + recordings = load_har_record(data_path) + + # Create packed dataframe + dfs = [] + for filename, row in recordings.iterrows(): + classname = row.classname + filename = filename + #print(filename, classname) + + d = row.data + d = d.reset_index() + d['subject'] = 'unknown' + d['file'] = filename + d['activity'] = classname + dfs.append(d) + + #p = 'data/processed/pamap2.parquet' + + out = pandas.concat(dfs, ignore_index=True) + out.to_parquet(out_path) + + #return + # Sanity check + df = pandas.read_parquet(out_path) + print(df.columns) + print(df.activity.value_counts()) + print(df.file.value_counts()) + print(df.head(5)) + + +if __name__ == '__main__': + main() diff --git a/examples/har_trees/har_record.py b/examples/har_trees/har_record.py index 460158a..a351e23 100644 --- a/examples/har_trees/har_record.py +++ b/examples/har_trees/har_record.py @@ -3,6 +3,10 @@ import machine from machine import Pin, I2C +# On M5StickC we need to set HOLD pin to stay alive when on battery +hold_pin = machine.Pin(4, machine.Pin.OUT) +hold_pin.value(1) + from mpu6886 import MPU6886 # mpremote mip install "github:peterhinch/micropython-async/v3/primitives" @@ -19,13 +23,14 @@ # for display # mpremote mip install "github:peterhinch/micropython-nano-gui" -from color_setup import ssd from gui.core.writer import Writer from gui.core.nanogui import refresh from gui.widgets.meter import Meter from gui.widgets.label import Label import gui.fonts.courier20 as fixed +from color_setup import init_screen + # Cleanup after import frees considerable memory gc.collect() @@ -46,7 +51,7 @@ def decode_samples(buf : bytearray, samples : array.array, bytes_per_sample): samples[(i*3)+2] = z -def render_display(selected_class, recording : bool): +def render_display(ssd, selected_class, recording : bool): start_time = time.ticks_ms() ssd.fill(0) @@ -75,19 +80,29 @@ def render_display(selected_class, recording : bool): # Configuration classes = [ - 'jumpingjack', - 'lunge', - 'squat', + 'idle', + 'snake', + 'updown', + 'wave', 'other', ] samplerate = 100 chunk_length = 50 -file_duration = 10.0 +file_duration = 4.0 data_dir = 'har_record' def main(): + + # Internal LED on M5StickC PLUS2 + led_pin = machine.Pin(19, machine.Pin.OUT) + led_pin.value(1) + time.sleep(0.10) + led_pin.value(0) + + ssd = init_screen() + mpu = MPU6886(I2C(0, sda=21, scl=22, freq=100000)) # Enable FIFO at a fixed samplerate @@ -97,19 +112,12 @@ def main(): chunk = bytearray(mpu.bytes_per_sample*chunk_length) # raw bytes decoded = array.array('h', (0 for _ in range(3*chunk_length))) # decoded int16 - # Internal LED on M5StickC PLUS2 - led_pin = machine.Pin(19, machine.Pin.OUT) - - # On M5StickC we need to set HOLD pin to stay alive when on battery - hold_pin = machine.Pin(4, machine.Pin.OUT) - hold_pin.value(1) - # Support cycling between classes, to indicate which is being recorded class_selected = 0 def update_display(): c = classes[class_selected] - render_display(c, recorder._recording) + render_display(ssd, c, recorder._recording) led_pin.value(1 if recorder._recording else 0) def on_longpress(): @@ -117,6 +125,8 @@ def on_longpress(): if recorder._recording: recorder.stop() else: + c = classes[class_selected] + recorder.set_class(c) recorder.start() update_display() @@ -126,10 +136,9 @@ def on_doubleclick(): class_selected += 1 if class_selected >= len(classes): class_selected = 0 - c = classes[class_selected] - recorder.set_class(c) update_display() + c = classes[class_selected] print(f'har-record-cycle class={c}') button_pin = machine.Pin(37, machine.Pin.IN, machine.Pin.PULL_UP) # Button A on M5StickC PLUS2 @@ -174,3 +183,4 @@ async def run(): if __name__ == '__main__': main() + diff --git a/examples/har_trees/har_run.py b/examples/har_trees/har_run.py index ec476f8..2016fa0 100644 --- a/examples/har_trees/har_run.py +++ b/examples/har_trees/har_run.py @@ -64,10 +64,10 @@ def main(): model = emlearn_trees.new(10, 1000, 10) - dataset = 'har_uci' + dataset = 'uci_har' #dataset = 'har_exercise_1' - model_path = f'{dataset}_trees.csv' + model_path = f'{dataset}.trees.csv' # Load a CSV file with the model with open(model_path, 'r') as f: diff --git a/examples/har_trees/har_train.py b/examples/har_trees/har_train.py index 040cd12..8262a1e 100644 --- a/examples/har_trees/har_train.py +++ b/examples/har_trees/har_train.py @@ -6,10 +6,13 @@ import tempfile import subprocess import json +import itertools +import yaml import pandas import numpy import structlog +import joblib from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import f1_score, make_scorer, get_scorer, PrecisionRecallDisplay @@ -148,7 +151,7 @@ def assign_window_label(labels, majority=0.66): def timebased_features(windows : list[pandas.DataFrame], columns : list[str], - micropython_bin='micropython') -> pandas.DataFrame: + python_bin='python') -> pandas.DataFrame: #print('w', len(windows), columns) @@ -173,13 +176,13 @@ def timebased_features(windows : list[pandas.DataFrame], # Run MicroPython program args = [ - micropython_bin, + python_bin, feature_extraction_script, data_path, features_path, ] cmd = ' '.join(args) - #log.debug('run-micropython', cmd=cmd) + #log.debug('run-timebased', cmd=cmd) try: out = subprocess.check_output(args) except subprocess.CalledProcessError as e: @@ -195,6 +198,18 @@ def timebased_features(windows : list[pandas.DataFrame], return df +def batched_iterator(iterable, batch_size): + """Yield lists of size batch_size from iterable""" + iterator = iter(iterable) + while batch := list(itertools.islice(iterator, batch_size)): + yield batch + +def process_in_parallel_streaming(gen, process_item, batch_size=1000, n_jobs=-1): + for batch in batched_iterator(gen, batch_size): + yield from joblib.Parallel(n_jobs=n_jobs)( + joblib.delayed(process_item)(item) for item in batch + ) + def extract_features(sensordata : pandas.DataFrame, columns : list[str], groupby, @@ -220,9 +235,8 @@ def extract_features(sensordata : pandas.DataFrame, # Split into fixed-length windows features_values = [] - generator = extract_windows(sensordata, window_length, window_hop, groupby=groupby, time_column=time_column) - for windows in generator: - + + def process_one(windows) -> pandas.DataFrame: # drop invalid data windows = [ w for w in windows if not w[columns].isnull().values.any() ] @@ -247,6 +261,14 @@ def extract_features(sensordata : pandas.DataFrame, df[idx_column] = [w[idx_column].iloc[0] for w in windows] df = df.set_index(index_columns) + return df + + + data_generator = extract_windows(sensordata, window_length, window_hop, groupby=groupby, time_column=time_column) + feature_generator = process_in_parallel_streaming(data_generator, process_one, batch_size=10) + + for df in feature_generator: + features_values.append(df) out = pandas.concat(features_values) @@ -264,7 +286,14 @@ def export_model(path, out): cmodel.save(name='harmodel', format='csv', file=out) +def load_config(file_path): + + with open(file_path, 'r') as f: + data = yaml.safe_load(f) + return data + def run_pipeline(run, hyperparameters, dataset, + config, data_dir, out_dir, model_settings=dict(), @@ -272,68 +301,8 @@ def run_pipeline(run, hyperparameters, dataset, features='timebased', ): - dataset_config = { - 'uci_har': dict( - groups=['subject', 'experiment'], - data_columns = ['acc_x', 'acc_y', 'acc_z'], - classes = [ - #'STAND_TO_LIE', - #'SIT_TO_LIE', - #'LIE_TO_SIT', - #'STAND_TO_SIT', - #'LIE_TO_STAND', - #'SIT_TO_STAND', - 'STANDING', 'LAYING', 'SITTING', - 'WALKING', 'WALKING_UPSTAIRS', 'WALKING_DOWNSTAIRS', - ], - ), - 'pamap2': dict( - groups=['subject'], - data_columns = ['hand_acceleration_16g_x', 'hand_acceleration_16g_y', 'hand_acceleration_16g_z'], - classes = [ - #'transient', - 'walking', 'ironing', 'lying', 'standing', - 'Nordic_walking', 'sitting', 'vacuum_cleaning', - 'cycling', 'ascending_stairs', 'descending_stairs', - 'running', 'rope_jumping', - ], - ), - 'har_exercise_1': dict( - groups=['file'], - data_columns = ['x', 'y', 'z'], - classes = [ - #'mixed', - 'squat', 'jumpingjack', 'lunge', 'other', - ], - ), - 'toothbrush_hussain2021': dict( - groups=['subject'], - label_column = 'is_brushing', - time_column = 'elapsed', - data_columns = ['acc_x', 'acc_y', 'acc_z'], - #data_columns = ['gravity_x', 'gravity_y', 'gravity_z'], - #data_columns = ['motion_x', 'motion_y', 'motion_z'], - classes = [ - #'mixed', - 'True', 'False', - ], - ), - 'toothbrush_jonnor': dict( - groups=['session'], - label_column = 'is_brushing', - time_column = 'time', - data_columns = ['x', 'y', 'z'], - #data_columns = ['gravity_x', 'gravity_y', 'gravity_z'], - #data_columns = ['motion_x', 'motion_y', 'motion_z'], - classes = [ - #'mixed', - 'True', 'False', - ], - ), - } + dataset_config = load_config(config) - if not dataset in dataset_config.keys(): - raise ValueError(f"Unknown dataset {dataset}") if not os.path.exists(out_dir): os.makedirs(out_dir) @@ -341,30 +310,35 @@ def run_pipeline(run, hyperparameters, dataset, data_path = os.path.join(data_dir, f'{dataset}.parquet') data_load_start = time.time() + log.info('data-load-start', dataset=dataset) data = pandas.read_parquet(data_path) #print(data.index.names) #print(data.columns) - groups = dataset_config[dataset]['groups'] - data_columns = dataset_config[dataset]['data_columns'] - enabled_classes = dataset_config[dataset]['classes'] - label_column = dataset_config[dataset].get('label_column', 'activity') - time_column = dataset_config[dataset].get('time_column', 'time') - sensitivity = dataset_config[dataset].get('sensitivity', 2.0) + groups = dataset_config['groups'] + data_columns = dataset_config['data_columns'] + enabled_classes = dataset_config['classes'] + label_column = dataset_config.get('label_column', 'activity') + time_column = dataset_config.get('time_column', 'time') + sensitivity = dataset_config.get('sensitivity', 4.0) data[label_column] = data[label_column].astype(str) data_load_duration = time.time() - data_load_start - log.info('data-loaded', dataset=dataset, samples=len(data), duration=data_load_duration) + log.info('data-load-end', dataset=dataset, samples=len(data), duration=data_load_duration) feature_extraction_start = time.time() + log.info('feature-extraction-start', + dataset=dataset, + ) + window_length = model_settings['window_length'] features = extract_features(data, columns=data_columns, groupby=groups, features=features, sensitivity=sensitivity, - window_length=model_settings['window_length'], + window_length=window_length, window_hop=model_settings['window_hop'], label_column=label_column, time_column=time_column, @@ -412,16 +386,18 @@ def run_pipeline(run, hyperparameters, dataset, pickle.dump(estimator, file=f) # Export model with emlearn - model_path = os.path.join(out_dir, f'{dataset}_trees.csv') + model_path = os.path.join(out_dir, f'{dataset}.trees.csv') export_model(estimator_path, model_path) - # Save testdata + # Save metadata classes = estimator.classes_ class_mapping = dict(zip(classes, range(len(classes)))) - meta_path = os.path.join(out_dir, f'{dataset}.meta.json') + meta_path = os.path.join(out_dir, f'{dataset}.meta.json') + metadata = dict(classes=class_mapping, window_length=window_length) with open(meta_path, 'w') as f: - f.write(json.dumps(class_mapping)) + f.write(json.dumps(metadata)) + # Save testdata testdata_path = os.path.join(out_dir, f'{dataset}.testdata.npz') testdata = features.groupby(label_column, as_index=False).sample(n=10) # convert to class number/index @@ -461,6 +437,8 @@ def parse(): parser.add_argument('--dataset', type=str, default='uci_har', help='Which dataset to use') + parser.add_argument('--config', type=str, default='data/configurations/uci_har.yaml', + help='Which dataset/training config to use') parser.add_argument('--data-dir', metavar='DIRECTORY', type=str, default='./data/processed', help='Where the input data is stored') parser.add_argument('--out-dir', metavar='DIRECTORY', type=str, default='./', @@ -481,9 +459,6 @@ def parse(): def main(): args = parse() - dataset = args.dataset - out_dir = args.out_dir - data_dir = args.data_dir run_id = uuid.uuid4().hex.upper()[0:6] @@ -499,6 +474,7 @@ def main(): } results = run_pipeline(dataset=args.dataset, + config=args.config, out_dir=args.out_dir, data_dir=args.data_dir, run=run_id, diff --git a/examples/har_trees/har_uci.testdata.npz b/examples/har_trees/har_uci.testdata.npz deleted file mode 100644 index 8d417a5..0000000 Binary files a/examples/har_trees/har_uci.testdata.npz and /dev/null differ diff --git a/examples/har_trees/har_uci_trees.csv b/examples/har_trees/har_uci_trees.csv deleted file mode 100644 index 2d7b709..0000000 --- a/examples/har_trees/har_uci_trees.csv +++ /dev/null @@ -1,293 +0,0 @@ -f,92 -c,6 -l,0 -l,1 -l,2 -l,5 -l,3 -l,4 -r,0 -r,57 -r,114 -r,170 -r,226 -n,1,529.5,-1,1 -n,80,50.5,1,22 -n,13,-75.5,1,11 -n,20,257.0,1,-2 -n,13,-144.5,1,7 -n,8,13065.0,1,4 -n,89,2611.0,1,-3 -n,13,-320.5,1,-3 -n,5,1124.0,-3,-3 -n,20,19.5,-3,1 -n,18,1963.0,-3,-3 -n,24,-63.0,-3,1 -n,30,1346.0,-3,-3 -n,36,16575.5,1,9 -n,86,4552.0,1,7 -n,13,9.5,1,3 -n,20,-46.0,-2,1 -n,22,135.5,-3,-2 -n,25,-78.0,-2,1 -n,24,77.5,1,-2 -n,12,63.5,-2,-2 -n,25,47.5,-2,-3 -n,7,909.5,-2,-2 -n,1,1928.5,1,25 -n,0,1106.5,1,9 -n,74,930.0,1,5 -n,74,763.0,1,3 -n,4,903.5,1,-4 -n,44,29450.5,-4,-4 -n,50,-26.5,-4,-4 -n,4,810.5,-4,1 -n,82,2665.5,1,-5 -n,32,2836.5,-5,-4 -n,13,-207.5,1,8 -n,67,155.5,1,2 -n,27,156.5,-4,-4 -n,57,230.5,-4,1 -n,13,-310.5,-5,1 -n,1,1839.5,1,-5 -n,62,66.5,-5,1 -n,14,-403.5,-5,-5 -n,63,64.5,-4,1 -n,1,1875.5,1,-5 -n,61,27.5,-5,1 -n,50,-28.5,1,2 -n,61,39.5,-5,-5 -n,10,-92.5,1,-5 -n,24,-78.0,-5,-5 -n,67,198.5,1,2 -n,21,252.5,-6,-4 -n,33,1467.5,1,5 -n,3,1117.5,1,-5 -n,50,-50.5,1,2 -n,8,13308.0,-6,-6 -n,21,204.5,-6,-6 -n,32,3456.0,1,-6 -n,15,-158.0,-4,-6 -n,3,439.5,-1,1 -n,77,-228.5,1,35 -n,6,540.5,1,30 -n,15,-155.5,1,10 -n,73,64.5,1,4 -n,4,975.5,1,-4 -n,73,58.5,1,-4 -n,89,2474.5,-4,-4 -n,8,12700.5,1,3 -n,17,475.5,1,-4 -n,53,150.5,-4,-4 -n,67,350.5,1,-6 -n,0,1117.5,-4,-5 -n,33,1363.5,1,4 -n,1,1900.5,-5,1 -n,32,2775.5,-6,1 -n,36,17531.5,-6,-6 -n,1,1918.0,1,13 -n,63,75.5,1,3 -n,18,2326.5,-4,1 -n,26,197.5,-4,-4 -n,23,-344.5,-4,1 -n,73,70.5,1,4 -n,32,2613.5,1,-4 -n,32,2380.0,1,-5 -n,20,92.5,-5,-5 -n,1,1758.5,1,2 -n,63,97.5,-5,-5 -n,16,220.5,-5,1 -n,21,338.5,-5,-5 -n,67,275.5,-4,1 -n,20,-82.0,-6,-5 -n,1,2005.5,-4,1 -n,33,1475.5,1,-6 -n,50,-56.5,1,-6 -n,20,-209.5,-6,-6 -n,14,-78.5,1,11 -n,20,202.5,1,-2 -n,15,-142.5,1,7 -n,4,1154.5,1,4 -n,14,-325.5,1,2 -n,35,1606.5,-3,-3 -n,90,29191.0,-3,-3 -n,15,-203.5,-3,1 -n,27,76.5,-3,-3 -n,23,-32.5,-3,1 -n,18,1284.5,-3,-3 -n,31,1420.5,1,8 -n,11,112.5,1,5 -n,36,14756.5,1,2 -n,18,247.0,-2,-2 -n,77,-30.5,-3,1 -n,33,1377.5,-2,-2 -n,85,-4542.5,-2,1 -n,74,107.5,-2,-2 -n,31,1459.5,-2,1 -n,12,125.5,-2,-2 -n,3,435.5,-1,1 -n,72,11.5,1,21 -n,14,-78.5,1,11 -n,21,288.5,1,-2 -n,18,1582.0,1,3 -n,20,-23.5,-2,1 -n,35,1334.0,-3,-3 -n,10,-166.5,1,5 -n,23,-51.5,1,3 -n,25,-145.5,-3,1 -n,25,-75.5,-3,-3 -n,8,13140.5,-3,-3 -n,20,74.5,-3,-3 -n,31,1430.5,1,8 -n,10,3.5,1,3 -n,24,-39.0,-2,1 -n,1,1188.5,-2,-3 -n,25,-86.5,-3,1 -n,67,64.5,1,2 -n,7,1155.5,-2,-2 -n,7,1158.5,-2,-2 -n,21,161.0,-2,-2 -n,1,1985.5,1,26 -n,64,893.5,1,7 -n,18,2519.0,1,2 -n,64,692.0,-4,-5 -n,2,715.5,1,2 -n,64,843.5,-4,-4 -n,74,738.0,1,-4 -n,14,-324.5,-4,-4 -n,13,-303.5,1,5 -n,81,241.5,1,-5 -n,34,514.0,-4,1 -n,67,288.5,1,-4 -n,4,890.0,-4,-4 -n,23,-367.5,-4,1 -n,1,1837.5,1,9 -n,67,152.0,-5,1 -n,48,-312.5,1,-5 -n,22,-779.0,-5,1 -n,74,874.5,1,3 -n,32,2560.5,1,-5 -n,5,1304.5,-5,-5 -n,22,-624.0,-5,1 -n,57,291.5,-5,-5 -n,74,825.0,-4,1 -n,32,2666.5,-5,1 -n,33,1451.5,-6,-5 -n,33,1433.5,1,5 -n,3,1091.5,1,-6 -n,74,751.0,-6,1 -n,50,-45.5,1,-6 -n,87,96.5,-6,-6 -n,73,75.5,-4,1 -n,3,1077.5,1,-5 -n,7,1175.5,-6,-6 -n,80,50.5,1,23 -n,4,434.0,-1,1 -n,13,-77.5,1,11 -n,22,231.5,1,-2 -n,13,-165.5,1,6 -n,20,-53.5,1,3 -n,74,163.0,-3,1 -n,34,25.5,-3,-3 -n,6,13.5,1,-3 -n,54,1166.5,-3,-3 -n,22,-130.5,-3,1 -n,24,25.0,-3,1 -n,22,76.5,-3,-3 -n,15,102.5,1,8 -n,12,-85.5,-3,1 -n,28,2997.0,1,-2 -n,36,14756.5,1,3 -n,17,23.5,-2,1 -n,28,703.5,-2,-2 -n,28,1459.0,-3,1 -n,43,2657.5,-2,-2 -n,5,1147.5,-2,1 -n,20,93.5,-2,-2 -n,1,2041.5,1,27 -n,67,222.5,1,11 -n,15,-81.5,1,8 -n,4,921.5,1,4 -n,82,2414.0,1,-4 -n,74,845.0,1,-4 -n,35,1660.0,-4,-4 -n,68,-257.0,-5,1 -n,64,1014.0,1,-4 -n,18,3323.5,-4,-4 -n,35,1574.5,1,-4 -n,64,1064.0,-5,-5 -n,3,1070.5,1,7 -n,33,1453.5,1,2 -n,58,-342.5,-6,-6 -n,10,-317.5,1,2 -n,25,-170.5,-4,-4 -n,1,1799.5,1,-6 -n,11,119.5,-5,-4 -n,1,1909.5,1,7 -n,74,673.5,-5,1 -n,33,1582.0,1,-5 -n,25,-62.5,-5,1 -n,10,-107.0,1,-5 -n,3,1108.5,-5,1 -n,42,4244.0,-5,-5 -n,79,0.5,-6,-5 -n,51,77.5,1,4 -n,50,-45.5,1,-6 -n,67,244.5,-6,1 -n,48,-685.5,-6,-6 -n,52,150.5,-4,1 -n,20,-25.5,-6,-6 -n,80,50.5,1,23 -n,8,5088.5,-1,1 -n,14,-73.5,1,11 -n,18,1954.5,1,6 -n,22,-114.5,1,2 -n,21,-163.5,-3,-2 -n,36,16393.0,1,-3 -n,3,1169.5,1,-3 -n,1,1170.5,-3,-3 -n,2,993.5,-3,1 -n,24,-80.5,1,2 -n,25,-143.5,-3,-3 -n,18,2095.0,-3,-3 -n,13,104.5,1,7 -n,27,268.5,1,-2 -n,85,-4578.5,-3,1 -n,38,-249.5,-2,1 -n,27,80.5,-2,1 -n,77,-30.5,-2,1 -n,31,1339.5,-2,-2 -n,3,1139.5,1,2 -n,86,4328.5,-2,-2 -n,2,1140.5,-2,-2 -n,1,2041.5,1,25 -n,15,-155.5,1,8 -n,74,764.5,1,3 -n,2,728.5,1,-4 -n,4,917.5,-4,-4 -n,8,12698.5,1,3 -n,18,5438.5,1,-4 -n,2,611.5,-4,-4 -n,4,912.0,-4,-5 -n,5,1266.5,1,6 -n,67,201.5,1,2 -n,2,733.5,-4,-4 -n,20,-262.0,1,2 -n,31,993.5,-4,-4 -n,1,1743.5,-5,-6 -n,32,2611.5,1,5 -n,74,799.0,1,3 -n,64,988.5,-4,1 -n,34,498.5,-5,-5 -n,4,920.5,-5,-5 -n,58,-416.5,1,3 -n,64,1445.5,-5,1 -n,1,1866.5,-5,-5 -n,67,216.5,-4,1 -n,70,-15.5,-6,-5 -n,79,3.5,1,-6 -n,13,-241.5,1,2 -n,21,241.5,-6,-6 -n,48,-799.5,-6,1 -n,74,771.5,-6,1 -n,71,32.5,-6,-6 \ No newline at end of file diff --git a/examples/har_trees/npyfile.py b/examples/har_trees/npyfile.py new file mode 100644 index 0000000..a50ff3f --- /dev/null +++ b/examples/har_trees/npyfile.py @@ -0,0 +1,310 @@ + +""" +Support for Numpy .npy files for MicroPython + +References: +https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html +https://numpy.org/doc/1.13/neps/npy-format.html#a-simple-file-format-for-numpy-arrays +""" + +import struct +import array + +NPY_MAGIC = b'\x93NUMPY' + +format_mapping = { + # npy format => (array.array typecode, itemsize in bytes) + # floating point + b'f8': ('d', 8), + b'f4': ('f', 4), + + # bytes + b'i1': ('b', 1), + b'u1': ('B', 1), + +} + +# find the correct array.array formats to use +# can unfortunately depend on platform +# In particular on 64 bit, the integers are 8 bytes, and on 32 bit they are 4 +def array_typecode_itemsize(typecode): + arr = array.array(typecode, [0]) + b = bytes(arr) + return len(b) + +for typecode in ['q', 'l', 'h', 'i']: + size = array_typecode_itemsize(typecode) + dummy = bytes(size) + struct.unpack(typecode, dummy) + key = f'i{size}'.encode('ascii') + format_mapping[key] = (typecode, size) + +for typecode in ['Q', 'L', 'H', 'I']: + size = array_typecode_itemsize(typecode) + dummy = bytes(size) + struct.unpack(typecode, dummy) + key = f'u{size}'.encode('ascii') + format_mapping[key] = (typecode, size) + + +def find_section(data, prefix, suffix): + start = data.index(prefix) + len(prefix) + end = start + data[start:].index(suffix) + + section = data[start:end] + return section + +def array_tobytes_generator(arr): + # array.array.tobytes() is missing in MicroPython =/ + typecode = array_typecode(arr) + for item in arr: + buf = struct.pack(typecode, item) + yield buf + +def array_frombytes(typecode, buf): + + if True: + arr = array.array(typecode, buf) + return arr + else: + # XXX: dead code + arr = array.array(typecode, buf) + itemsize = len(buf) // len(arr) + for idx in range(len(arr)): + start = idx*itemsize + arr[idx] = struct.unpack_from('<'+typecode, buf, start)[0] + + return arr + +def array_typecode(arr): + typecode = str(arr)[7:8] + return typecode + +def compute_items(shape): + total_items = 1 + for d in shape: + total_items *= d + return total_items + +class Reader(): + + def __init__(self, filelike, header_maxlength=16*10): + + if isinstance(filelike, str): + self.file = open(filelike, 'rb') + else: + self.file = filelike + + self.header_maxlength = header_maxlength + + def close(self): + if self.file: + self.file.close() + self.file = None + + def __enter__(self): + self._read_header() + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self.close() + + def _read_header(self): + + # Read header data + data = self.file.read(self.header_maxlength) + + # Check magic + magic = data[0:len(NPY_MAGIC)] + assert magic == NPY_MAGIC, magic + + # Check version + major, minor = struct.unpack_from('BB', data, len(NPY_MAGIC)) + if major == 0x01: + header_length = struct.unpack_from('= 1, dimensions + assert dimensions <= 5, dimensions + + # Construct header info + dtype_matches = [ key for key, (tc, size) in format_mapping.items() if tc == self.typecode ] + assert len(dtype_matches) == 1, dtype_matches + dtype = '<'+dtype_matches[0].decode('ascii') + if len(shape) == 1: + shape_str = f'{shape[0]},' # 1-length tuple must have trailing , + else: + shape_str = ','.join((str(d) for d in shape)) + + header = f"{{'descr': '{dtype}', 'fortran_order': False, 'shape': ({shape_str}), }}" + #print('wh', header) + + # Padded to ensure data start is aligened to 16 bytes + data_start = len(NPY_MAGIC)+2+2+len(header) + padding = 16-(data_start % 16) + header = header + (' ' * padding) + header_length = len(header) + data_start = len(NPY_MAGIC)+2+2+len(header) + assert data_start % 16 == 0, data_start + + self.file.write(NPY_MAGIC) + self.file.write(bytes([0x01, 0x00])) # version + self.file.write(struct.pack(' tuple[tuple, array.array]: + """ + Load array from .npy file + + Convenience function for doing it in one shot. + For streaming, use npyfile.Reader instead + """ + + chunks = [] + with Reader(filelike) as reader: + # Just read everything in one chunk + total_items = compute_items(reader.shape) + for c in reader.read_data_chunks(total_items): + chunks.append(c) + + assert len(chunks) == 1 + return reader.shape, chunks[0] + +def save(filelike, arr : array.array, shape=None): + """ + Save array as .npy file + + Convenience function for doing it in one shot. + For streaming, use npyfile.Writer instead + """ + + if shape is None: + # default to 1d + shape = (len(arr), ) + + typecode = array_typecode(arr) + total = compute_items(shape) + assert total == len(arr), (shape, total, len(arr)) + + with Writer(filelike, shape, typecode) as f: + f.write_values(arr) + + diff --git a/examples/har_trees/requirements.txt b/examples/har_trees/requirements.txt index f337a66..3f39f03 100644 --- a/examples/har_trees/requirements.txt +++ b/examples/har_trees/requirements.txt @@ -1,8 +1,10 @@ pandas pyarrow git+https://github.com/jonnor/leaf-clustering -emlearn +emlearn>=0.22.0 scikit-learn setuptools structlog matplotlib +mpremote +pyyaml diff --git a/examples/har_trees/timebased.py b/examples/har_trees/timebased.py index 46022ef..5a056c8 100644 --- a/examples/har_trees/timebased.py +++ b/examples/har_trees/timebased.py @@ -9,13 +9,28 @@ # The input data must alreayd be segmented in windows. For each window, all features are computed. # Author: Atis Elsts, 2019 - import os import sys import math import array import time +if sys.implementation.name != 'micropython': + # shim for micropython.native + class micropython: + @classmethod + def native(cls, func): + # do nothing + return func + + def _time_ticks_ms(): + return int(time.time()/1000.0) + def _time_ticks_diff(a, b): + return a-b + + time.ticks_ms = _time_ticks_ms + time.ticks_diff = _time_ticks_diff + ######################################### DATA_TYPECODE = 'h' # int16 @@ -114,17 +129,23 @@ def jerk_filter(inp, out): @micropython.native def norm_filter_l1(x, y, z, out): for i in range(len(x)): - out[i] = abs(x[i]) + abs(y[i]) + abs(z[i]) + v = abs(x[i]) + abs(y[i]) + abs(z[i]) + v = min(v, 2**15-1) + out[i] = v @micropython.native def norm_filter_l2(x, y, z, out): for i in range(len(x)): - out[i] = (x[i]*x[i] + y[i]*y[i] + z[i]*z[i])**0.5 + v = (x[i]*x[i] + y[i]*y[i] + z[i]*z[i])**0.5 + v = min(v, 2**15-1) + out[i] = v @micropython.native def norm_filter_l2_squared(x, y, z, out): for i in range(len(x)): - out[i] = (x[i]*x[i] + y[i]*y[i] + z[i]*z[i]) + v = (x[i]*x[i] + y[i]*y[i] + z[i]*z[i]) + v = min(v, 2**15-1) + out[i] = v ########################################## diff --git a/examples/har_trees/uci_har.testdata.npz b/examples/har_trees/uci_har.testdata.npz new file mode 100644 index 0000000..5388f58 Binary files /dev/null and b/examples/har_trees/uci_har.testdata.npz differ diff --git a/examples/har_trees/uci_har.trees.csv b/examples/har_trees/uci_har.trees.csv new file mode 100644 index 0000000..3fb22fd --- /dev/null +++ b/examples/har_trees/uci_har.trees.csv @@ -0,0 +1,146 @@ +f,92 +c,6 +l,0 +l,2 +l,1 +l,4 +l,5 +l,3 +r,0 +r,12 +r,26 +r,39 +r,52 +r,65 +r,79 +r,92 +r,105 +r,116 +n,52,125.5,1,6 +n,2,2986.5,-1,1 +n,14,-562.5,1,3 +n,17,1227.5,-2,1 +n,0,8146.5,-2,-2 +n,30,10500.5,-3,-3 +n,33,9907.5,1,2 +n,32,20807.5,-4,-4 +n,1,14335.0,1,-4 +n,13,-1484.5,1,2 +n,36,7031.0,-5,-5 +n,58,-2309.5,-6,-6 +n,0,3122.5,-1,1 +n,80,325.0,1,6 +n,13,-631.0,1,3 +n,18,13721.5,-2,1 +n,82,597.5,-2,-2 +n,0,8154.0,-3,1 +n,18,5463.0,-3,-3 +n,67,1438.5,1,2 +n,0,8025.5,-5,-5 +n,1,13919.5,1,3 +n,3,7628.0,-5,1 +n,7,8366.5,-6,-6 +n,3,7616.5,1,-4 +n,33,9589.5,-4,-4 +n,4,2565.0,-1,1 +n,62,148.5,1,5 +n,14,-562.5,1,3 +n,58,-1234.0,1,-2 +n,82,586.5,-2,-2 +n,5,8197.0,-3,-3 +n,1,14431.5,1,6 +n,30,11772.5,1,4 +n,74,4999.0,-5,1 +n,1,13214.5,1,-6 +n,0,8193.5,-6,-6 +n,13,-2235.0,-5,-5 +n,50,-409.5,-4,-4 +n,7,3254.5,-1,1 +n,80,365.5,1,6 +n,13,-540.0,1,3 +n,18,13965.5,-2,1 +n,20,-387.5,-2,-2 +n,3,8162.5,-3,1 +n,56,2.5,-3,-3 +n,78,6076.5,1,5 +n,13,-1484.5,1,3 +n,63,759.5,1,-6 +n,74,5225.5,-5,-5 +n,3,7736.5,-4,-6 +n,50,-426.5,-4,-4 +n,52,136.5,1,7 +n,5,3185.0,-1,1 +n,15,-528.5,1,3 +n,17,1227.5,-2,1 +n,26,79.5,-2,-2 +n,7,8164.5,-3,1 +n,18,5208.0,-3,-3 +n,33,9959.0,1,2 +n,50,-386.5,-4,-4 +n,4,6445.0,1,2 +n,53,1320.5,-5,-4 +n,73,438.5,-5,1 +n,5,9289.0,-6,-6 +n,7,3372.0,-1,1 +n,34,1593.5,1,6 +n,13,-551.5,1,3 +n,14,-1245.5,1,-2 +n,78,169.5,-2,-2 +n,35,10448.0,1,-3 +n,17,447.0,-3,-3 +n,50,-420.5,1,2 +n,1,14966.5,-4,-4 +n,3,7753.5,1,3 +n,1,13658.0,1,-4 +n,67,1780.0,-5,-5 +n,57,1785.5,-5,1 +n,2,4954.5,-6,-6 +n,1,10040.0,1,7 +n,5,3202.0,-1,1 +n,10,-544.5,1,3 +n,10,-1216.5,1,-2 +n,10,-1749.5,-2,-2 +n,35,10501.5,1,-3 +n,16,67.5,-3,-3 +n,50,-420.5,1,2 +n,1,14966.5,-4,-4 +n,15,-540.0,1,3 +n,73,438.5,-5,1 +n,36,7692.0,-6,-5 +n,3,7821.5,-4,-6 +n,5,3185.0,-1,1 +n,6,836.5,1,5 +n,14,-562.5,1,3 +n,14,-1245.5,1,-2 +n,11,-1512.5,-2,-2 +n,30,10499.5,-3,-3 +n,1,14534.0,1,6 +n,0,7991.5,1,2 +n,13,-2303.5,-5,-5 +n,74,4869.5,-5,1 +n,1,13316.5,1,-6 +n,5,9482.5,-6,-6 +n,33,9959.0,-4,-4 +n,3,3168.0,-1,1 +n,6,797.0,1,4 +n,12,-767.5,1,2 +n,15,-1222.5,-2,-2 +n,5,8174.0,-3,-3 +n,33,9959.0,1,2 +n,32,20477.5,-4,-4 +n,0,8015.5,1,2 +n,73,709.5,-5,-5 +n,73,447.5,-5,1 +n,32,19215.0,-6,-6 +n,52,148.0,1,6 +n,5,3185.0,-1,1 +n,58,-551.5,1,3 +n,14,-1234.0,1,-2 +n,35,11171.0,-2,-2 +n,3,8151.5,-3,-3 +n,1,13863.0,1,5 +n,0,7993.5,1,2 +n,14,-3156.5,-5,-5 +n,64,6804.0,-5,1 +n,71,182.5,-6,-6 +n,33,9948.0,-4,-4 \ No newline at end of file diff --git a/paper/AccelerometerExample.ipynb b/paper/AccelerometerExample.ipynb new file mode 100644 index 0000000..3bf072d --- /dev/null +++ b/paper/AccelerometerExample.ipynb @@ -0,0 +1,1343 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e428c621-eaa3-4992-abc2-2649cc013e0f", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "49e672de-47ef-49f8-900a-842c6d2cddf7", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas\n", + "import numpy\n", + "import os\n", + "\n", + "import plotly.io as pio\n", + "pio.renderers.default='notebook'\n", + "\n", + "from plotting import plot_timeline, find_runs, configure_xaxis\n", + "from plotting import make_label_colors, make_timeline_plot\n", + "\n", + "\n", + "#import seaborn\n", + "#from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a69ae41-2f6d-4cbd-9fce-ee18e05a3877", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "58e5bc29-2bfb-44c2-8770-a8a9db5cf51e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
activityheartratehand_temperaturehand_acceleration_16g_xhand_acceleration_16g_yhand_acceleration_16g_zhand_acceleration_6g_xhand_acceleration_6g_yhand_acceleration_6g_zhand_gyro_x...ankle_chest_hand_acceleration_16g_zankle_chest_hand_acceleration_6g_xankle_chest_hand_acceleration_6g_yankle_chest_hand_acceleration_6g_zankle_chest_hand_gyro_xankle_chest_hand_gyro_yankle_chest_hand_gyro_zankle_chest_hand_magnetometer_xankle_chest_hand_magnetometer_yankle_chest_hand_magnetometer_z
subjecttime
subject1010 days 00:00:08.380000transient104.030.00002.3722308.600743.510482.439548.761653.35465-0.092217...-0.0997979.646890-1.5557600.3104040.0083000.009250-0.017580-61.1888-38.9599-58.1438
0 days 00:00:08.390000transientNaN30.00002.1883708.565603.661792.394948.550813.64207-0.024413...-0.2156879.616700-1.6163000.280488-0.006577-0.0046380.000368-59.8479-38.8919-58.5253
0 days 00:00:08.400000transientNaN30.00002.3735708.601073.548982.305148.536443.73280-0.057976...0.0929149.631730-1.5860500.2803110.0030140.0001480.022495-60.7361-39.4138-58.3999
0 days 00:00:08.410000transientNaN30.00002.0747308.528533.660212.335288.536223.73277-0.002352...0.0545459.631970-1.6313500.3409970.003175-0.0203010.011275-60.4091-38.7635-58.3956
0 days 00:00:08.420000transientNaN30.00002.2293608.831223.700002.230558.597413.762950.012269...-0.0608099.646990-1.6464700.3409650.012698-0.014303-0.002823-61.5199-39.3879-58.2694
.....................................................................
subject1070 days 00:52:21.860000runningNaN32.4375-16.89810036.83070-9.03542-21.0821038.29330-9.642457.472180...-4.1376100.7525605.001190-4.0223902.214250-0.9996744.130140-33.8582-24.289118.1954
0 days 00:52:21.870000runningNaN32.4375-10.76430035.74550-6.20728-16.1218037.59390-8.724846.596900...-3.051850-0.5816414.290900-3.3016601.878830-1.2854505.018300-34.9019-22.689219.1827
0 days 00:52:21.880000runningNaN32.4375-5.81525032.72480-4.46800-10.3112036.01390-6.160194.488900...-2.876610-1.0000502.703470-2.6586601.841810-1.7602205.733740-36.2194-19.915220.7350
0 days 00:52:21.890000runningNaN32.4375-2.32641028.67670-4.32511-5.1415132.76120-4.663382.339430...-2.6133100.2629991.358090-2.2826602.114310-2.1130206.359770-38.4197-17.482622.5682
0 days 00:52:21.900000runningNaN32.4375-0.58442923.61750-4.24346-1.8636028.02580-4.414740.675832...-2.8452002.711970-0.456113-2.0779302.397300-2.4399306.796220-38.8587-14.238823.6976
\n", + "

2872533 rows × 41 columns

\n", + "
" + ], + "text/plain": [ + " activity heartrate hand_temperature \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 transient 104.0 30.0000 \n", + " 0 days 00:00:08.390000 transient NaN 30.0000 \n", + " 0 days 00:00:08.400000 transient NaN 30.0000 \n", + " 0 days 00:00:08.410000 transient NaN 30.0000 \n", + " 0 days 00:00:08.420000 transient NaN 30.0000 \n", + "... ... ... ... \n", + "subject107 0 days 00:52:21.860000 running NaN 32.4375 \n", + " 0 days 00:52:21.870000 running NaN 32.4375 \n", + " 0 days 00:52:21.880000 running NaN 32.4375 \n", + " 0 days 00:52:21.890000 running NaN 32.4375 \n", + " 0 days 00:52:21.900000 running NaN 32.4375 \n", + "\n", + " hand_acceleration_16g_x \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 2.372230 \n", + " 0 days 00:00:08.390000 2.188370 \n", + " 0 days 00:00:08.400000 2.373570 \n", + " 0 days 00:00:08.410000 2.074730 \n", + " 0 days 00:00:08.420000 2.229360 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -16.898100 \n", + " 0 days 00:52:21.870000 -10.764300 \n", + " 0 days 00:52:21.880000 -5.815250 \n", + " 0 days 00:52:21.890000 -2.326410 \n", + " 0 days 00:52:21.900000 -0.584429 \n", + "\n", + " hand_acceleration_16g_y \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 8.60074 \n", + " 0 days 00:00:08.390000 8.56560 \n", + " 0 days 00:00:08.400000 8.60107 \n", + " 0 days 00:00:08.410000 8.52853 \n", + " 0 days 00:00:08.420000 8.83122 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 36.83070 \n", + " 0 days 00:52:21.870000 35.74550 \n", + " 0 days 00:52:21.880000 32.72480 \n", + " 0 days 00:52:21.890000 28.67670 \n", + " 0 days 00:52:21.900000 23.61750 \n", + "\n", + " hand_acceleration_16g_z \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 3.51048 \n", + " 0 days 00:00:08.390000 3.66179 \n", + " 0 days 00:00:08.400000 3.54898 \n", + " 0 days 00:00:08.410000 3.66021 \n", + " 0 days 00:00:08.420000 3.70000 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -9.03542 \n", + " 0 days 00:52:21.870000 -6.20728 \n", + " 0 days 00:52:21.880000 -4.46800 \n", + " 0 days 00:52:21.890000 -4.32511 \n", + " 0 days 00:52:21.900000 -4.24346 \n", + "\n", + " hand_acceleration_6g_x \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 2.43954 \n", + " 0 days 00:00:08.390000 2.39494 \n", + " 0 days 00:00:08.400000 2.30514 \n", + " 0 days 00:00:08.410000 2.33528 \n", + " 0 days 00:00:08.420000 2.23055 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -21.08210 \n", + " 0 days 00:52:21.870000 -16.12180 \n", + " 0 days 00:52:21.880000 -10.31120 \n", + " 0 days 00:52:21.890000 -5.14151 \n", + " 0 days 00:52:21.900000 -1.86360 \n", + "\n", + " hand_acceleration_6g_y \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 8.76165 \n", + " 0 days 00:00:08.390000 8.55081 \n", + " 0 days 00:00:08.400000 8.53644 \n", + " 0 days 00:00:08.410000 8.53622 \n", + " 0 days 00:00:08.420000 8.59741 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 38.29330 \n", + " 0 days 00:52:21.870000 37.59390 \n", + " 0 days 00:52:21.880000 36.01390 \n", + " 0 days 00:52:21.890000 32.76120 \n", + " 0 days 00:52:21.900000 28.02580 \n", + "\n", + " hand_acceleration_6g_z hand_gyro_x ... \\\n", + "subject time ... \n", + "subject101 0 days 00:00:08.380000 3.35465 -0.092217 ... \n", + " 0 days 00:00:08.390000 3.64207 -0.024413 ... \n", + " 0 days 00:00:08.400000 3.73280 -0.057976 ... \n", + " 0 days 00:00:08.410000 3.73277 -0.002352 ... \n", + " 0 days 00:00:08.420000 3.76295 0.012269 ... \n", + "... ... ... ... \n", + "subject107 0 days 00:52:21.860000 -9.64245 7.472180 ... \n", + " 0 days 00:52:21.870000 -8.72484 6.596900 ... \n", + " 0 days 00:52:21.880000 -6.16019 4.488900 ... \n", + " 0 days 00:52:21.890000 -4.66338 2.339430 ... \n", + " 0 days 00:52:21.900000 -4.41474 0.675832 ... \n", + "\n", + " ankle_chest_hand_acceleration_16g_z \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 -0.099797 \n", + " 0 days 00:00:08.390000 -0.215687 \n", + " 0 days 00:00:08.400000 0.092914 \n", + " 0 days 00:00:08.410000 0.054545 \n", + " 0 days 00:00:08.420000 -0.060809 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -4.137610 \n", + " 0 days 00:52:21.870000 -3.051850 \n", + " 0 days 00:52:21.880000 -2.876610 \n", + " 0 days 00:52:21.890000 -2.613310 \n", + " 0 days 00:52:21.900000 -2.845200 \n", + "\n", + " ankle_chest_hand_acceleration_6g_x \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 9.646890 \n", + " 0 days 00:00:08.390000 9.616700 \n", + " 0 days 00:00:08.400000 9.631730 \n", + " 0 days 00:00:08.410000 9.631970 \n", + " 0 days 00:00:08.420000 9.646990 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 0.752560 \n", + " 0 days 00:52:21.870000 -0.581641 \n", + " 0 days 00:52:21.880000 -1.000050 \n", + " 0 days 00:52:21.890000 0.262999 \n", + " 0 days 00:52:21.900000 2.711970 \n", + "\n", + " ankle_chest_hand_acceleration_6g_y \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 -1.555760 \n", + " 0 days 00:00:08.390000 -1.616300 \n", + " 0 days 00:00:08.400000 -1.586050 \n", + " 0 days 00:00:08.410000 -1.631350 \n", + " 0 days 00:00:08.420000 -1.646470 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 5.001190 \n", + " 0 days 00:52:21.870000 4.290900 \n", + " 0 days 00:52:21.880000 2.703470 \n", + " 0 days 00:52:21.890000 1.358090 \n", + " 0 days 00:52:21.900000 -0.456113 \n", + "\n", + " ankle_chest_hand_acceleration_6g_z \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 0.310404 \n", + " 0 days 00:00:08.390000 0.280488 \n", + " 0 days 00:00:08.400000 0.280311 \n", + " 0 days 00:00:08.410000 0.340997 \n", + " 0 days 00:00:08.420000 0.340965 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -4.022390 \n", + " 0 days 00:52:21.870000 -3.301660 \n", + " 0 days 00:52:21.880000 -2.658660 \n", + " 0 days 00:52:21.890000 -2.282660 \n", + " 0 days 00:52:21.900000 -2.077930 \n", + "\n", + " ankle_chest_hand_gyro_x \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 0.008300 \n", + " 0 days 00:00:08.390000 -0.006577 \n", + " 0 days 00:00:08.400000 0.003014 \n", + " 0 days 00:00:08.410000 0.003175 \n", + " 0 days 00:00:08.420000 0.012698 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 2.214250 \n", + " 0 days 00:52:21.870000 1.878830 \n", + " 0 days 00:52:21.880000 1.841810 \n", + " 0 days 00:52:21.890000 2.114310 \n", + " 0 days 00:52:21.900000 2.397300 \n", + "\n", + " ankle_chest_hand_gyro_y \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 0.009250 \n", + " 0 days 00:00:08.390000 -0.004638 \n", + " 0 days 00:00:08.400000 0.000148 \n", + " 0 days 00:00:08.410000 -0.020301 \n", + " 0 days 00:00:08.420000 -0.014303 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -0.999674 \n", + " 0 days 00:52:21.870000 -1.285450 \n", + " 0 days 00:52:21.880000 -1.760220 \n", + " 0 days 00:52:21.890000 -2.113020 \n", + " 0 days 00:52:21.900000 -2.439930 \n", + "\n", + " ankle_chest_hand_gyro_z \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 -0.017580 \n", + " 0 days 00:00:08.390000 0.000368 \n", + " 0 days 00:00:08.400000 0.022495 \n", + " 0 days 00:00:08.410000 0.011275 \n", + " 0 days 00:00:08.420000 -0.002823 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 4.130140 \n", + " 0 days 00:52:21.870000 5.018300 \n", + " 0 days 00:52:21.880000 5.733740 \n", + " 0 days 00:52:21.890000 6.359770 \n", + " 0 days 00:52:21.900000 6.796220 \n", + "\n", + " ankle_chest_hand_magnetometer_x \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 -61.1888 \n", + " 0 days 00:00:08.390000 -59.8479 \n", + " 0 days 00:00:08.400000 -60.7361 \n", + " 0 days 00:00:08.410000 -60.4091 \n", + " 0 days 00:00:08.420000 -61.5199 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -33.8582 \n", + " 0 days 00:52:21.870000 -34.9019 \n", + " 0 days 00:52:21.880000 -36.2194 \n", + " 0 days 00:52:21.890000 -38.4197 \n", + " 0 days 00:52:21.900000 -38.8587 \n", + "\n", + " ankle_chest_hand_magnetometer_y \\\n", + "subject time \n", + "subject101 0 days 00:00:08.380000 -38.9599 \n", + " 0 days 00:00:08.390000 -38.8919 \n", + " 0 days 00:00:08.400000 -39.4138 \n", + " 0 days 00:00:08.410000 -38.7635 \n", + " 0 days 00:00:08.420000 -39.3879 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 -24.2891 \n", + " 0 days 00:52:21.870000 -22.6892 \n", + " 0 days 00:52:21.880000 -19.9152 \n", + " 0 days 00:52:21.890000 -17.4826 \n", + " 0 days 00:52:21.900000 -14.2388 \n", + "\n", + " ankle_chest_hand_magnetometer_z \n", + "subject time \n", + "subject101 0 days 00:00:08.380000 -58.1438 \n", + " 0 days 00:00:08.390000 -58.5253 \n", + " 0 days 00:00:08.400000 -58.3999 \n", + " 0 days 00:00:08.410000 -58.3956 \n", + " 0 days 00:00:08.420000 -58.2694 \n", + "... ... \n", + "subject107 0 days 00:52:21.860000 18.1954 \n", + " 0 days 00:52:21.870000 19.1827 \n", + " 0 days 00:52:21.880000 20.7350 \n", + " 0 days 00:52:21.890000 22.5682 \n", + " 0 days 00:52:21.900000 23.6976 \n", + "\n", + "[2872533 rows x 41 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_path = '../examples/har_trees/data/processed/pamap2.parquet'\n", + "data = pandas.read_parquet(data_path)\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0cbecc36-1664-4b77-bee2-6ce42480dda8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['activity', 'heartrate', 'hand_temperature', 'hand_acceleration_16g_x',\n", + " 'hand_acceleration_16g_y', 'hand_acceleration_16g_z',\n", + " 'hand_acceleration_6g_x', 'hand_acceleration_6g_y',\n", + " 'hand_acceleration_6g_z', 'hand_gyro_x', 'hand_gyro_y', 'hand_gyro_z',\n", + " 'hand_magnetometer_x', 'hand_magnetometer_y', 'hand_magnetometer_z',\n", + " 'chest_hand_temperature', 'chest_hand_acceleration_16g_x',\n", + " 'chest_hand_acceleration_16g_y', 'chest_hand_acceleration_16g_z',\n", + " 'chest_hand_acceleration_6g_x', 'chest_hand_acceleration_6g_y',\n", + " 'chest_hand_acceleration_6g_z', 'chest_hand_gyro_x',\n", + " 'chest_hand_gyro_y', 'chest_hand_gyro_z', 'chest_hand_magnetometer_x',\n", + " 'chest_hand_magnetometer_y', 'chest_hand_magnetometer_z',\n", + " 'ankle_chest_hand_temperature', 'ankle_chest_hand_acceleration_16g_x',\n", + " 'ankle_chest_hand_acceleration_16g_y',\n", + " 'ankle_chest_hand_acceleration_16g_z',\n", + " 'ankle_chest_hand_acceleration_6g_x',\n", + " 'ankle_chest_hand_acceleration_6g_y',\n", + " 'ankle_chest_hand_acceleration_6g_z', 'ankle_chest_hand_gyro_x',\n", + " 'ankle_chest_hand_gyro_y', 'ankle_chest_hand_gyro_z',\n", + " 'ankle_chest_hand_magnetometer_x', 'ankle_chest_hand_magnetometer_y',\n", + " 'ankle_chest_hand_magnetometer_z'],\n", + " dtype='object')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4f8d0d23-2bc1-4906-9eba-4acccd1680d0", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/jon/projects/emlearn-micropython/paper/plotting.py:27: FutureWarning:\n", + "\n", + "A grouping was used that is not in the columns of the DataFrame and so was excluded from the result. This grouping will be included in a future version of pandas. Add the grouping as a column of the DataFrame to silence this warning.\n", + "\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
labelstart_timeend_time
0transient0 days 00:00:05.6400000 days 00:00:55.190000
1lying0 days 00:00:55.2000000 days 00:04:49.490000
2transient0 days 00:04:49.5000000 days 00:05:52.150000
3sitting0 days 00:05:52.1600000 days 00:09:35.600000
4transient0 days 00:09:35.6100000 days 00:09:55.870000
5standing0 days 00:09:55.8800000 days 00:14:11.630000
6transient0 days 00:14:11.6400000 days 00:15:08.070000
7ironing0 days 00:15:08.0800000 days 00:19:56.870000
8transient0 days 00:19:56.8800000 days 00:21:17.770000
9vacuum_cleaning0 days 00:21:17.7800000 days 00:24:44.600000
10transient0 days 00:24:44.6100000 days 00:28:54.160000
11ascending_stairs0 days 00:28:54.1700000 days 00:30:21.410000
12transient0 days 00:30:21.4200000 days 00:32:01.240000
13descending_stairs0 days 00:32:01.2500000 days 00:33:19.150000
14transient0 days 00:33:19.1600000 days 00:40:29.100000
15ascending_stairs0 days 00:40:29.1100000 days 00:41:55.270000
16descending_stairs0 days 00:41:55.2800000 days 00:43:09.490000
17transient0 days 00:43:09.5000000 days 00:46:20.710000
18walking0 days 00:46:20.7200000 days 00:51:46.040000
19transient0 days 00:51:46.0500000 days 00:53:49.690000
20Nordic_walking0 days 00:53:49.7000000 days 00:58:47.080000
21transient0 days 00:58:47.0900000 days 00:59:47.880000
22cycling0 days 00:59:47.8900000 days 01:03:58.960000
23transient0 days 01:03:58.9700000 days 01:05:15.430000
24running0 days 01:05:15.4400000 days 01:06:47.810000
25transient0 days 01:06:47.8200000 days 01:08:33.060000
26rope_jumping0 days 01:08:33.0700000 days 01:10:45.680000
27transient0 days 01:10:45.6900000 days 01:14:35.630000
\n", + "
" + ], + "text/plain": [ + " label start_time end_time\n", + "0 transient 0 days 00:00:05.640000 0 days 00:00:55.190000\n", + "1 lying 0 days 00:00:55.200000 0 days 00:04:49.490000\n", + "2 transient 0 days 00:04:49.500000 0 days 00:05:52.150000\n", + "3 sitting 0 days 00:05:52.160000 0 days 00:09:35.600000\n", + "4 transient 0 days 00:09:35.610000 0 days 00:09:55.870000\n", + "5 standing 0 days 00:09:55.880000 0 days 00:14:11.630000\n", + "6 transient 0 days 00:14:11.640000 0 days 00:15:08.070000\n", + "7 ironing 0 days 00:15:08.080000 0 days 00:19:56.870000\n", + "8 transient 0 days 00:19:56.880000 0 days 00:21:17.770000\n", + "9 vacuum_cleaning 0 days 00:21:17.780000 0 days 00:24:44.600000\n", + "10 transient 0 days 00:24:44.610000 0 days 00:28:54.160000\n", + "11 ascending_stairs 0 days 00:28:54.170000 0 days 00:30:21.410000\n", + "12 transient 0 days 00:30:21.420000 0 days 00:32:01.240000\n", + "13 descending_stairs 0 days 00:32:01.250000 0 days 00:33:19.150000\n", + "14 transient 0 days 00:33:19.160000 0 days 00:40:29.100000\n", + "15 ascending_stairs 0 days 00:40:29.110000 0 days 00:41:55.270000\n", + "16 descending_stairs 0 days 00:41:55.280000 0 days 00:43:09.490000\n", + "17 transient 0 days 00:43:09.500000 0 days 00:46:20.710000\n", + "18 walking 0 days 00:46:20.720000 0 days 00:51:46.040000\n", + "19 transient 0 days 00:51:46.050000 0 days 00:53:49.690000\n", + "20 Nordic_walking 0 days 00:53:49.700000 0 days 00:58:47.080000\n", + "21 transient 0 days 00:58:47.090000 0 days 00:59:47.880000\n", + "22 cycling 0 days 00:59:47.890000 0 days 01:03:58.960000\n", + "23 transient 0 days 01:03:58.970000 0 days 01:05:15.430000\n", + "24 running 0 days 01:05:15.440000 0 days 01:06:47.810000\n", + "25 transient 0 days 01:06:47.820000 0 days 01:08:33.060000\n", + "26 rope_jumping 0 days 01:08:33.070000 0 days 01:10:45.680000\n", + "27 transient 0 days 01:10:45.690000 0 days 01:14:35.630000" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "one = data.loc['subject102']\n", + "\n", + "sensor_columns = [\n", + " 'hand_acceleration_6g_x',\n", + " 'hand_acceleration_6g_y',\n", + " 'hand_acceleration_6g_z',\n", + "]\n", + "\n", + "labeled_events = find_runs(one.activity)\n", + "labeled_events" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "0a3eb1f2-6ab5-47af-a18d-7e3a30e0df01", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/jon/projects/emlearn-micropython/paper/plotting.py:27: FutureWarning:\n", + "\n", + "A grouping was used that is not in the columns of the DataFrame and so was excluded from the result. This grouping will be included in a future version of pandas. Add the grouping as a column of the DataFrame to silence this warning.\n", + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "missing values 0.0033332962967078145 %\n", + "0 days 00:00:02.560000\n", + "run-cmd micropython /home/jon/projects/emlearn-micropython/paper/example.py input.npy output.npy\n", + "[0.67233639 0.71646764 0.6204899 0.69873676 0.5107016 ]\n", + "Exported model.csv\n", + "0 days 00:00:02.560000\n", + "run-cmd micropython /home/jon/projects/emlearn-micropython/paper/example.py input.npy output.npy model.csv\n", + "test scores 0.7964886561273107 0.756707790245025\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Nordic_walkingcyclingwalkingtransientactivity
time
0 days 00:48:00NaNNaNNaNNaNwalking
0 days 00:48:02.560000NaNNaNNaNNaNwalking
0 days 00:48:05.120000NaNNaNNaNNaNwalking
0 days 00:48:07.680000NaNNaNNaNNaNwalking
0 days 00:48:10.2400000.00.00.90.0walking
\n", + "
" + ], + "text/plain": [ + " Nordic_walking cycling walking transient activity\n", + "time \n", + "0 days 00:48:00 NaN NaN NaN NaN walking\n", + "0 days 00:48:02.560000 NaN NaN NaN NaN walking\n", + "0 days 00:48:05.120000 NaN NaN NaN NaN walking\n", + "0 days 00:48:07.680000 NaN NaN NaN NaN walking\n", + "0 days 00:48:10.240000 0.0 0.0 0.9 0.0 walking" + ] + }, + "execution_count": 67, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "import emlearn\n", + "\n", + "from processing import process_data, make_label_track, convert_to_raw\n", + "\n", + "from sklearn.model_selection import cross_validate\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.dummy import DummyClassifier\n", + "\n", + "def train_and_run(data : pandas.DataFrame, label_column='activity'):\n", + " \n", + " events = find_runs(data[label_column])\n", + " sensor_data = convert_to_raw(data[sensor_columns])\n", + " features = process_data(sensor_data)\n", + " \n", + " q = features.quantile([0.01, 0.50, 0.99])\n", + " #print(q)\n", + " \n", + " labels = make_label_track(features.index, events)\n", + " combined = pandas.merge(features, sensor_data, left_index=True, right_index=True)\n", + " combined['activity'] = labels\n", + "\n", + " \n", + " estimator = RandomForestClassifier(n_estimators=10, min_samples_leaf=10)\n", + "\n", + " feature_columns = features.columns\n", + " X = combined[feature_columns]\n", + " Y = combined[label_column]\n", + "\n", + " # run cross-validation for basic indicator of fit\n", + " res = cross_validate(estimator, X=X, y=Y, scoring='f1_macro', return_estimator=True)\n", + " print(res['test_score'])\n", + "\n", + " # retrain on entire dataset. XXX: this is data leakage.\n", + " # Not acceptable in real model, but here we are just looking to illustrate\n", + " estimator.fit(X, Y)\n", + "\n", + " # Convert model with emlearn\n", + " model_path = 'model.csv'\n", + " converted = emlearn.convert(estimator)\n", + " converted.save(file=model_path, format='csv')\n", + " print('Exported', model_path)\n", + "\n", + " from sklearn.metrics import f1_score\n", + " def f1_macro(y_true, y_pred):\n", + " return f1_score(y_true, y_pred, average='macro')\n", + " class_names = estimator.classes_\n", + " ref_preds = pandas.DataFrame(estimator.predict_proba(X), index=X.index, columns=estimator.classes_)\n", + " ref_score = f1_macro(Y, class_names[numpy.argmax(ref_preds, axis=1)])\n", + "\n", + " #print(estimator.feature_names_in_)\n", + " \n", + " # Run model predictions with emlearn\n", + " out = process_data(sensor_data, model_path=model_path, classes=list(estimator.classes_))\n", + " preds = out[estimator.classes_]\n", + " c_score = f1_macro(Y, class_names[numpy.argmax(preds, axis=1)])\n", + " #print(preds.head())\n", + "\n", + " print('test scores', ref_score, c_score)\n", + " \n", + " rolling_window = 5\n", + " smooth = preds[class_columns].rolling(rolling_window).median()\n", + " smooth['activity'] = make_label_track(smooth.index, events)\n", + "\n", + " return combined, smooth, estimator\n", + "\n", + "class_columns = ['Nordic_walking', 'cycling', 'walking', \n", + " #'rope_jumping',\n", + " #'running',\n", + " 'transient'\n", + " ]\n", + "sub = one.loc[pandas.Timedelta(minutes=48):pandas.Timedelta(minutes=63)]\n", + "combined, smooth, estimator = train_and_run(sub)\n", + "smooth.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "4c38d4ec-a4eb-4066-8c4b-21204bb84bfb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAGwCAYAAAAQdOnRAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjUsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvWftoOwAAAAlwSFlzAAAPYQAAD2EBqD+naQAATUhJREFUeJzt3X1YVGX+P/D3YYQRFGZAwRl1GhIf0FAbM82xTNOSnmM3WylDehDb0kyzTa5UwjRckTZqW5UeeGhb3db8UotFEqltZILGWJmLSTIIzmgpzki0Aw7n94c/ZnfiWWaYOeP7dV3nys8599zzObdeM5/uc+4zgiiKIoiIiIjI6/l5OgEiIiIi6hoWbkREREQSwcKNiIiISCJYuBERERFJBAs3IiIiIolg4UZEREQkESzciIiIiCSij6cTINdpbm7GyZMnERwcDEEQPJ0OERERdYEoijh//jwGDx4MP7+O59RYuPmQkydPQqPReDoNIiIiugQnTpzA0KFDO2zDws2HBAcHA7j4Fx8SEuLhbIiIiKgrrFYrNBqN43u8IyzcfEjL5dE7NuyETB7Y5dcdTE9wV0pERETURV25zYmLE4iIiIgkgoUbERERkUSwcCMiIiKSCBZuPSCKIpKSkhAWFgZBEFBeXu4UGwwGT6dIREREPoSFWw8UFhYiJycHBQUFMJlMqKmpcYpjYmIgCALy8/O71N/OnTsxefJkBAYGIjQ0FPfcc49b8yciIiJp4arSHqisrIRarYZerwcAGI1Gp7g73nvvPSxYsAAvvvgibrrpJly4cAHffvutq1MmIiIiCWPhdokSExORm5sL4OLyXa1WC6PR6BS3iIuLAwBotVpUVVW16uvChQtYsmQJ0tPT8cgjjzj2jxkzpsMcbDYbbDabI7ZarZd8PkREROT9eKn0EmVmZmLNmjUYOnQoTCYT9u7d6xSXlZWhrKwMAJCdne3Y15avvvoKtbW18PPzg06ng1qtxq233trpjFtaWhoUCoVj468mEBER+TYWbpdIoVAgODgYMpkMKpUKWq3WKQ4PD0d4eDgAQKlUOva15YcffgAAPP/881i5ciUKCgoQGhqK6dOn4+zZs+3mkJycDIvF4thOnDjh+hMlIiIir8HCzQs0NzcDAJ577jn89re/xTXXXIPs7GwIgoB//OMf7b5OLpcjJCTEaSMiIiLfxcLNC6jVagDO97TJ5XIMGzYM1dXVnkqLiIiIvAwLNzfz9/eH3W7vsM0111wDuVyOiooKx76mpiZUVVU5LXIgIiKiyxsLNzeLjIxEcXExzGYz6urqAAClpaWIjo5GbW0tACAkJASPPfYYUlJSsGvXLlRUVOD3v/89AGDOnDkey52IiIi8Cws3N8vIyEBRURE0Gg10Oh0AoKGhARUVFWhqanK0S09Px9y5c/Hggw/i2muvhdFoxKefforQ0FBPpU5EREReRhBFUfR0EuQaVqsVCoUC4xdvhkwe2OXXHUxPcGNWRERE1JGW72+LxdLpQkPOuBERERFJBH85wQd9tjaejwYhIiLyQZxxIyIiIpIIFm5EREREEsHCjYiIiEgieI+bD5q2citXlRIREfkgzrgRERERSQQLNyIiIiKJYOHWAVEUkZSUhLCwMAiCgPLycqfYYDB4OkUiIiK6jLBw60BhYSFycnJQUFAAk8mEmpoapzgmJgaCICA/P7/TvtatWwe9Xo+goCAolcpWx8+cOYPY2FgMHjwYcrkcGo0GixYtgtVqdf2JERERkSSxcOtAZWUl1Go19Ho9VCoVjEajU9ynT9fXdjQ2NmLOnDmOH4//NT8/P9x999344IMPcPToUeTk5OCTTz7BY4895qrTISIiIonjqtJ2JCYmIjc3FwAgCAK0Wi2MRqNT3CIuLg4AoNVqUVVV1WZ/qampAICcnJw2j4eGhjoVdVqtFo8//jjS09N7eipERETkI1i4tSMzMxNRUVHIyspCWVkZbDYb8vLyHLFMJgMAREREIDs7G7GxsY59rnDy5Ens2LEDN954Y7ttbDYbbDabI+ZlVSIiIt/GS6XtUCgUCA4Ohkwmg0qlglardYrDw8MRHh4OAFAqlY59PRUfH4+goCAMGTIEISEheOONN9ptm5aWBoVC4dg0Gk2P35+IiIi8Fws3L/OnP/0JX331Fd5//31UVlZi2bJl7bZNTk6GxWJxbCdOnOjFTImIiKi38VKpl1GpVFCpVIiOjkZYWBhuuOEGrFq1Cmq1ulVbuVwOuVzugSyJiIjIEzjj1kP+/v6w2+1u6bu5uRkAnO5jIyIiossXC7ceioyMRHFxMcxmM+rq6gAApaWliI6ORm1traNddXU1DAYDqqurYbfbYTAYYDAYUF9fDwD48MMPkZ2djW+//RZVVVXYuXMnHnvsMUydOhWRkZGeODUiIiLyMizceigjIwNFRUXQaDTQ6XQAgIaGBlRUVKCpqcnRbvXq1dDpdEhJSUF9fT10Oh10Oh0OHDgAAAgMDMTrr7+O66+/HqNHj8bSpUtx1113oaCgwCPnRURERN5HEEVR9HQS5BpWqxUKhQLjF2+GTB7Y5dcdTE9wY1ZERETUkZbvb4vFgpCQkA7bcsaNiIiISCK4qtQHfbY2vtOKnYiIiKSHM25EREREEsHCjYiIiEgiWLgRERERSQTvcfNB01Zu7daqUoArS4mIiKSAM25EREREEsHCjYiIiEgiWLgRERERSQQLtw6IooikpCSEhYVBEASUl5c7xQaDwdMpEhER0WWEhVsHCgsLkZOTg4KCAphMJtTU1DjFMTExEAQB+fn5nfa1bt066PV6BAUFQalUtjp+6NAhxMfHQ6PRIDAwEKNHj0ZmZqbrT4qIiIgki6tKO1BZWQm1Wg29Xg8AMBqNTnF3NDY2Ys6cOZgyZQrefPPNVscPHjyIiIgI/PWvf4VGo8EXX3yBpKQkyGQyLFq0qMfnQkRERNLHwq0diYmJyM3NBQAIggCtVguj0egUt4iLiwMAaLVaVFVVtdlfamoqACAnJ6fN4w8//LBTPGzYMOzbtw87duxot3Cz2Wyw2WyO2Gq1dn5iREREJFks3NqRmZmJqKgoZGVloaysDDabDXl5eY5YJpMBACIiIpCdnY3Y2FjHPlexWCwICwtr93haWpqjICQiIiLfx8KtHQqFAsHBwZDJZFCpVADQKm6hVCpb7eupL774An//+9+xc+fOdtskJydj2bJljthqtUKj0bg0DyIiIvIeLNy80Lfffou7774bKSkpuOWWW9ptJ5fLIZfLezEzIiIi8iSuKvUy3333HWbOnImkpCSsXLnS0+kQERGRF2Hh1kP+/v6w2+0u6evw4cOYMWMG5s+fj3Xr1rmkTyIiIvIdLNx6KDIyEsXFxTCbzairqwMAlJaWIjo6GrW1tY521dXVMBgMqK6uht1uh8FggMFgQH19PYCLl0dnzJiBW265BcuWLYPZbIbZbMaPP/7okfMiIiIi78PCrYcyMjJQVFQEjUYDnU4HAGhoaEBFRQWampoc7VavXg2dToeUlBTU19dDp9NBp9PhwIEDAIDt27fjxx9/xF//+leo1WrHdu2113rkvIiIiMj7CKIoip5OglzDarVCoVBg/OLNkMkDu/Xag+kJbsqKiIiIOtLy/W2xWBASEtJhW864EREREUkEHwfigz5bG99pxU5ERETSwxk3IiIiIolg4UZEREQkESzciIiIiCSC97j5oGkrt3Z7VWlXcOUpERGRZ3HGjYiIiEgiWLgRERERSQQLtx4QRRFJSUkICwuDIAgoLy93ig0Gg6dTJCIiIh/Cwq0HCgsLkZOTg4KCAphMJtTU1DjFMTExEAQB+fn5nfa1bt066PV6BAUFQalUuj13IiIikh4uTuiByspKqNVq6PV6AIDRaHSKu6OxsRFz5szBlClT8Oabb7o6VSIiIvIBLNwuUWJiInJzcwEAgiBAq9XCaDQ6xS3i4uIAAFqtFlVVVW32l5qaCgDIyclxX9JEREQkaSzcLlFmZiaioqKQlZWFsrIy2Gw25OXlOWKZTAYAiIiIQHZ2NmJjYx37XMVms8Fmszliq9Xq0v6JiIjIu7Bwu0QKhQLBwcGQyWRQqVQA0CpuoVQqW+1zhbS0NMdMHREREfk+Lk6QsOTkZFgsFsd24sQJT6dEREREbsQZNwmTy+WQy+WeToOIiIh6CWfc3Mzf3x92u93TaRAREZEPYOHmZpGRkSguLobZbEZdXR0AoLS0FNHR0aitrXW0q66uhsFgQHV1Nex2OwwGAwwGA+rr6z2VOhEREXkZFm5ulpGRgaKiImg0Guh0OgBAQ0MDKioq0NTU5Gi3evVq6HQ6pKSkoL6+HjqdDjqdDgcOHPBU6kRERORlBFEURU8nQa5htVqhUCgwfvFmyOSBLu//YHqCy/skIiK63LV8f1ssFoSEhHTYljNuRERERBLBVaU+6LO18Z1W7ERERCQ9nHEjIiIikggWbkREREQSwcKNiIiISCJ4j5sPmrZyq1tWlXYFV54SERG5D2fciIiIiCSChRsRERGRRLBwIyIiIpIIFm49IIoikpKSEBYWBkEQUF5e7hQbDAZPp0hEREQ+hIVbDxQWFiInJwcFBQUwmUyoqalximNiYiAIAvLz8zvt6+jRo7j77rsxcOBAhISE4Prrr8fu3bvdfxJEREQkGSzceqCyshJqtRp6vR4qlQpGo9Ep7tOn64t277jjDly4cAGffvopDh48iPHjx+OOO+6A2Wx24xkQERGRlPBxIJcoMTERubm5AABBEKDVamE0Gp3iFnFxcQAArVaLqqqqVn399NNP+P777/Hmm29i3LhxAID169fjL3/5C7799luoVCo3nw0RERFJAQu3S5SZmYmoqChkZWWhrKwMNpsNeXl5jlgmkwEAIiIikJ2djdjYWMe+XxswYABGjRqFvLw8TJgwAXK5HFu2bEFERASuueaadnOw2Wyw2WyO2Gq1uvYkiYiIyKuwcLtECoUCwcHBkMlkjhmxX8ctlEplh7NmgiDgk08+wT333IPg4GD4+fkhIiIChYWFCA0Nbfd1aWlpSE1Ndc0JERERkdfjPW5eQBRFPPHEE4iIiMC//vUvlJaW4p577sGdd94Jk8nU7uuSk5NhsVgc24kTJ3oxayIiIuptnHHzAp9++ikKCgpQV1eHkJAQAMBf/vIXFBUVITc3FytWrGjzdXK5HHK5vDdTJSIiIg/ijJub+fv7w263d9imoaEBAODn5/zX4efnh+bmZrflRkRERNLCws3NIiMjUVxcDLPZjLq6OgBAaWkpoqOjUVtbCwCYMmUKQkNDMX/+fBw6dAhHjx7FM888g+PHj+P222/3ZPpERETkRVi4uVlGRgaKioqg0Wig0+kAXJxhq6ioQFNTEwBg4MCBKCwsRH19PW666SZMnDgRn3/+Od5//32MHz/ek+kTERGRFxFEURQ9nQS5htVqhUKhwPjFmyGTB3okh4PpCR55XyIiIqlq+f62WCyOe93bwxk3IiIiIongqlIf9Nna+E4rdiIiIpIezrgRERERSQQLNyIiIiKJYOFGREREJBG8x80HTVu51WOrStvClaZERESuwRk3IiIiIolg4UZEREQkESzcekAURSQlJSEsLAyCIKC8vNwpNhgMnk6RiIiIfAgLtx4oLCxETk4OCgoKYDKZUFNT4xTHxMRAEATk5+d32tdXX32Fm2++GUqlEgMGDEBSUhLq6+vdfxJEREQkGSzceqCyshJqtRp6vR4qlQpGo9Ep7tOna2s/Tp48iVmzZmH48OHYv38/CgsLcfjwYSQmJrr3BIiIiEhSuKr0EiUmJiI3NxcAIAgCtFotjEajU9wiLi4OAKDValFVVdWqr4KCAvj7++O1116Dn9/FWnrz5s0YN24cjh07huHDh7v5bIiIiEgKWLhdoszMTERFRSErKwtlZWWw2WzIy8tzxDKZDAAQERGB7OxsxMbGOvb9ms1mQ0BAgKNoA4DAwIuP8/j888/bLdxsNhtsNpsjtlqtrjo9IiIi8kK8VHqJFAoFgoODIZPJoFKpoNVqneLw8HCEh4cDAJRKpWNfW2666SaYzWakp6ejsbERdXV1WLFiBQDAZDK1m0NaWhoUCoVj02g0rj9RIiIi8hos3LzAVVddhdzcXGRkZCAoKAgqlQpXXnklBg0a5DQL92vJycmwWCyO7cSJE72YNREREfU2Xir1Evfffz/uv/9+nDp1Cv369YMgCHjppZcwbNiwdl8jl8shl8t7MUsiIiLyJBZububv7w+73d7l9oMGDQIAvPXWW+jbty9uvvlmd6VGREREEsNLpW4WGRmJ4uJimM1m1NXVAQBKS0sRHR2N2tpaR7s///nP+Oqrr3D06FG89tprWLRoEdLS0qBUKj2UOREREXkbFm5ulpGRgaKiImg0Guh0OgBAQ0MDKioq0NTU5GhXWlqKm2++GWPHjkVWVha2bNmCJ5980lNpExERkRcSRFEUPZ0EuYbVaoVCocD4xZshkwd6Oh2Hg+kJnk6BiIjIa7V8f1ssFoSEhHTYljNuRERERBLBxQk+6LO18Z1W7ERERCQ9nHEjIiIikggWbkREREQSwcKNiIiISCJ4j5sPmrZyq1etKv01rjIlIiK6NJxxIyIiIpIIFm5EREREEsHCjYiIiEgiWLh1QBRFJCUlISwsDIIgoLy83Ck2GAyeTpGIiIguIyzcOlBYWIicnBwUFBTAZDKhpqbGKY6JiYEgCMjPz++0r3Xr1kGv1yMoKKjdH44XBKHVtm3bNteeFBEREUkWV5V2oLKyEmq1Gnq9HgBgNBqd4u5obGzEnDlzMGXKFLz55pvttsvOzkZsbKwjbq/IIyIiossPC7d2JCYmIjc3F8DFmTCtVguj0egUt4iLiwMAaLVaVFVVtdlfamoqACAnJ6fD91UqlVCpVD3MnoiIiHwRL5W2IzMzE2vWrMHQoUNhMpmwd+9ep7isrAxlZWUALs6StezrqSeeeAIDBw7EpEmT8NZbb0EUxXbb2mw2WK1Wp42IiIh8F2fc2qFQKBAcHAyZTOaYAft13MJVs2Rr1qzBTTfdhKCgIOzatQuPP/446uvr8eSTT7bZPi0tzTGTR0RERL6PhZsXWbVqlePPOp0OP//8M9LT09st3JKTk7Fs2TJHbLVaodFo3J4nEREReQYvlXqxyZMno6amBjabrc3jcrkcISEhThsRERH5LhZuPeTv7w+73e6Wvg0GA0JDQyGXy93SPxEREUkLL5X2UGRkJIqLizF16lTI5XKEhoaitLQUCQkJKC4uxpAhQwAA1dXVOHv2LKqrq2G32x0P7x0+fDj69++Pf/7znzh16hSuu+469O3bF0VFRXjxxRexfPlyD54dEREReRMWbj2UkZGBZcuW4fXXX8eQIUNQVVWFhoYGVFRUoKmpydFu9erVjseLABfvYQOA3bt3Y/r06fD398drr72GpUuXQhRFDB8+HC+99BIWLFjQ6+dERERE3kkQO3reBEmK1WqFQqHA+MWbIZMHejqddh1MT/B0CkRERF6j5fvbYrF0er8673EjIiIikgheKvVBn62N5wpTIiIiH8QZNyIiIiKJYOFGREREJBEs3IiIiIgkgve4+aBpK7dyVSkREZEP4owbERERkUSwcCMiIiKSCBZuRERERBLBwq2HRFFEUlISwsLCIAgCysvLneKW3yQlIiIi6ikWbj1UWFiInJwcFBQUwGQyoaamximOiYmBIAjIz8/vsJ89e/ZAEIQ2t7Kyst45GSIiIvJqXFXaQ5WVlVCr1dDr9QAAo9HoFHeVXq+HyWRy2rdq1SoUFxdj4sSJLsuXiIiIpIuFWw8kJiYiNzcXACAIArRaLYxGo1PcIi4uDgCg1WpRVVXVqq+AgACoVCpH3NTUhPfffx+LFy+GIAhtvr/NZoPNZnPEVqu1x+dERERE3ouFWw9kZmYiKioKWVlZKCsrg81mQ15eniOWyWQAgIiICGRnZyM2NtaxrzMffPABzpw5g4ceeqjdNmlpaUhNTXXJuRAREZH3Y+HWAwqFAsHBwZDJZI7Zsl/HLZRKZat9HXnzzTcxe/ZsDB06tN02ycnJWLZsmSO2Wq3QaDTdPAsiIiKSChZuXqimpgYff/wx3n333Q7byeVyyOXyXsqKiIiIPI2rSr1QdnY2BgwYgLvuusvTqRAREZEXYeHWC/z9/WG327vUVhRFZGdnIyEhAf7+/m7OjIiIiKSEhVsviIyMRHFxMcxmM+rq6gAApaWliI6ORm1trVPbTz/9FMePH8ejjz7qiVSJiIjIi7Fw6wUZGRkoKiqCRqOBTqcDADQ0NKCiogJNTU1Obd98803o9XpER0d7IlUiIiLyYoIoiqKnkyDXsFqtUCgUGL94M2TyQE+n066D6QmeToGIiMhrtHx/WywWhISEdNiWq0p90Gdr4zv9iyciIiLpuaRLpZWVlVi5ciXi4+Nx+vRpAMBHH32Ew4cPuzQ5IiIiIvqvbhdue/fuxdixY7F//37s2LED9fX1AIBDhw4hJSXF5QkSERER0UXdLtxWrFiBtWvXoqioCAEBAY79N910E7788kuXJkdERERE/9Xte9y++eYb/O1vf2u1PyIiAj/99JNLkqKembZyq1cvTmgLFywQERF1rtszbkqlEiaTqdX+8vJyDBkyxCVJEREREVFr3S7c5s6di2effRZmsxmCIKC5uRklJSVYvnw5EhI4a0JERETkLt0u3F588UVER0dDo9Ggvr4eY8aMwbRp06DX67Fy5Up35EhERERE6GbhJooizGYzXnnlFfzwww8oKCjAX//6V/z73//G22+/DZlM5q48vZYoikhKSkJYWBgEQUB5eblTbDAYPJ0iERER+YhuF27Dhw9HTU0NNBoNbrvtNtx3330YMWKEu/LzeoWFhcjJyUFBQQFMJhNqamqc4piYGAiCgPz8/E77uuuuu3DFFVegb9++UKvVePDBB3Hy5En3nwQRERFJQrcKNz8/P4wYMQJnzpxxVz6SU1lZCbVaDb1eD5VKBaPR6BT36dP1hbszZszAu+++i4qKCrz33nuorKzEvffe68bsiYiISEq6/Vul//znP7FhwwZs2rQJMTEx7spLEhITE5Gbm+uItVotjEajUwyg1b6qqqou9f/BBx/gnnvugc1mg7+/f6ftpfJbpW3h40CIiOhy5dbfKk1ISEBDQwPGjx+PgIAABAY6Fwhnz57tbpeSlZmZiaioKGRlZaGsrAw2mw15eXmOuOWev4iICGRnZyM2NrbL9wGePXsW77zzDvR6fbtFm81mg81mc8RWq7XnJ0VEREReq9uF28svv+yGNKRJoVAgODgYMpkMKpUKAFrFLZRKZat9bXn22Wfx5z//GQ0NDbjuuutQUFDQbtu0tDSkpqb27CSIiIhIMrpduM2fP98dedD/98wzz+CRRx6B0WhEamoqEhISUFBQAEEQWrVNTk7GsmXLHLHVaoVGo+nNdImIiKgXdbtwq66u7vD4FVdcccnJEDBw4EAMHDgQI0eOxOjRo6HRaPDll19iypQprdrK5XLI5XIPZElERESe0O3CLTIyss3ZnxZ2u71HCfkif3//SxqX5uZmAHC6j42IiIguX90u3MrLy53ipqYmlJeX46WXXsK6detclpgviYyMRHFxMaZOnQq5XI7Q0FCUlpYiISEBxcXFGDJkCPbv34+ysjJcf/31CA0NRWVlJVatWoWoqKg2Z9uIiIjo8tPtwm38+PGt9k2cOBGDBw9Geno6fvOb37gkMV+SkZGBZcuW4fXXX8eQIUNQVVWFhoYGVFRUoKmpCQAQFBSEHTt2ICUlBT///DPUajViY2OxcuVKXg4lIiIiAJfwHLf2HDt2DOPHj8fPP//siu7oEvA5bkRERNLj1ue4/fpZYaIowmQy4fnnn7+sf/qKiIiIyN26XbgplcpWixNEUYRGo8G2bdtclhhdus/WxndasRMREZH0dLtw2717t1Ps5+eH8PBwDB8+vFu/y0lERERE3dPtSksQBOj1+lZF2oULF/DZZ59h2rRpLkuOiIiIiP7Lr7svmDFjRpu/R2qxWDBjxgyXJEVERERErXV7xk0UxTYfwHvmzBn069fPJUlRz0xbuVVyq0o7w1WnRERE3SjcWp7PJggCEhMTnZ4tZrfb8fXXX0Ov17s+QyIiIiIC0I3CTaFQALg44xYcHIzAwP/O6AQEBOC6667DggULXJ8hEREREQHoRuGWnZ0N4OLPNy1fvpyXRYmIiIh6WbcXJ6SkpFw2RZsoikhKSkJYWBgEQUB5eblTbDAYPJ0iERERXUa6XbgBwPbt23Hffffhuuuuw4QJE5w2X1JYWIicnBwUFBTAZDKhpqbGKY6JiYEgCMjPz++0r3Xr1kGv1yMoKAhKpbLNNk8++SSuueYayOVyXH311S49FyIiIpK+bhdur7zyCh566CEMGjQI5eXlmDRpEgYMGIAffvgBt956qzty9JjKykqo1Wro9XqoVCoYjUanuDsPHG5sbMScOXPw+9//vsN2Dz/8MH73u9/1NHUiIiLyQd1+HMhf/vIXZGVlIT4+Hjk5OfjDH/6AYcOGYfXq1W0+302qEhMTkZubC+DiSlqtVguj0egUt4iLiwMAaLVaVFVVtdlfamoqACAnJ6fd93zllVcAAD/++CO+/vrrTnO02Wyw2WyO+Ne/I0tERES+pdszbtXV1Y7HfgQGBuL8+fMAgAcffBBbt251bXYelJmZiTVr1mDo0KEwmUzYu3evU1xWVoaysjIAFxdutOzrTWlpaVAoFI5No9H06vsTERFR7+p24aZSqRwza1dccQW+/PJLAMDx48chiqJrs/MghUKB4OBgyGQyqFQqaLVapzg8PBzh4eEAAKVS6djXm5KTk2GxWBzbiRMnevX9iYiIqHd1+1LpTTfdhA8++AA6nQ4PPfQQli5diu3bt+PAgQOOh/RS75DL5U4PQiYiIiLf1u3CLSsrC83NzQCAJ554AgMGDMAXX3yBu+66CwsXLnR5gkRERER0UbcLNz8/P/j5/fcK69y5czF37lyXJiUl/v7+sNvtnk6DiIiILgOX9By3f/3rX5g3bx6mTJmC2tpaAMDbb7+Nzz//3KXJSUFkZCSKi4thNptRV1cHACgtLUV0dLRjbICLizoMBgOqq6tht9thMBhgMBhQX1/vaHPs2DEYDAaYzWb88ssvjjaNjY29fl5ERETkfbpduL333nuYPXs2AgMDUV5e7ngchcViwYsvvujyBL1dRkYGioqKoNFooNPpAAANDQ2oqKhAU1OTo93q1auh0+mQkpKC+vp66HQ66HQ6HDhwwNHm0UcfhU6nw5YtW3D06FFHm5MnT/b6eREREZH3EcRuLgXV6XRYunQpEhISEBwcjEOHDmHYsGEoLy/HrbfeCrPZ7K5cqRNWqxUKhQLjF2+GTB7o6XRc6mB6gqdTICIicouW72+LxYKQkJAO23b7HreKigpMmzat1X6FQoFz5851tztyg8/Wxnf6F09ERETSc0nPcTt27Fir/Z9//jmGDRvmkqSIiIiIqLVuF24LFizAkiVLsH//fgiCgJMnT+Kdd97B8uXLO/0dTiIiIiK6dF26VPr1118jJiYGfn5+SE5ORnNzM2bOnImGhgZMmzYNcrkcy5cvx+LFi92dLxEREdFlq0uLE2QyGUwmEyIiIjBs2DCUlZUhODgYx44dQ319PcaMGYP+/fv3Rr7Uge7c3EhERETeweWLE5RKJY4fP46IiAhUVVWhubkZAQEBGDNmjEsSJteatnKrz60q7QmuSCUiIl/RpcLtt7/9LW688Uao1WoIgoCJEydCJpO12faHH35waYJEREREdFGXCresrCz85je/wbFjx/Dkk09iwYIFCA4OdnduRERERPQ/uvwct9jYWADAwYMHsWTJksuicBNFEQsXLsT27dtRV1eHr776Cps2bXLE5eXluPrqqz2dJhEREV0muv04kOzs7MuiaAOAwsJC5OTkoKCgACaTCTU1NU5xTEwMBEFAfn5+p32tW7cOer0eQUFBUCqVbbaprq7G7bffjqCgIEREROCZZ57BhQsXXHtSREREJFnd/uWEy0llZSXUajX0ej0AwGg0OsXd0djYiDlz5mDKlCl48803Wx232+24/fbboVKp8MUXX8BkMiEhIQH+/v6X5W/AEhERUWss3NqRmJiI3NxcAIAgCNBqtTAajU5xi7i4OACAVqtFVVVVm/2lpqYCAHJycto8vmvXLnz33Xf45JNPMGjQIFx99dV44YUX8Oyzz+L5559HQECAi86MiIiIpKrbl0ovF5mZmVizZg2GDh0Kk8mEvXv3OsVlZWUoKysDcPHyccu+S7Vv3z6MHTsWgwYNcuybPXs2rFYrDh8+3OZrbDYbrFar00ZERES+izNu7VAoFAgODoZMJoNKpQKAVnELpVLZal93mc1mp6INgCM2m81tviYtLc0xk0dERES+jzNuEpacnAyLxeLYTpw44emUiIiIyI044+YlVCoVSktLnfadOnXKcawtcrkccrnc7bkRERGRd+CMWw/5+/vDbrf3uJ8pU6bgm2++wenTpx37ioqKEBISwp8WIyIiIgAs3HosMjISxcXFMJvNqKurAwCUlpYiOjoatbW1jnbV1dUwGAyorq6G3W6HwWCAwWBAfX09AOCWW27BmDFj8OCDD+LQoUP4+OOPsXLlSjzxxBOcVSMiIiIALNx6LCMjA0VFRdBoNNDpdACAhoYGVFRUoKmpydFu9erV0Ol0SElJQX19PXQ6HXQ6HQ4cOAAAkMlkKCgogEwmw5QpUzBv3jwkJCRgzZo1HjkvIiIi8j6CKIqip5Mg17BarVAoFBi/eDNk8kBPp+M1DqYneDoFIiKidrV8f1ssFoSEhHTYljNuRERERBLBVaU+6LO18Z1W7ERERCQ9nHEjIiIikggWbkREREQSwcKNiIiISCJ4j5sPmrZyK1eVdhFXnBIRkZRwxo2IiIhIIli4EREREUkECzciIiIiiWDh1gFRFJGUlISwsDAIgoDy8nKn2GAweDpFIiIiuoywcOtAYWEhcnJyUFBQAJPJhJqaGqc4JiYGgiAgPz+/077WrVsHvV6PoKAgKJXKDtueOXMGQ4cOhSAIOHfunEvOhYiIiKSPhVsHKisroVarodfroVKpYDQaneI+fbq+KLexsRFz5szB73//+07bPvLIIxg3blxPUiciIiIfxMeBtCMxMRG5ubkAAEEQoNVqYTQaneIWcXFxAACtVouqqqo2+0tNTQUA5OTkdPi+mzZtwrlz57B69Wp89NFHHba12Wyw2WyO2Gq1dtieiIiIpI2FWzsyMzMRFRWFrKwslJWVwWazIS8vzxHLZDIAQEREBLKzsxEbG+vYd6m+++47rFmzBvv378cPP/zQafu0tDRHQUhERES+j5dK26FQKBAcHAyZTAaVSgWtVusUh4eHIzw8HACgVCod+y6VzWZDfHw80tPTccUVV3TpNcnJybBYLI7txIkTl/z+RERE5P044+YlkpOTMXr0aMybN6/Lr5HL5ZDL5W7MioiIiLwJZ9y8xKeffop//OMf6NOnD/r06YOZM2cCAAYOHIiUlBQPZ0dERETegDNuPeTv7w+73d7jft577z388ssvjrisrAwPP/ww/vWvfyEqKqrH/RMREZH0sXDrocjISBQXF2Pq1KmQy+UIDQ1FaWkpEhISUFxcjCFDhgAAqqurcfbsWVRXV8Nutzse3jt8+HD079+/VXH2008/AQBGjx7d6XPfiIiI6PLAS6U9lJGRgaKiImg0Guh0OgBAQ0MDKioq0NTU5Gi3evVq6HQ6pKSkoL6+HjqdDjqdDgcOHPBU6kRERCQxgiiKoqeTINewWq1QKBQYv3gzZPJAT6cjCQfTEzydAhERXeZavr8tFgtCQkI6bMtLpT7os7Xxnf7FExERkfTwUikRERGRRLBwIyIiIpIIFm5EREREEsHCjYiIiEgiuDjBB01buZWrSiWGq1uJiKgrOONGREREJBEs3IiIiIgkgoVbD4miiKSkJISFhUEQBJSXlzvFLT9tRURERNRTLNx6qLCwEDk5OSgoKIDJZEJNTY1THBMTA0EQkJ+f32lfZ8+exQMPPICQkBAolUo88sgjqK+vd/9JEBERkSRwcUIPVVZWQq1WQ6/XAwCMRqNT3B0PPPAATCYTioqK0NTUhIceeghJSUn429/+5uq0iYiISIJYuPVAYmIicnNzAQCCIECr1cJoNDrFLeLi4gAAWq0WVVVVrfo6cuQICgsLUVZWhokTJwIAXn31Vdx2223YuHEjBg8e7OazISIiIm/Hwq0HMjMzERUVhaysLJSVlcFmsyEvL88Ry2QyAEBERASys7MRGxvr2Pdr+/btg1KpdBRtADBr1iz4+flh//79jsLvf9lsNthsNkdstVpdfIZERETkTVi49YBCoUBwcDBkMhlUKhUAtIpbKJXKVvv+l9lsRkREhNO+Pn36ICwsDGazuc3XpKWlITU1tYdnQURERFLBxQkSlpycDIvF4thOnDjh6ZSIiIjIjTjj5iVUKhVOnz7ttO/ChQs4e/ZsuzN1crkccrm8N9IjIiIiL8AZt17g7+8Pu93eYZspU6bg3LlzOHjwoGPfp59+iubmZkyePNndKRIREZEEsHDrBZGRkSguLobZbEZdXR0AoLS0FNHR0aitrQUAjB49GrGxsViwYAFKS0tRUlKCRYsWYe7cuVxRSkRERABYuPWKjIwMFBUVQaPRQKfTAQAaGhpQUVGBpqYmR7t33nkH0dHRmDlzJm677TZcf/31yMrK8lTaRERE5GUEURRFTydBrmG1WqFQKDB+8WbI5IGeToe64WB6gqdTICIiD2n5/rZYLAgJCemwLWfciIiIiCSCq0p90Gdr4zut2ImIiEh6OONGREREJBEs3IiIiIgkgoUbERERkUTwHjcfNG3lVq4qpQ5xFSsRkTRxxo2IiIhIIli4EREREUkECzciIiIiifDKwk0URSQlJSEsLAyCIKC8vNwpNhgMnk6RiIiIqNd5ZeFWWFiInJwcFBQUwGQyoaamximOiYmBIAjIz8/3dKpEREREvcYrV5VWVlZCrVZDr9cDAIxGo1MsJY2NjQgICPB0GkREROQDvG7GLTExEYsXL0Z1dTUEQUBkZGSrODIyEgAQFxfn2NcV77//PiZMmIC+ffti2LBhSE1NxYULFxzHBUHAG2+8gbi4OAQFBWHEiBH44IMPnPr49ttvceutt6J///4YNGgQHnzwQfz000+O49OnT8eiRYvw1FNPYeDAgZg9ezYA4IMPPsCIESPQt29fzJgxA7m5uRAEAefOncPPP/+MkJAQbN++3em98vPz0a9fP5w/f77N87HZbLBarU4bERER+S6vK9wyMzOxZs0aDB06FCaTCXv37nWKy8rKUFZWBgDIzs527OvMv/71LyQkJGDJkiX47rvvsGXLFuTk5GDdunVO7VJTU3Hffffh66+/xm233YYHHngAZ8+eBQCcO3cON910E3Q6HQ4cOIDCwkKcOnUK9913n1Mfubm5CAgIQElJCTZv3ozjx4/j3nvvxT333INDhw5h4cKFeO655xzt+/Xrh7lz5yI7O9upn+zsbNx7770IDg5u85zS0tKgUCgcm0aj6XyAiYiISLK8rnBTKBQIDg6GTCaDSqWCVqt1isPDwxEeHg4AUCqVjn2dSU1NxYoVKzB//nwMGzYMN998M1544QVs2bLFqV1iYiLi4+MxfPhwvPjii6ivr0dpaSkA4M9//jN0Oh1efPFFREdHQ6fT4a233sLu3btx9OhRRx8jRozAhg0bMGrUKIwaNQpbtmzBqFGjkJ6ejlGjRmHu3LlITEx0et9HH30UH3/8MUwmEwDg9OnT+PDDD/Hwww+3e07JycmwWCyO7cSJE10aYyIiIpImr7zHzR0OHTqEkpISpxk2u92O//znP2hoaEBQUBAAYNy4cY7j/fr1Q0hICE6fPu3oY/fu3ejfv3+r/isrKzFy5EgAwDXXXON0rKKiAtdee63TvkmTJrWKr7rqKuTm5mLFihX461//Cq1Wi2nTprV7TnK5HHK5vCunT0RERD7gsinc6uvrkZqait/85jetjvXt29fxZ39/f6djgiCgubnZ0cedd96JP/7xj636UKvVjj/369fvknJ89NFH8dprr2HFihXIzs7GQw89BEEQLqkvIiIi8j2SLdz8/f1ht9u73H7ChAmoqKjA8OHDL/k9J0yYgPfeew+RkZHo06frQzdq1Ch8+OGHTvvaui9v3rx5+MMf/oBXXnkF3333HebPn3/JuRIREZHv8bp73LoqMjISxcXFMJvNqKur67T96tWrkZeXh9TUVBw+fBhHjhzBtm3bsHLlyi6/5xNPPIGzZ88iPj4eZWVlqKysxMcff4yHHnqowyJy4cKF+Pe//41nn30WR48exbvvvoucnBwAcJpRCw0NxW9+8xs888wzuOWWWzB06NAu50ZERES+T7KFW0ZGBoqKiqDRaKDT6TptP3v2bBQUFGDXrl249tprcd111+FPf/oTtFptl99z8ODBKCkpgd1uxy233IKxY8fiqaeeglKphJ9f+0N55ZVXYvv27dixYwfGjRuHTZs2OVaV/voetUceeQSNjY0dLkogIiKiy5MgiqLo6SQuR+vWrcPmzZtbrQR9++23sXTpUpw8ebLbD+61Wq1QKBQYv3gzZPJAV6ZLPuZgeoKnUyAiov+v5fvbYrEgJCSkw7aSvcdNav7yl7/g2muvxYABA1BSUoL09HQsWrTIcbyhoQEmkwnr16/HwoUL+WsLRERE1IrPFG5XXXUVjEZjm8e2bNmCBx54oJczcvb9999j7dq1OHv2LK644go8/fTTSE5OdhzfsGED1q1bh2nTpjntvxSfrY3vtGInIiIi6fGZS6VGoxFNTU1tHhs0aFC7vz7gS7oz1UpERETe4bK8VNqdRQZEREREUiTZVaVERERElxufmXGj/5q2citXlZJbcDUqEZFnccaNiIiISCJYuBERERFJBAu3HhJFEUlJSQgLC4MgCCgvL3eKDQaDp1MkIiIiH8HCrYcKCwuRk5ODgoICmEwm1NTUOMUxMTEQBAH5+fld7tNms+Hqq69m4UdEREROuDihhyorK6FWq6HX6wFcfJ7c/8aX4g9/+AMGDx6MQ4cOuSpNIiIi8gEs3HogMTERubm5AABBEKDVah2/3tASt4iLiwNw8XlzVVVV7fb50UcfYdeuXXjvvffw0UcfuS95IiIikhwWbj2QmZmJqKgoZGVloaysDDabDXl5eY5YJpMBACIiIpCdnY3Y2FjHvracOnUKCxYsQH5+PoKCgjp9f5vNBpvN5oitVmvPT4qIiIi8Fgu3HlAoFAgODoZMJoNKpQKAVnELpVLZat//EkURiYmJeOyxxzBx4sQOZ+VapKWlITU1tUfnQERERNLBxQle4tVXX8X58+e79QPzycnJsFgsju3EiRNuzJCIiIg8jYWbl/j000+xb98+yOVy9OnTB8OHDwcATJw4EfPnz2/zNXK5HCEhIU4bERER+S5eKu0F/v7+sNvtHbZ55ZVXsHbtWkd88uRJzJ49G3//+98xefJkd6dIREREEsAZt14QGRmJ4uJimM1m1NXVAQBKS0sRHR2N2tpaAMAVV1yBmJgYxzZy5EgAQFRUFIYOHeqx3ImIiMh7sHDrBRkZGSgqKoJGo4FOpwMANDQ0oKKiAk1NTR7OjoiIiKRCEEVR9HQS5BpWqxUKhQLjF2+GTB7o6XTIBx1MT/B0CkREPqfl+9tisXR6vzpn3IiIiIgkgosTfNBna+O5wpSIiMgHccaNiIiISCJYuBERERFJBAs3IiIiIongPW4+aNrKrVxVSnQJuGqWiLwdZ9yIiIiIJIKFGxEREZFEsHAjIiIikggWbj0kiiKSkpIQFhYGQRBQXl7uFBsMBk+nSERERD6ChVsPFRYWIicnBwUFBTCZTKipqXGKY2JiIAgC8vPzO+ynqqoKjzzyCK688koEBgYiKioKKSkpaGxs7J0TISIiIq/HVaU9VFlZCbVaDb1eDwAwGo1OcVf9+9//RnNzM7Zs2YLhw4fj22+/xYIFC/Dzzz9j48aN7kidiIiIJIaFWw8kJiYiNzcXACAIArRaLYxGo1PcIi4uDgCg1WpRVVXVqq/Y2FjExsY64mHDhqGiogKbNm1qt3Cz2Wyw2WyO2Gq19viciIiIyHuxcOuBzMxMREVFISsrC2VlZbDZbMjLy3PEMpkMABAREYHs7GzExsY69nWFxWJBWFhYu8fT0tKQmpra4/MgIiIiaeA9bj2gUCgQHBwMmUwGlUoFrVbrFIeHhyM8PBwAoFQqHfu64tixY3j11VexcOHCdtskJyfDYrE4thMnTrjkvIiIiMg7ccbNC9XW1iI2NhZz5szBggUL2m0nl8shl8t7MTMiIiLyJM64eZmTJ09ixowZ0Ov1yMrK8nQ6RERE5EVYuPUCf39/2O32TtvV1tZi+vTpuOaaa5CdnQ0/P/71EBER0X+xMugFkZGRKC4uhtlsRl1dHQCgtLQU0dHRqK2tBfDfou2KK67Axo0b8eOPP8JsNsNsNnsydSIiIvIiLNx6QUZGBoqKiqDRaKDT6QAADQ0NqKioQFNTEwCgqKgIx44dQ3FxMYYOHQq1Wu3YiIiIiABAEEVR9HQS5BpWqxUKhQLjF2+GTB7o6XSIJOdgeoKnUyCiy1DL97fFYkFISEiHbTnjRkRERCQRfByID/psbXynFTsRERFJD2fciIiIiCSChRsRERGRRLBwIyIiIpII3uPmg6at3MpVpUQSwtWsRNRVnHEjIiIikggWbkREREQSwcKth0RRRFJSEsLCwiAIAsrLy51ig8Hg6RSJiIjIR7Bw66HCwkLk5OSgoKAAJpMJNTU1TnFMTAwEQUB+fn6nfUVGRkIQBKdt/fr17j8JIiIikgQuTuihyspKqNVq6PV6AIDRaHSKu2vNmjVYsGCBIw4ODnZJnkRERCR9LNx6IDExEbm5uQAAQRCg1WphNBqd4hZxcXEAAK1Wi6qqqnb7DA4Ohkqlcl/SREREJFm8VNoDmZmZWLNmDYYOHQqTyYS9e/c6xWVlZSgrKwMAZGdnO/Z1ZP369RgwYAB0Oh3S09Nx4cKFdtvabDZYrVanjYiIiHwXZ9x6QKFQIDg4GDKZzDFL9uu4hVKp7HQm7cknn8SECRMQFhaGL774AsnJyTCZTHjppZfabJ+WlobU1FTXnAwRERF5PRZuXmTZsmWOP48bNw4BAQFYuHAh0tLSIJfLW7VPTk52eo3VaoVGo+mVXImIiKj3sXDzYpMnT8aFCxdQVVWFUaNGtToul8vbLOiIiIjIN/Eet17g7+8Pu93e7dcZDAb4+fkhIiLCDVkRERGR1LBw6wWRkZEoLi6G2WxGXV0dAKC0tBTR0dGora0FAOzbtw8vv/wyDh06hB9++AHvvPMOli5dinnz5iE0NNST6RMREZGXYOHWCzIyMlBUVASNRgOdTgcAaGhoQEVFBZqamgBcvOy5bds23Hjjjbjqqquwbt06LF26FFlZWZ5MnYiIiLyIIIqi6OkkyDWsVisUCgXGL94MmTzQ0+kQURcdTE/wdApE5EEt398WiwUhISEdtuWMGxEREZFEcFWpD/psbXynFTsRERFJD2fciIiIiCSChRsRERGRRLBwIyIiIpII3uPmg6at3MpVpURERC7mDSvAOeNGREREJBEs3IiIiIgkgoUbERERkUT4fOE2ffp0PPXUU55Oo0siIyPx8ssvezoNIiIi8lI+X7h15vXXX8cNN9yA0NBQhIaGYtasWSgtLfV0WkREREStXPaF2549exAfH4/du3dj37590Gg0uOWWW1BbW+vp1IiIiIiceLRwmz59OhYtWoRFixZBoVBg4MCBWLVqFVp+995ms2H58uUYMmQI+vXrh8mTJ2PPnj2O1585cwbx8fEYMmQIgoKCMHbsWGzdurXD99y5cycUCgXeeecdAMA777yDxx9/HFdffTWio6PxxhtvoLm5GcXFxY7XREZG4oUXXkB8fDz69euHIUOG4LXXXnPq99y5c3j00UcRHh6OkJAQ3HTTTTh06JDjeGVlJe6++24MGjQI/fv3x7XXXotPPvmkw1zfeOMNKJVKp1z+l81mg9VqddqIiIjId3l8xi03Nxd9+vRBaWkpMjMz8dJLL+GNN94AACxatAj79u3Dtm3b8PXXX2POnDmIjY3F999/DwD4z3/+g2uuuQY7d+7Et99+i6SkJDz44IPtXur829/+hvj4eLzzzjt44IEH2mzT0NCApqYmhIWFOe1PT0/H+PHjUV5ejhUrVmDJkiUoKipyHJ8zZw5Onz6Njz76CAcPHsSECRMwc+ZMnD17FgBQX1+P2267DcXFxSgvL0dsbCzuvPNOVFdXt5nHhg0bsGLFCuzatQszZ85ss01aWhoUCoVj02g0HYw0ERERSZ0gtkxvecD06dNx+vRpHD58GIIgAABWrFiBDz74AIWFhRg2bBiqq6sxePBgx2tmzZqFSZMm4cUXX2yzzzvuuAPR0dHYuHGj4z2uvvpqjBgxAs899xzef/993Hjjje3m9Pjjj+Pjjz/G4cOH0bdvXwAXZ9xGjx6Njz76yNFu7ty5sFqt+PDDD/H555/j9ttvx+nTpyGXyx1thg8fjj/84Q9ISkpq871iYmLw2GOPYdGiRY73eeqpp2AymfD222+jqKgIV111Vbu52mw22Gw2R2y1WqHRaDB+8WY+gJeIiMjF3PUAXqvVCoVCAYvFgpCQkA7bevyXE6677jpH0QYAU6ZMQUZGBr755hvY7XaMHDnSqb3NZsOAAQMAAHa7HS+++CLeffdd1NbWorGxETabDUFBQU6v2b59O06fPo2SkhJce+217eayfv16bNu2DXv27HEUbf+b16/jlhWghw4dQn19vSOvFr/88gsqKysBXJxxe/7557Fz506YTCZcuHABv/zyS6sZt4yMDPz88884cOAAhg0b1m6uACCXy50KRSIiIvJtHi/c2lNfXw+ZTIaDBw9CJpM5Hevfvz+Ai5cvMzMz8fLLL2Ps2LHo168fnnrqKTQ2Njq11+l0+Oqrr/DWW29h4sSJToVii40bN2L9+vX45JNPMG7cuG7nqlarne6/a6FUKgEAy5cvR1FRETZu3Ijhw4cjMDAQ9957b6tcb7jhBuzcuRPvvvsuVqxY0a08iIiIyLd5vHDbv3+/U/zll19ixIgR0Ol0sNvtOH36NG644YY2X1tSUoK7774b8+bNAwA0Nzfj6NGjGDNmjFO7qKgoZGRkYPr06ZDJZPjzn//sdHzDhg1Yt24dPv74Y0ycOLHN9/ryyy9bxaNHjwYATJgwAWazGX369EFkZGS7uSYmJiIuLg7AxWKvqqqqVbtJkyZh0aJFiI2NRZ8+fbB8+fI2+yMiIqLLj8cXJ1RXV2PZsmWoqKjA1q1b8eqrr2LJkiUYOXIkHnjgASQkJGDHjh04fvw4SktLkZaWhp07dwIARowYgaKiInzxxRc4cuQIFi5ciFOnTrX5PiNHjsTu3bvx3nvvOT2Q949//CNWrVqFt956C5GRkTCbzTCbzaivr3d6fUlJCTZs2ICjR4/itddewz/+8Q8sWbIEwMX77qZMmYJ77rkHu3btQlVVFb744gs899xzOHDggCPXHTt2wGAw4NChQ7j//vvR3NzcZq56vR4ffvghUlNT+UBeIiIicvD4jFtCQgJ++eUXTJo0CTKZDEuWLHHczJ+dnY21a9fi6aefRm1tLQYOHIjrrrsOd9xxBwBg5cqV+OGHHzB79mwEBQUhKSkJ99xzDywWS5vvNWrUKHz66aeOmbeMjAxs2rQJjY2NuPfee53apqSk4Pnnn3fETz/9NA4cOIDU1FSEhITgpZdewuzZswEAgiDgww8/xHPPPYeHHnoIP/74I1QqFaZNm4ZBgwYBAF566SU8/PDD0Ov1GDhwIJ599tkOH99x/fXXY+fOnbjtttsgk8mwePHiSx5jIiIi8g0eX1V69dVXe/2sUstqT2//6ayWVSlcVUpEROR63rCq1OOXSomIiIioazx+qZRc77O18Z1W7ERERCQ9Hi3c2np8hjdqa/WnN2q56s2fviIiIpKOlu/trty9xhk3H3LmzBkA4E9fERERSdD58+ehUCg6bMPCzYe0/L5qdXV1p3/xdGlaflbsxIkTvBztJhxj9+L4uh/H2P18bYxFUcT58+edfuKzPSzcfIif38W1JgqFwif+IXuzkJAQjrGbcYzdi+Prfhxj9/OlMe7qhAtXlRIRERFJBAs3IiIiIolg4eZD5HI5UlJSIJfLPZ2Kz+IYux/H2L04vu7HMXa/y3mMPfrLCURERETUdZxxIyIiIpIIFm5EREREEsHCjYiIiEgiWLgRERERSQQLNy/32muvITIyEn379sXkyZNRWlraYft//OMfiI6ORt++fTF27Fh8+OGHTsdFUcTq1auhVqsRGBiIWbNm4fvvv3fnKXg1V49vYmIiBEFw2mJjY915Cl6vO2N8+PBh/Pa3v0VkZCQEQcDLL7/c4z4vB64e4+eff77Vv+Po6Gg3noF36874vv7667jhhhsQGhqK0NBQzJo1q1V7fg635uox9unPYpG81rZt28SAgADxrbfeEg8fPiwuWLBAVCqV4qlTp9psX1JSIspkMnHDhg3id999J65cuVL09/cXv/nmG0eb9evXiwqFQszPzxcPHTok3nXXXeKVV14p/vLLL711Wl7DHeM7f/58MTY2VjSZTI7t7NmzvXVKXqe7Y1xaWiouX75c3Lp1q6hSqcQ//elPPe7T17ljjFNSUsSrrrrK6d/xjz/+6OYz8U7dHd/7779ffO2118Ty8nLxyJEjYmJioqhQKMSamhpHG34OO3PHGPvyZzELNy82adIk8YknnnDEdrtdHDx4sJiWltZm+/vuu0+8/fbbnfZNnjxZXLhwoSiKotjc3CyqVCoxPT3dcfzcuXOiXC4Xt27d6oYz8G6uHl9RvPhhcffdd7slXynq7hj/L61W22ZR0ZM+fZE7xjglJUUcP368C7OUrp7+e7tw4YIYHBws5ubmiqLIz+G2uHqMRdG3P4t5qdRLNTY24uDBg5g1a5Zjn5+fH2bNmoV9+/a1+Zp9+/Y5tQeA2bNnO9ofP34cZrPZqY1CocDkyZPb7dNXuWN8W+zZswcREREYNWoUfv/73+PMmTOuPwEJuJQx9kSfUubO8fj+++8xePBgDBs2DA888ACqq6t7mq7kuGJ8Gxoa0NTUhLCwMAD8HP41d4xxC1/9LGbh5qV++ukn2O12DBo0yGn/oEGDYDab23yN2WzusH3Lf7vTp69yx/gCQGxsLPLy8lBcXIw//vGP2Lt3L2699VbY7XbXn4SXu5Qx9kSfUuau8Zg8eTJycnJQWFiITZs24fjx47jhhhtw/vz5nqYsKa4Y32effRaDBw92FCb8HHbmjjEGfPuzuI+nEyDyJXPnznX8eezYsRg3bhyioqKwZ88ezJw504OZEXXdrbfe6vjzuHHjMHnyZGi1Wrz77rt45JFHPJiZtKxfvx7btm3Dnj170LdvX0+n45PaG2Nf/izmjJuXGjhwIGQyGU6dOuW0/9SpU1CpVG2+RqVSddi+5b/d6dNXuWN82zJs2DAMHDgQx44d63nSEnMpY+yJPqWst8ZDqVRi5MiRl92/456M78aNG7F+/Xrs2rUL48aNc+zn57Azd4xxW3zps5iFm5cKCAjANddcg+LiYse+5uZmFBcXY8qUKW2+ZsqUKU7tAaCoqMjR/sorr4RKpXJqY7VasX///nb79FXuGN+21NTU4MyZM1Cr1a5JXEIuZYw90aeU9dZ41NfXo7Ky8rL7d3yp47thwwa88MILKCwsxMSJE52O8XPYmTvGuC0+9Vns6dUR1L5t27aJcrlczMnJEb/77jsxKSlJVCqVotlsFkVRFB988EFxxYoVjvYlJSVinz59xI0bN4pHjhwRU1JS2nwciFKpFN9//33x66+/Fu++++7Ldhm6q8f3/Pnz4vLly8V9+/aJx48fFz/55BNxwoQJ4ogRI8T//Oc/HjlHT+vuGNtsNrG8vFwsLy8X1Wq1uHz5crG8vFz8/vvvu9zn5cYdY/z000+Le/bsEY8fPy6WlJSIs2bNEgcOHCiePn2618/P07o7vuvXrxcDAgLE7du3Oz2K4vz5805t+Dn8X64eY1//LGbh5uVeffVV8YorrhADAgLESZMmiV9++aXj2I033ijOnz/fqf27774rjhw5UgwICBCvuuoqcefOnU7Hm5ubxVWrVomDBg0S5XK5OHPmTLGioqI3TsUruXJ8GxoaxFtuuUUMDw8X/f39Ra1WKy5YsOCyLShadGeMjx8/LgJotd14441d7vNy5Oox/t3vfieq1WoxICBAHDJkiPi73/1OPHbsWC+ekXfpzvhqtdo2xzclJcXRhp/DrblyjH39s1gQRVHs3Tk+IiIiIroUvMeNiIiISCJYuBERERFJBAs3IiIiIolg4UZEREQkESzciIiIiCSChRsRERGRRLBwIyIiIpIIFm5EREREEsHCjYjoV6ZPn46nnnrK02kQEbXCX04gIvqVs2fPwt/fH8HBwZ5OpZU9e/ZgxowZqKurg1Kp9HQ6RNTL+ng6ASIibxMWFubpFNrU1NTk6RSIyMN4qZSI6Ff+91JpZGQk1q5di4SEBPTv3x9arRYffPABfvzxR9x9993o378/xo0bhwMHDjhen5OTA6VSifz8fIwYMQJ9+/bF7NmzceLECaf32bRpE6KiohAQEIBRo0bh7bffdjouCAI2bdqEu+66C/369cOCBQswY8YMAEBoaCgEQUBiYiIAoLCwENdffz2USiUGDBiAO+64A5WVlY6+qqqqIAgCduzYgRkzZiAoKAjjx4/Hvn37nN6zpKQE06dPR1BQEEJDQzF79mzU1dUBAJqbm5GWloYrr7wSgYGBGD9+PLZv3+6SMSeirmHhRkTUiT/96U+YOnUqysvLcfvtt+PBBx9EQkIC5s2bh6+++gpRUVFISEjA/9550tDQgHXr1iEvLw8lJSU4d+4c5s6d6zj+f//3f1iyZAmefvppfPvtt1i4cCEeeugh7N692+m9n3/+ecTFxeGbb75Bamoq3nvvPQBARUUFTCYTMjMzAQA///wzli1bhgMHDqC4uBh+fn6Ii4tDc3OzU3/PPfccli9fDoPBgJEjRyI+Ph4XLlwAABgMBsycORNjxozBvn378Pnnn+POO++E3W4HAKSlpSEvLw+bN2/G4cOHsXTpUsybNw979+51/aATUdtEIiJycuONN4pLliwRRVEUtVqtOG/ePMcxk8kkAhBXrVrl2Ldv3z4RgGgymURRFMXs7GwRgPjll1862hw5ckQEIO7fv18URVHU6/XiggULnN53zpw54m233eaIAYhPPfWUU5vdu3eLAMS6uroOz+HHH38UAYjffPONKIqiePz4cRGA+MYbbzjaHD58WAQgHjlyRBRFUYyPjxenTp3aZn//+c9/xKCgIPGLL75w2v/II4+I8fHxHeZCRK7DGTciok6MGzfO8edBgwYBAMaOHdtq3+nTpx37+vTpg2uvvdYRR0dHQ6lU4siRIwCAI0eOYOrUqU7vM3XqVMfxFhMnTuxSjt9//z3i4+MxbNgwhISEIDIyEgBQXV3d7rmo1WqnvFtm3Npy7NgxNDQ04Oabb0b//v0dW15entMlWSJyLy5OICLqhL+/v+PPgiC0u+/XlyVdoV+/fl1qd+edd0Kr1eL111/H4MGD0dzcjJiYGDQ2Njq16yjvwMDAdvuvr68HAOzcuRNDhgxxOiaXy7uUIxH1HGfciIjc4MKFC04LFioqKnDu3DmMHj0aADB69GiUlJQ4vaakpARjxozpsN+AgAAAcNx3BgBnzpxBRUUFVq5ciZkzZ2L06NGOBQXdMW7cOBQXF7d5bMyYMZDL5aiursbw4cOdNo1G0+33IqJLwxk3IiI38Pf3x+LFi/HKK6+gT58+WLRoEa677jpMmjQJAPDMM8/gvvvug06nw6xZs/DPf/4TO3bswCeffNJhv1qtFoIgoKCgALfddhsCAwMRGhqKAQMGICsrC2q1GtXV1VixYkW3c05OTsbYsWPx+OOP47HHHkNAQAB2796NOXPmYODAgVi+fDmWLl2K5uZmXH/99bBYLCgpKUFISAjmz59/SeNERN3DGTciIjcICgrCs88+i/vvvx9Tp05F//798fe//91x/J577kFmZiY2btyIq666Clu2bEF2djamT5/eYb9DhgxBamoqVqxYgUGDBmHRokXw8/PDtm3bcPDgQcTExGDp0qVIT0/vds4jR47Erl27cOjQIUyaNAlTpkzB+++/jz59Lv4//gsvvIBVq1YhLS0No0ePRmxsLHbu3Ikrr7yy2+9FRJeGv5xARORiOTk5eOqpp3Du3DlPp0JEPoYzbkREREQSwcKNiIiISCJ4qZSIiIhIIjjjRkRERCQRLNyIiIiIJIKFGxEREZFEsHAjIiIikggWbkREREQSwcKNiIiISCJYuBERERFJBAs3IiIiIon4f30hN0zOIozfAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "feature_importance = pandas.Series(estimator.feature_importances_, index=estimator.feature_names_in_, name='importance').to_frame()\n", + "feature_importance.index.name = 'feature'\n", + "\n", + "import seaborn\n", + "seaborn.barplot(data=feature_importance.reset_index().sort_values('importance'), x='importance', y='feature')" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "3f2b6d0b-7a82-4f4a-ae63-bc126727a698", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "add-events {'col': 1, 'row': 1} y\n", + " peak2peak fft_energy\n", + "count 351.000000 3.510000e+02\n", + "mean 0.356169 5.880848e-01\n", + "std 0.340400 5.002171e-01\n", + "min 0.003904 -7.625756e-08\n", + "25% 0.144560 3.689122e-01\n", + "50% 0.223043 4.718358e-01\n", + "75% 0.487939 7.283085e-01\n", + "max 2.440344 6.391219e+00\n", + "[0. 0.4 0.8 1.2 1.6 2. 2.3 2.7 3.1 3.5 3.9 4.3 4.7 5.1 5.5 5.9]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/jon/projects/emlearn-micropython/paper/plotting.py:27: FutureWarning:\n", + "\n", + "A grouping was used that is not in the columns of the DataFrame and so was excluded from the result. This grouping will be included in a future version of pandas. Add the grouping as a column of the DataFrame to silence this warning.\n", + "\n", + "/home/jon/projects/emlearn-micropython/paper/plotting.py:27: FutureWarning:\n", + "\n", + "A grouping was used that is not in the columns of the DataFrame and so was excluded from the result. This grouping will be included in a future version of pandas. Add the grouping as a column of the DataFrame to silence this warning.\n", + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "add-events {'col': 1, 'row': 4} y4\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "\n", + "colors ={\n", + " 'Nordic_walking': 'rgb(251,128,114)',\n", + " 'cycling': 'rgb(179,222,105)',\n", + " 'walking': 'rgb(253,180,98)',\n", + " 'transient': 'rgb(217,217,217)',\n", + " 'running': 'rgb(255,237,111)',\n", + " 'rope_jumping': '#2E91E5',\n", + " 'lying': 'rgb(255,255,179)',\n", + " 'sitting': 'rgb(190,186,218)',\n", + " 'standing': 'rgb(251,128,114)',\n", + " 'ironing': 'rgb(128,177,211)',\n", + " 'vacuum_cleaning': 'rgb(253,180,98)',\n", + " 'ascending_stairs': 'rgb(179,222,105)',\n", + " 'descending_stairs': 'rgb(252,205,229)',\n", + "}\n", + "\n", + "width = 1600\n", + "aspect = 2.0\n", + "height = width/aspect\n", + "fig = make_timeline_plot(sub, combined, smooth, colors, class_names=class_columns, width=width, aspect=aspect)\n", + "\n", + "plot_path = 'physical_activity_recognition_pipeline.png'\n", + "fig.write_image(plot_path, scale=1.5, width=width, height=height)\n", + "fig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf621865-2c87-4b5a-9887-02d6ffd65d27", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ddbf30ef-bd90-42e5-b88c-d85e7c7721e3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0c36251-21ef-4e3c-b363-0709e7c54409", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "emlearn-micropython2", + "language": "python", + "name": "emlearn-micropython2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/paper/example.py b/paper/example.py new file mode 100644 index 0000000..2ea2f05 --- /dev/null +++ b/paper/example.py @@ -0,0 +1,124 @@ + +import emlearn_fft +import emlearn_trees +import npyfile + +import array + +class AccelerometerClassifier(): + def __init__(self, window_length : int, + max_trees=10, max_nodes=1000, + dimensions=3, model_file=None, fft_start=0, fft_end=16): + + self.window_length = window_length + self.dimensions = dimensions + self.fft_start = fft_start + self.fft_end = fft_end + + # Setup FFT + fft_length = self.window_length + self.fft = emlearn_fft.FFT(fft_length) + emlearn_fft.fill(self.fft, fft_length) + self.fft_real = array.array('f', (0 for _ in range(fft_length))) + self.fft_imag = array.array('f', (0 for _ in range(fft_length))) + + # Setup tree-based classification model + self.model = None + self.probabilties = None + if model_file is not None: + self.model = emlearn_trees.new(max_trees, max_nodes, 10) + with open(model_file) as f: + emlearn_trees.load_model(self.model, f) + classes = self.model.outputs() + self.probabilities = array.array('f', (0 for _ in range(classes))) + + def preprocess(self, samples : array.array): + assert len(samples) == (self.window_length*self.dimensions) + samples_length = len(samples) // self.dimensions + + # Preprocess using FFT and simple statistics + magnitude_min = float('inf') + magnitude_max = -float('inf') + for i in range(samples_length): + x = samples[(i*3)+0] / 2**15 + y = samples[(i*3)+1] / 2**15 + z = samples[(i*3)+2] / 2**15 + magnitude = x*x + y*y + z*z + if magnitude < magnitude_min: + magnitude_min = magnitude + if magnitude > magnitude_max: + magnitude_max = magnitude + self.fft_real[i] = magnitude + self.fft_imag[i] = 0 + + self.fft.run(self.fft_real, self.fft_imag) + peak2peak = (magnitude_max - magnitude_min) + # Normalize FFT by total energy + fft_energy = sum(self.fft_real) + if fft_energy > 1e-6: + for i in range(len(self.fft_real)): + self.fft_real[i] = 2**14 * (abs(self.fft_real[i]) / fft_energy) + else: + for i in range(len(self.fft_real)): + self.fft_real[i] = 0.0 + + # Pick relevant features + fft_features = list(self.fft_real[self.fft_start:self.fft_end]) + features = [ 2**14 * peak2peak, 2**10 * fft_energy ] + fft_features + return features + + def classify(self, features): + self.model.predict(features, self.probabilities) + return self.probabilities + +def process_file(inp, out, model=None): + + print('process-file', inp, out, model) + + window_length = 256 + pipeline = AccelerometerClassifier(window_length=window_length, model_file=model) + fft_features = pipeline.fft_end - pipeline.fft_start + + with npyfile.Reader(inp) as reader: + + # check input + assert len(reader.shape) == 2 + assert reader.shape[1] == pipeline.dimensions + assert reader.typecode == 'h' + + # determine expected output + chunksize = reader.shape[1]*pipeline.window_length + n_windows = reader.shape[0]//pipeline.window_length + out_dimensions = 2 + fft_features + if model is not None: + out_dimensions += pipeline.model.outputs() + out_shape = (n_windows, out_dimensions) + + # process the data + written = 0 + n_chunks = 0 + with npyfile.Writer(out, out_shape, 'f') as writer: + + for chunk in reader.read_data_chunks(chunksize): + if len(chunk) < chunksize: + continue # last window chunk might be incomplete, skip it + + features = list(pipeline.preprocess(chunk)) + arr = array.array('h', [int(f) for f in features]) + if model: + probabilities = pipeline.classify(arr) + else: + probabilities = [] + + out = array.array('f', features + list(probabilities)) + assert len(out) == out_dimensions, (len(out), out_dimensions) + writer.write_values(out) + written += len(out) + n_chunks += 1 + print(out_shape, written, n_chunks, written/n_chunks) + +if __name__ == '__main__': + import sys + assert len(sys.argv) >= 3 + model = sys.argv[3] if len(sys.argv) > 3 else None + process_file(sys.argv[1], sys.argv[2], model) diff --git a/paper/notes.md b/paper/notes.md new file mode 100644 index 0000000..abdbe20 --- /dev/null +++ b/paper/notes.md @@ -0,0 +1,116 @@ + +# Howto + +## Building PDF from paper + +docker run --rm --volume $PWD/paper:/data --user $(id -u):$(id -g) --env JOURNAL=joss openjournals/inara + +# TODO + +Non-critical + +- Add linreg to the API reference +- Fixup hardware support page. armv6m now should work? +- Add a documentation page about the wider MicroPython Data Science ecosystem? +- Donate another 100 USD to JOSS + +# Process + ++ Write the paper ++ Submit ++ Respond to review comments + +https://joss.readthedocs.io/en/latest/submitting.html + +## Usage example + +Ideally show 2 modules? +Can be in a composite example. + +For plot would need to output data. npyfile? +Point out that the same code runs on microcontroller (such as ESP32, RP2350, STM32 etc) + +# Paper + +WIP in joss-paper branch of emlearn-micropython + +## Examples +https://joss.readthedocs.io/en/latest/example_paper.html + +## Outline +Should be 250-1000 words. +Scope. 2-3 pages, plus references + +- Summary. 1-3 paragraphs. Max 1/2 page +- Statement of need. Up to 1/2 page +- Package contents. Table of the modules? +- Usage example. Short but illustrative. One attractive plot +- References + +## What they ask for + + + A list of the authors of the software and their affiliations, using the correct format (see the example below). + + A summary describing the high-level functionality and purpose of the software for a diverse, non-specialist audience. + + A Statement of need section that clearly illustrates the research purpose of the software and places it in the context of related work. + + A list of key references, including to other software addressing related needs. Note that the references should include full names of venues, e.g., journals and conferences, not abbreviations only understood in the context of a specific discipline. + + Mention (if applicable) a representative set of past or ongoing research projects using the software and recent scholarly publications enabled by it. + + Acknowledgement of any financial support. +s + +## Focus ++ Make it clear that it is relevant *for research* (eg in TinyML applications) ++ Make it likely that it will be cited. + +# Related softwares + +Things to cite + +emlearn, TinyMaix +scikit-learn, keras, tensorflow / tf lite micro. numpy? scipy? + +Alternatives +ulab, OpenMV. +Generating Python code. Using m2cgen, etc + +For the implemented methods, the original papers + +# Suggesting substantial scholarly effort + +makes addressing research challenges significantly better (e.g., faster, easier, simpler). + +emlearn-micropython makes research in Machine Learning for embedded systems easier. +This can both be applied research, and application oriented. Data collection and prototyping +Along with research in methods. By providing an example approach for developing ML methods for deployment on microcontrollers with MicroPython + + +# Suggesting citability +Already have one citation, +https://www.sciencedirect.com/science/article/pii/S2352711024001493 + +These papers identify a need for MicroPython to improve performance on numeric workloads +https://www.mdpi.com/2079-9292/12/1/143 +https://ieeexplore.ieee.org/abstract/document/9292199/ +https://link.springer.com/chapter/10.1007/978-3-030-43364-2_4 + +Real-Time Human Activity Recognition on Embedded Equipment: A Comparative Study +The C implementation on ESP32 still has a shorter processing time (0.0022 s) compared to MicroPython on ESP32 (0.15 s). + + +OpenMV +https://arxiv.org/abs/1711.10464 + +Example research targeting TinyML+MicroPython +Could benefit from emlearn-micropython +https://www.mdpi.com/1424-8220/23/4/2344 (used ulab) +https://ieeexplore.ieee.org/abstract/document/8656727 +https://elifesciences.org/articles/67846 +https://www.sciencedirect.com/science/article/pii/S2772375523000138 + + + diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 0000000..35b7dde --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,351 @@ + +@misc{micropython, + author = {George, Damien P and contributors}, + title = {MicroPython - Python for Microcontrollers}, + year = {2014}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/micropython/micropython}} +} + +@misc{micropython_native_module, + author = {George, Damien P and contributors}, + title = {MicroPython - Native machine code in .mpy files}, + year = {2020}, + publisher = {MicroPython}, + journal = {MicroPython documentation}, + howpublished = {\url{https://docs.micropython.org/en/latest/develop/natmod.html}} +} + +@misc{TinyMaix, + author = {Wu, Caesar and contributors}, + title = {TinyMaix - a tiny inference library for microcontrollers}, + year = {2022}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/sipeed/TinyMaix}} +} + +@misc{ulab, + author = {Vörös, Zoltán and contributors}, + title = {ulab - numpy-like fast vector module for micropython, circuitpython, and their derivatives}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/v923z/micropython-ulab}} +} + +@misc{m2cgen, + author = {Zeigerman, Iaroslav and Titov, Nikita and Yershov, Viktor and contributors}, + title = {m2cgen - Transform ML models into a native code with zero dependencies}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/BayesWitnesses/m2cgen}} +} + +@misc{tflite_micro, + author = {Google LLC and contributors}, + title = {tflite-micro - Tensorflow Lite for Microcontrollers}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/tensorflow/tflite-micro}} +} + + +@inproceedings{pamap2_dataset, + title={Introducing a new benchmarked dataset for activity monitoring}, + author={Reiss, Attila and Stricker, Didier}, + booktitle={2012 16th international symposium on wearable computers}, + pages={108--109}, + year={2012}, + organization={IEEE} +} + + +@ARTICLE{scipy, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} + +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} + +@misc{keras, + title={Keras}, + author={Chollet, Fran\c{c}ois and others}, + year={2015}, + howpublished={\url{https://keras.io}}, +} + +@misc{tensorflow, +title={ {TensorFlow}: Large-Scale Machine Learning on Heterogeneous Systems}, +url={https://www.tensorflow.org/}, +note={Software available from tensorflow.org}, +author={ + Mart\'{i}n~Abadi and + Ashish~Agarwal and + Paul~Barham and + Eugene~Brevdo and + Zhifeng~Chen and + Craig~Citro and + Greg~S.~Corrado and + Andy~Davis and + Jeffrey~Dean and + Matthieu~Devin and + Sanjay~Ghemawat and + Ian~Goodfellow and + Andrew~Harp and + Geoffrey~Irving and + Michael~Isard and + Yangqing Jia and + Rafal~Jozefowicz and + Lukasz~Kaiser and + Manjunath~Kudlur and + Josh~Levenberg and + Dandelion~Man\'{e} and + Rajat~Monga and + Sherry~Moore and + Derek~Murray and + Chris~Olah and + Mike~Schuster and + Jonathon~Shlens and + Benoit~Steiner and + Ilya~Sutskever and + Kunal~Talwar and + Paul~Tucker and + Vincent~Vanhoucke and + Vijay~Vasudevan and + Fernanda~Vi\'{e}gas and + Oriol~Vinyals and + Pete~Warden and + Martin~Wattenberg and + Martin~Wicke and + Yuan~Yu and + Xiaoqiang~Zheng}, + year={2015}, +} + +@article{OpenMV, + author = {Ibrahim Abdelkader and + Yasser El{-}Sonbaty and + Mohamed El{-}Habrouk}, + title = {Openmv: {A} Python powered, extensible machine vision camera}, + journal = {CoRR}, + volume = {abs/1711.10464}, + year = {2017}, + url = {http://arxiv.org/abs/1711.10464}, + eprinttype = {arXiv}, + eprint = {1711.10464}, + timestamp = {Mon, 13 Aug 2018 16:47:34 +0200}, + biburl = {https://dblp.org/rec/journals/corr/abs-1711-10464.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{karavaev2024tinydecisiontreeclassifier, + title={TinyDecisionTreeClassifier: Embedded C++ library for training and applying decision trees on the edge}, + author={Karavaev, Aleksei and Hejda, Jan and Kutilek, Patrik and Volf, Petr and Sokol, Marek and Leova, Lydie}, + journal={SoftwareX}, + volume={27}, + pages={101778}, + year={2024}, + publisher={Elsevier} +} + +@misc{emlearn, + author = {Nordby, Jon and Cooke, Mark and Horvath, Adam}, + title = {{emlearn: Machine Learning inference engine for + Microcontrollers and Embedded Devices}}, + month = mar, + year = 2019, + doi = {10.5281/zenodo.2589394}, + url = {https://doi.org/10.5281/zenodo.2589394} +} + +@article{tinyml_review_ray2022, + title={A review on TinyML: State-of-the-art and prospects}, + author={Ray, Partha Pratim}, + journal={Journal of King Saud University-Computer and Information Sciences}, + volume={34}, + number={4}, + pages={1595--1623}, + year={2022}, + publisher={Elsevier} +} + + +@Article{numpy, + title = {Array programming with {NumPy}}, + author = {Charles R. Harris and K. Jarrod Millman and St{\'{e}}fan J. + van der Walt and Ralf Gommers and Pauli Virtanen and David + Cournapeau and Eric Wieser and Julian Taylor and Sebastian + Berg and Nathaniel J. Smith and Robert Kern and Matti Picus + and Stephan Hoyer and Marten H. van Kerkwijk and Matthew + Brett and Allan Haldane and Jaime Fern{\'{a}}ndez del + R{\'{i}}o and Mark Wiebe and Pearu Peterson and Pierre + G{\'{e}}rard-Marchant and Kevin Sheppard and Tyler Reddy and + Warren Weckesser and Hameer Abbasi and Christoph Gohlke and + Travis E. Oliphant}, + year = {2020}, + month = sep, + journal = {Nature}, + volume = {585}, + number = {7825}, + pages = {357--362}, + doi = {10.1038/s41586-020-2649-2}, + publisher = {Springer Science and Business Media {LLC}}, + url = {https://doi.org/10.1038/s41586-020-2649-2} +} + +@article{antonini_adaptable_2023, + title = {An Adaptable and Unsupervised {TinyML} Anomaly Detection System for Extreme Industrial Environments}, + volume = {23}, + rights = {http://creativecommons.org/licenses/by/3.0/}, + issn = {1424-8220}, + url = {https://www.mdpi.com/1424-8220/23/4/2344}, + doi = {10.3390/s23042344}, + abstract = {Industrial assets often feature multiple sensing devices to keep track of their status by monitoring certain physical parameters. These readings can be analyzed with machine learning ({ML}) tools to identify potential failures through anomaly detection, allowing operators to take appropriate corrective actions. Typically, these analyses are conducted on servers located in data centers or the cloud. However, this approach increases system complexity and is susceptible to failure in cases where connectivity is unavailable. Furthermore, this communication restriction limits the approach’s applicability in extreme industrial environments where operating conditions affect communication and access to the system. This paper proposes and evaluates an end-to-end adaptable and configurable anomaly detection system that uses the Internet of Things ({IoT}), edge computing, and Tiny-{MLOps} methodologies in an extreme industrial environment such as submersible pumps. The system runs on an {IoT} sensing Kit, based on an {ESP}32 microcontroller and {MicroPython} firmware, located near the data source. The processing pipeline on the sensing device collects data, trains an anomaly detection model, and alerts an external gateway in the event of an anomaly. The anomaly detection model uses the isolation forest algorithm, which can be trained on the microcontroller in just 1.2 to 6.4 s and detect an anomaly in less than 16 milliseconds with an ensemble of 50 trees and 80 {KB} of {RAM}. Additionally, the system employs blockchain technology to provide a transparent and irrefutable repository of anomalies.}, + pages = {2344}, + number = {4}, + journaltitle = {Sensors}, + author = {Antonini, Mattia and Pincheira, Miguel and Vecchio, Massimo and Antonelli, Fabio}, + urldate = {2025-01-19}, + date = {2023-01}, + langid = {english}, + note = {Number: 4 +Publisher: Multidisciplinary Digital Publishing Institute}, + keywords = {anomaly detection, blockchain, Internet of Things, machine learning, Tiny-{MLOps}, {TinyML}}, + file = {Full Text PDF:/home/jon/Zotero/storage/S35SMC2T/Antonini et al. - 2023 - An Adaptable and Unsupervised TinyML Anomaly Detec.pdf:application/pdf}, +} + +@inproceedings{kokoulin_hierarchical_2019, + title = {Hierarchical Convolutional Neural Network Architecture in Distributed Facial Recognition System}, + url = {https://ieeexplore.ieee.org/abstract/document/8656727}, + doi = {10.1109/EIConRus.2019.8656727}, + abstract = {Authors propose an efficient distributed facial recognition system based on the cascade of Convolutional Neural Networks. The traditional approach utilizing the centralized schema of recognition system has its main disadvantage concluding a high traffic and computing load of the central processing server: the more video surveillance cameras are served by the central unit, the more its load rate. But most of the time the high-quality video stream does not consist of facial information and the computational resources are being wasted. The main principle of our recognition system is the distributed hierarchical processing network utilizing the "coarse-fine" paradigm. Each source video stream is processed in-place by the tiny {SoC} computer which acts as an Edge Computing Unit and detects the presence of a face fragment in a video frame and crops the bounds of the {ROI}. The resulting stream including {ROI} is relayed to the main server if the face is detected.}, + eventtitle = {2019 {IEEE} Conference of Russian Young Researchers in Electrical and Electronic Engineering ({EIConRus})}, + pages = {258--262}, + booktitle = {2019 {IEEE} Conference of Russian Young Researchers in Electrical and Electronic Engineering ({EIConRus})}, + author = {Kokoulin, Andrey N. and Tur, Aleksandr I. and Yuzhakov, Aleksandr A. and Knyazev, Aleksandr I.}, + urldate = {2025-01-19}, + date = {2019-01}, + note = {{ISSN}: 2376-6565}, + keywords = {Cameras, Convolutional Neural Network, Detectors, Distributed System, Face, Face recognition, Facial Recognition, Histograms, Servers, Streaming media}, + file = {IEEE Xplore Abstract Record:/home/jon/Zotero/storage/4H2BUBB4/8656727.html:text/html}, +} + +@article{akam_open-source_2022, + title = {Open-source, Python-based, hardware and software for controlling behavioural neuroscience experiments}, + volume = {11}, + issn = {2050-084X}, + url = {https://doi.org/10.7554/eLife.67846}, + doi = {10.7554/eLife.67846}, + abstract = {Laboratory behavioural tasks are an essential research tool. As questions asked of behaviour and brain activity become more sophisticated, the ability to specify and run richly structured tasks becomes more important. An increasing focus on reproducibility also necessitates accurate communication of task logic to other researchers. To these ends, we developed {pyControl}, a system of open-source hardware and software for controlling behavioural experiments comprising a simple yet flexible Python-based syntax for specifying tasks as extended state machines, hardware modules for building behavioural setups, and a graphical user interface designed for efficiently running high-throughput experiments on many setups in parallel, all with extensive online documentation. These tools make it quicker, easier, and cheaper to implement rich behavioural tasks at scale. As important, {pyControl} facilitates communication and reproducibility of behavioural experiments through a highly readable task definition syntax and self-documenting features. Here, we outline the system’s design and rationale, present validation experiments characterising system performance, and demonstrate example applications in freely moving and head-fixed mouse behaviour.}, + pages = {e67846}, + journaltitle = {{eLife}}, + author = {Akam, Thomas and Lustig, Andy and Rowland, James M and Kapanaiah, Sampath {KT} and Esteve-Agraz, Joan and Panniello, Mariangela and Márquez, Cristina and Kohl, Michael M and Kätzel, Dennis and Costa, Rui M and Walton, Mark E}, + editor = {Kemere, Caleb and Wassum, Kate M and Kemere, Caleb and Siegle, Josh}, + urldate = {2025-01-19}, + date = {2022-01-19}, + note = {Publisher: {eLife} Sciences Publications, Ltd}, + keywords = {Behaviour, Hardware, open source, Software}, + file = {Full Text PDF:/home/jon/Zotero/storage/PPXPRQ5L/Akam et al. - 2022 - Open-source, Python-based, hardware and software f.pdf:application/pdf}, +} + +@article{bordin_yamashita_coffee_2023, + title = {Coffee disease classification at the edge using deep learning}, + volume = {4}, + issn = {2772-3755}, + url = {https://www.sciencedirect.com/science/article/pii/S2772375523000138}, + doi = {10.1016/j.atech.2023.100183}, + abstract = {Brazil is the world’s largest producer and exporter of coffee and the second largest consumer of the beverage. The aim of this study is to embed convolutional networks in a low-cost microcontrolled board to classify coffee leaf diseases in loco, without the need for an internet connection. Early identification of diseases in coffee plantations is crucial for productivity and production quality. Two datasets were used, in addition to images taken with the development board itself, totaling more than 6000 images of six different types of diseases. The proposed architectures (cascade and single-stage), when embedded, presented accuracy values around 98\% and 96\%, respectively, demonstrating their ability to assist in the diagnosis of diseases in coffee farms, especially those managed by producers with less resources.}, + pages = {100183}, + journaltitle = {Smart Agricultural Technology}, + shortjournal = {Smart Agricultural Technology}, + author = {Bordin Yamashita, João Vitor Yukio and Leite, João Paulo R. R.}, + urldate = {2025-01-19}, + date = {2023-08-01}, + keywords = {Artificial intelligence, Coffee diseases, Convolutional networks, Deep learning, Edge computing}, + file = {ScienceDirect Snapshot:/home/jon/Zotero/storage/787EE7L4/S2772375523000138.html:text/html}, +} + +@article{plauska_performance_2023, + title = {Performance Evaluation of C/C++, {MicroPython}, Rust and {TinyGo} Programming Languages on {ESP}32 Microcontroller}, + volume = {12}, + rights = {http://creativecommons.org/licenses/by/3.0/}, + issn = {2079-9292}, + url = {https://www.mdpi.com/2079-9292/12/1/143}, + doi = {10.3390/electronics12010143}, + abstract = {The rapid growth of the Internet of Things ({IoT}) and its applications requires high computational efficiency, low-cost, and low-power solutions for various {IoT} devices. These include a wide range of microcontrollers that are used to collect, process, and transmit {IoT} data. {ESP}32 is a microcontroller with built-in wireless connectivity, suitable for various {IoT} applications. The {ESP}32 chip is gaining more popularity, both in academia and in the developer community, supported by a number of software libraries and programming languages. While low- and middle-level languages, such as C/C++ and Rust, are believed to be the most efficient, {TinyGo} and {MicroPython} are more developer-friendly low-complexity languages, suitable for beginners and allowing more rapid coding. This paper evaluates the efficiency of the available {ESP}32 programming languages, namely C/C++, {MicroPython}, Rust, and {TinyGo}, by comparing their execution performance. Several popular data and signal processing algorithms were implemented in these languages, and their execution times were compared: Fast Fourier Transform ({FFT}), Cyclic Redundancy Check ({CRC}), Secure Hash Algorithm ({SHA}), Infinite Impulse Response ({IIR}), and Finite Impulse Response ({FIR}) filters. The results show that the C/C++ implementations were fastest in most cases, closely followed by {TinyGo} and Rust, while {MicroPython} programs were many times slower than implementations in other programming languages. Therefore, the C/C++, {TinyGo}, and Rust languages are more suitable when execution and response time are the key factors, while Python can be used for less strict system requirements, enabling a faster and less complicated development process.}, + pages = {143}, + number = {1}, + journaltitle = {Electronics}, + author = {Plauska, Ignas and Liutkevičius, Agnius and Janavičiūtė, Audronė}, + urldate = {2025-01-19}, + date = {2023-01-01}, + langid = {english}, + note = {Number: 1 +Publisher: Multidisciplinary Digital Publishing Institute}, + keywords = {C/C++, {ESP}32, microcontroller, {MicroPython}, performance evaluation, Rust, {TinyGo}}, + file = {Full Text PDF:/home/jon/Zotero/storage/SDRQNE3L/Plauska et al. - 2023 - Performance Evaluation of CC++, MicroPython, Rust.pdf:application/pdf}, +} + +@inproceedings{ionescu_investigating_2020, + title = {Investigating the performance of {MicroPython} and C on {ESP}32 and {STM}32 microcontrollers}, + url = {https://ieeexplore.ieee.org/abstract/document/9292199}, + doi = {10.1109/SIITME50350.2020.9292199}, + abstract = {Python is a programming language that is used both by entry level programmers and advanced researchers. {MicroPython} is a software implementation of Python that runs on microcontrollers. This paper will investigate the {MicroPython} execution performance compared to similar C native code on low cost microcontrollers: {STM}32 and {ESP}32. The comparison will target: memory allocation speed; {SHA}-256 and {CRC}-32 performance and will present conclusions regarding the encountered problems and ways to improve the application performance.}, + eventtitle = {2020 {IEEE} 26th International Symposium for Design and Technology in Electronic Packaging ({SIITME})}, + pages = {234--237}, + booktitle = {2020 {IEEE} 26th International Symposium for Design and Technology in Electronic Packaging ({SIITME})}, + author = {Ionescu, Valeriu Manuel and Enescu, Florentina Magda}, + urldate = {2025-01-19}, + date = {2020-10}, + note = {{ISSN}: 2642-7036}, + keywords = {{CRC}, {ESP}32, Hardware, hash, Libraries, Microcontrollers, {MicroPython}, performance, Performance evaluation, Python, Resource management, Software, {STM}32}, + file = {IEEE Xplore Abstract Record:/home/jon/Zotero/storage/Y69KNDWU/9292199.html:text/html}, +} + +@inproceedings{dokic_micropython_2020, + location = {Cham}, + title = {{MicroPython} or Arduino C for {ESP}32 - Efficiency for Neural Network Edge Devices}, + isbn = {978-3-030-43364-2}, + doi = {10.1007/978-3-030-43364-2_4}, + abstract = {In the last few years, microcontrollers used for {IoT} devices became more and more powerful, and many authors have started to use them in machine learning systems. Most of the authors used them just for data collecting for {ML} algorithms in the clouds, but some of them implemented {ML} algorithms on the microcontrollers. The goal of this paper is to analyses the neural networks data propagation speed of one popular {SoC} (Espressif System company {ESP}32) with simple neural networks implementation with two different development environment, Arduino {IDE} and {MycroPython}. Neural networks with one hidden layer are used with a different number of neurons. This {SoC} is analysed because some companies started to produce them with {UXGA} (Ultra Extended Graphics Array) camera implemented and it can be used to distribute computing load from central {ML} servers.}, + pages = {33--43}, + booktitle = {Intelligent Computing Systems}, + publisher = {Springer International Publishing}, + author = {Dokic, Kristian and Radisic, Bojan and Cobovic, Mirko}, + editor = {Brito-Loeza, Carlos and Espinosa-Romero, Arturo and Martin-Gonzalez, Anabel and Safi, Asad}, + date = {2020}, + langid = {english}, +} diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 0000000..820a593 --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,135 @@ + +--- +title: 'emlearn-micropython: Machine Learning and Digital Signal Processing for MicroPython' +tags: + - MicroPython + - TinyML + - Machine Learning + - Microcontroller + - Embedded Systems +authors: + - name: Jon Nordby + orcid: 0000-0002-0245-7456 + affiliation: "1" +affiliations: + - name: Soundsensing, Norway + index: 1 +date: 19 January 2025 +bibliography: paper.bib + +--- + +# Summary + +emlearn-micropython enables sensor data analysis on low-cost microcontrollers. +The library provides implementations of a set of algorithms commonly used for sensor data processing. +This includes Digital Signal Processing techniques such as Infinite Impulse Response and Fast Fourier Transform, +as well as Machine Learning inference such as Random Forest, Nearest Neighbors and Convolutional Neural Networks. +Any kind of sensor data can be processed, including audio, images, radar, and accelerometer data. + +The library builds on MicroPython, a tiny Python implementation designed for microcontrollers. +The modules expose a high-level Python API and are implemented in C code for computational efficiency. +This enables engineers, researchers, and makers that are familiar with Python +to build efficient embedded systems that perform automatic analysis of sensor data. +A range of hardware architectures are supported, including ARM Cortex M, Xtensa/ESP32 and x86-64. + +# Statement of need + +Over the last decade, it has become possible to create low-cost embedded devices that can automatically collect and analyze sensor data. +These devices often combine one or more MEMS sensors with a microcontroller, +and then using a combination of digital signal processing and machine learning algorithms to extract relevant information from the sensors. +The development and utilization of such systems is an active area of research +with a wide range of applications in science, industry, and consumer products [@tinyml_review_ray2022]. + +Python is among the most commonly used application languages for machine learning and data science. +Thanks to the MicroPython [@micropython] project, it has become feasible to use Python also on microcontrollers, +and this is an attractive proposition for practitioners that are familiar with Python. +However, research has identified that running computationally intensive algorithms in Python +with MicroPython can be inefficient [@plauska_performance_2023; @ionescu_investigating_2020; @dokic_micropython_2020]. +This also limits the effectiveness of tools that generate Python code, such as m2cgen [@m2cgen]. + +The library ulab [@ulab] implements efficient numeric computing facilities for MicroPython, +including the core parts of numpy [@numpy], plus some parts of scipy [@scipy]. +However, as of 2025, there are no implementations of machine learning algorithms available in ulab. + +OpenMV [@OpenMV] is a project for machine-vision/computer-vision applications using high-end microcontrollers. +They have their own distribution of MicroPython, which includes some additional DSP and ML functionality, +including image and audio classifiers based on TensorFlow Lite for Microcontrollers [@tflite_micro]. +However, their solution only officially supports the OpenMV hardware, which limits applicability. + +For these reasons, we saw a need to develop a software library for MicroPython with the following properties: +1) supports inference for common machine learning algorithms, +2) is computationally efficient (in terms of execution speed, program space, and RAM usage), +3) run on any hardware supported by MicroPython, +4) can be installed easily. + +Our goal is to make research and development in applied machine learning for embedded systems +easier for those that prefer developing in Python over conventional C or C++. +We believe that this is attractive for many researchers and engineers. It may also be relevant in an educational context. + +Within one year of the first release, +emlearn-micropython was referenced in a work for on-device learning of decision trees [@karavaev2024tinydecisiontreeclassifier]. + +# Package contents + +The emlearn-micropython software package provides a selection of machine learning inference algorithms, +along with some functions for digital signal processing. +The algorithms have been selected based on what is useful and commonly used in embedded systems for processing sensor data. +The implementations are designed to be compatible with established packages, +notably scikit-learn [@scikit-learn], Keras [@keras] and scipy [@scipy]. +\autoref{table_emlearn_micropython_modules} provides a listing of the provided functionality. + +The software is distributed as MicroPython native modules [@micropython_native_module], +which can be installed at runtime using the `mip` package manager. +The modules provided by emlearn-micropython are independent of each other, and typically a few kilobytes large. +This makes it easy to install just what is needed for a particular application, +and to fit in the limited program memory provided by the target microcontroller. + + +| Module | Description | Corresponds to | +|:-------------------|:-------------------------------------|:----------------------------------| +| emlearn_trees | Decision tree ensembles | sklearn RandomForestClassifier | +| emlearn_neighbors | Nearest Neighbors | sklearn KNeighborsClassifier | +| emlearn_cnn | Convolutional Neural Network | keras Model+Conv2D | +| emlearn_linreg | Linear Regression | sklearn ElasticNet | +| emlearn_fft | Fast Fourier Transform | scipy.fft.fft | +| emlearn_iir | Infinite Impulse Response filters | scipy.signal.sosfilt | +| emlearn_arrayutils | Fast utilities for array.array | N/A | + +Table: Overview of modules provided by emlearn-micropython \label{table_emlearn_micropython_modules} + +Most of the modules are implemented as wrappers of functionality provided in the emlearn C library [@emlearn]. +However, the emlearn_cnn module is implemented using the TinyMaix library [@TinyMaix]. + +# Usage example + +As an illustrative example of sensor data analysis with emlearn-micropython, +we show how data from an accelerometer can be used to recognize human activities. +This can, for example, be deployed in a fitness bracelet or smartphone. + +Example data is taken from the PAMAP2 Physical Activity Monitoring dataset [@pamap2_dataset]. +The tri-axial data stream from the wrist-mounted accelerometer is split into consecutive fixed-length windows. +Each window is then processed using Fast Fourier Transform (with `emlearn_fft`), +to extract the energy at frequencies characteristic of human activities (typically below 10 Hz). +These features are then classified using a Random Forest Classifier (with `emlearn_trees`). +A running median filter is applied to the predictions to smooth out noise. +The data at these processing stages is shown in \autoref{fig:physical_activity_recognition_pipeline}. + +![Data pipeline for recognizing physical activities from accelerometer data using emlearn-micropython. Top plot shows input data from the 3-axis accelerometer. Middle plots show extracted features. The bottom plot slows the output probabilities from the classification model. The colored sections indicate the labeled activity (ground-truth).](physical_activity_recognition_pipeline.png){#fig:physical_activity_recognition_pipeline width=100% } + +The emlearn-micropython documentation +contains complete example code for Human Activity Recognition, image classification, and more. +The documentation can be found at + +# Acknowledgements + +We would like to thank +Volodymyr Shymanskyy for his work on improving native module support in MicroPython, +Damien P. George and Alessandro Gatti for fixes to native modules, +and Jeremy Meyer for user testing of the emlearn_cnn module. + +Soundsensing has received financial support in the period +from the Research Council of Norway for the EARONEDGE project (project code 337085). + +# References + diff --git a/paper/physical_activity_recognition_pipeline.png b/paper/physical_activity_recognition_pipeline.png new file mode 100644 index 0000000..6753a73 Binary files /dev/null and b/paper/physical_activity_recognition_pipeline.png differ diff --git a/paper/plotting.py b/paper/plotting.py new file mode 100644 index 0000000..7a4ff66 --- /dev/null +++ b/paper/plotting.py @@ -0,0 +1,314 @@ + +import numpy +import pandas + +import plotly.express as px +import plotly.graph_objects as go + +from plotly.colors import qualitative + +from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler + +def find_runs(labels : pandas.Series): + + # Detect changes + change_points = labels != labels.shift() + run_ids = change_points.cumsum() + + # Extract runs + def foo(g): + #print(g.iloc[0], g.index[0], g.index[-1]) + out = pandas.Series({ + 'label': g.iloc[0].activity, + 'start_time': g.index[0], + 'end_time': g.index[-1], + }) + return out + runs = labels.to_frame().groupby(run_ids, as_index=False).apply(foo) + return runs + +def get_subplot_axes(rows, cols, row, col): + subplot_num = (row - 1) * cols + col + if subplot_num == 1: + return 'x', 'y' + else: + return f'x{subplot_num}', f'y{subplot_num}' + +def add_events(fig, df, label_colors, subplot={}, cols=0, font_size=16, font_family='sans-serif'): + + xaxis, yaxis = get_subplot_axes(fig, cols=cols, **subplot) + print('add-events', subplot, yaxis) + + # Plot rectangles + for _, row in df.iterrows(): + fig.add_shape( + type='rect', + x0=row['start_time'], + x1=row['end_time'], + y0=0, + y1=1, + xref='x', + yref=f'{yaxis} domain', + fillcolor=label_colors[row['label']], + opacity=0.3, + line_width=0, + layer='below', + **subplot, + ) + + # Optional label annotation + fig.add_annotation( + x=(row['start_time'] + (row['end_time'] - row['start_time']) / 2), + y=1.02, + text=row['label'], + showarrow=False, + xref='x', + yref=f'{yaxis} domain', + font=dict(size=font_size, family=font_family), + **subplot, + ) + +def time_ticks(times, every=30, skip_start=0): + n_times = len(times) + start = times.min() + skip_start + end = times.max() + + def minute_second_format(t : float): + """MM:SS""" + return f"{int(t//60):02}:{int(t%60):02}" + + tick_vals = numpy.arange(start, end, every) + tick_text = [ minute_second_format(t) for t in tick_vals] + + return tick_vals, tick_text + +def make_label_colors(labels) -> dict[str, object]: + + color_pool = qualitative.Set3 + qualitative.Dark24 + qualitative.Pastel1 + label_colors = dict(zip(labels, color_pool)) + assert len(labels) <= len(color_pool) + + return label_colors + +def plot_timeline(fig, df, + data: list[str], + label_colors=None, + data_colors=None, + time='time', + label='activity', + subplot={}, + cols=1, + opacity=0.5, + ): + + df = df.reset_index() + df[time] = convert_times(df[time]) + df = df.sort_values(time) + + if label_colors is None: + label_colors = make_label_colors(df[label].unique()) + + if data_colors is None: + data_colors = make_label_colors(list(set(data))) + + if label is not None: + df = df.set_index(time) + events = find_runs(df[label]) + events = events[~events.label.isin(['transient'])] + df = df.reset_index() + add_events(fig, events, label_colors=label_colors, subplot=subplot, cols=cols) + + # Add each axis as a line + for column in data: + y = df[column] + #print(column) + #print(df[time]) + trace = go.Scatter(x=df[time], + y=y, + mode='lines', + name=column, + line=dict(color=data_colors[column]), + opacity=opacity, + ) + fig.add_trace(trace, **subplot) + +def convert_times(times): + out = times + out = out / pandas.Timedelta(seconds=1) + out -= out.min() + return out + +def configure_xaxis(fig, times, every=60, col=1, row=1, standoff=10): + + times = convert_times(times) + + tick_vals, tick_text = time_ticks(times, skip_start=30) + + # Customize layout + fig.update_xaxes( + tickmode='array', + tickvals=tick_vals, + ticktext=tick_text, + col=col, row=row, + ) + + fig.add_annotation( + text="Time (MM:SS)", + x=0, # Left edge of plot area + y=-0.035, # Below the plot (negative moves down) + xref="x domain", # Relative to subplot domain + yref="paper", # Relative to entire figure + xanchor="left", # Left-align text + yanchor="top", # Anchor to top of text + showarrow=False, + font=dict(size=14) + ) + + +def plot_heatmap(fig, data, columns, y_labels=None, colorscale='RdBu', time='time', zmax=1.0, zmin=None, subplot={}): + + if zmin is None: + zmin = -zmax + + from plotly import graph_objects as go + + if y_labels is None: + y_labels = columns + + data[time] = convert_times(data[time]) + + x_labels = data[time] + z_data = data[columns].T.values # Transpose for proper orientation + + # Create heatmap + fig.add_heatmap( + z=z_data, + x=x_labels, + y=y_labels, + colorscale=colorscale, + showscale=False, + zmin=zmin, + zmax=zmax, + **subplot, + ) + + return fig + + +def fft_freq_from_bin(bin_idx, length, sample_rate): + """Convert FFT bin index to frequency""" + return bin_idx * sample_rate / length + +def make_timeline_plot(data, features, predictions, + colors, + class_names, + width = 1600, + aspect = 2.0, + ): + + from plotly.subplots import make_subplots + from plotting import configure_xaxis, plot_heatmap + + # Create subplots + rows = 4 + fig = make_subplots( + rows=rows, cols=1, + row_titles=('Sensor input', 'FFT', 'Energy', 'Predictions'), + shared_xaxes=True, + vertical_spacing=0.02, + row_heights=(0.4, 0.4, 0.10, 0.4), + ) + subplots = [ dict(col=1, row=i) for i in range(1, rows+1) ] + + # Input data + sensor_columns = ['hand_acceleration_6g_x', 'hand_acceleration_6g_y', 'hand_acceleration_6g_z'] + sensor_colors = dict(zip(sensor_columns, ['red', 'green', 'blue'])) + scaled_data = data.copy() + scaled_data.loc[:, sensor_columns] = scaled_data.loc[:, sensor_columns] / 10.0 # XXX: guessing appropriate scaling factor + plot_timeline(fig, scaled_data, data=sensor_columns, + label='activity', + label_colors=colors, + data_colors=sensor_colors, + subplot=subplots[0] + ) + fig.update_yaxes(range=(-5, 5), **subplots[0], title_text="Acceleration (g)") + + # Features + heatmap_colorscale = 'blues' + feature_columns = ['peak2peak', 'fft_energy'] + feature_names = ['P2P', 'FFT'] + #plot_timeline(fig, sub, data=['peak2peak', 'fft_energy'], label=None, subplot=subplots[1]) + scaler = RobustScaler(quantile_range=(5.0, 95.0), with_centering=False) + scaler.set_output(transform='pandas') + features_scaled = scaler.fit_transform(features[feature_columns]) + plot_heatmap(fig, + features_scaled.reset_index(), + columns=feature_columns, + y_labels=feature_names, + subplot=subplots[2], + zmin=0.0, zmax=3.0, + colorscale=heatmap_colorscale, + ) + print(features_scaled.describe()) + # FFT + fft_columns = list(features.columns[features.columns.str.contains('fft.', regex=False)]) + scaler = RobustScaler(quantile_range=(5.0, 95.0), with_centering=False) + scaler.set_output(transform='pandas') + fft_scaled = scaler.fit_transform(features[fft_columns]) / 3.0 + fft_length = 256 + samplerate = 100 + fft_frequencies = numpy.array([ fft_freq_from_bin(int(c.removeprefix('fft.')), fft_length, samplerate) for c in fft_columns ]).round(1) + print(fft_frequencies) + plot_heatmap(fig, + fft_scaled.reset_index(), + columns=fft_columns, + y_labels=fft_frequencies, + zmin=0.0, zmax=0.8, + subplot=subplots[1], + colorscale=heatmap_colorscale, + ) + fig.update_yaxes(title_text="Hz", **subplots[1]) + + # Predictions + plot_timeline(fig, predictions, data=class_names, + label='activity', + label_colors=colors, + data_colors=colors, + opacity=1.0, + subplot=subplots[3]) + fig.update_yaxes(range=(0, 1.0), title_text="Probability", **subplots[3]) + + + configure_xaxis(fig, data.index, every=60, row=4) + + # Customize layout + fig.update_layout( + template='plotly_white', + ) + + fig.update_layout( + legend=dict( + orientation="h", + yanchor="top", + y=-0.03, # Adjust distance from plot + xanchor="center", + x=0.5, + font=dict(size=10), # Smaller font if needed + itemwidth=30 # Control item spacing + ), + margin=dict(b=30) # Add bottom margin for legend space + ) + + fig.update_yaxes(tickformat=".1f") + + height = int(width / aspect) + fig.update_layout( + width=width, + height=height, + margin=dict(l=10, r=10, t=10, b=10), # Fixed margins + autosize=False + ) + + return fig + + diff --git a/paper/processing.py b/paper/processing.py new file mode 100644 index 0000000..493d8a3 --- /dev/null +++ b/paper/processing.py @@ -0,0 +1,93 @@ +import os +import subprocess +import tempfile + +import pandas +import numpy + + +def convert_to_raw(df: pandas.DataFrame, + in_max=60.0, + out_max=(2**15)-1, + dtype=numpy.int16): + + missing = df.isna().any().sum() / len(df) + print('missing values', 100*missing, '%') + df = df.fillna(0.0) + scaled = ((df / in_max) * out_max).clip(-out_max, out_max) + out = scaled.astype(dtype) + + return out + +def make_label_track(times, events, + label_column='label', default=None): + + out = pandas.Series(numpy.repeat(default, len(times)), index=times) + for idx, event in events.iterrows(): + s = event['start_time'] + e = event['end_time'] + out.loc[s:e] = event[label_column] + + return out + +def process_data(df : pandas.DataFrame, + micropython_bin='micropython', + samplerate = 100, + window_length = 256, + fft_start = 0, + fft_end = 16, + model_path = None, + classes = [], + ): + + fft_features = fft_end - fft_start + + here = os.path.dirname(__file__) + + hop_length = window_length + time_resolution = pandas.Timedelta(seconds=1)/samplerate * hop_length + print(time_resolution) + + # check input + data = numpy.ascontiguousarray(df.values) + assert len(data.shape) == 2, data.shape + assert data.shape[1] == 3, data.shape + assert data.dtype == numpy.int16, data.dtype + assert data.flags['C_CONTIGUOUS'], data.flags + + times = df.index + + with tempfile.TemporaryDirectory() as temp: + temp = '' + in_path = os.path.join(temp, 'input.npy') + out_path = os.path.join(temp, 'output.npy') + + # write input data + numpy.save(in_path, data, allow_pickle=False) + + # run the program, as subprocess + script = os.path.join(here, 'example.py') + args = [ + micropython_bin, + script, + in_path, + out_path, + ] + if model_path is not None: + args.append(model_path) + + cmd = ' '.join(args) + print('run-cmd', cmd) + subprocess.check_output(args) + + # load output data + assert os.path.exists(out_path) + out = numpy.load(out_path) + + fft_columns = [ f'fft.{i}' for i in range(fft_features) ] + columns = ['peak2peak', 'fft_energy'] + fft_columns + classes + df = pandas.DataFrame(out, columns=columns) + + df['time'] = times.min() + (time_resolution * numpy.arange(0, len(df))) + df = df.set_index('time') + return df diff --git a/prepare_california.py b/prepare_california.py new file mode 100644 index 0000000..264df83 --- /dev/null +++ b/prepare_california.py @@ -0,0 +1,76 @@ + +#!/usr/bin/env python3 +""" +Download and preprocess California housing dataset for MicroPython testing. +Saves scaled train/test splits as .npy files. +""" + +import numpy as np +from sklearn.datasets import fetch_california_housing +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler + +def prepare_california_housing_data(): + """Download, preprocess and save California housing dataset.""" + + print("Downloading California housing dataset...") + # Load the dataset + housing = fetch_california_housing() + X, y = housing.data, housing.target + + print(f"Dataset shape: X={X.shape}, y={y.shape}") + print(f"Features: {housing.feature_names}") + print(f"Target: median house value in hundreds of thousands of dollars") + + # Split into train/test (80/20) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=42 + ) + + print(f"Train set: X={X_train.shape}, y={y_train.shape}") + print(f"Test set: X={X_test.shape}, y={y_test.shape}") + + # Scale the features (standardization) + scaler = StandardScaler() + X_train_scaled = scaler.fit_transform(X_train) + X_test_scaled = scaler.transform(X_test) + + print("\nScaling applied:") + print(f"Feature means: {scaler.mean_}") + print(f"Feature stds: {scaler.scale_}") + + # Convert to float32 for MicroPython compatibility + X_train_scaled = X_train_scaled.astype(np.float32) + X_test_scaled = X_test_scaled.astype(np.float32) + y_train = y_train.astype(np.float32) + y_test = y_test.astype(np.float32) + + # Save as .npy files + np.save('X_train.npy', X_train_scaled) + np.save('X_test.npy', X_test_scaled) + np.save('y_train.npy', y_train) + np.save('y_test.npy', y_test) + + print("\nSaved files:") + print(f"X_train.npy: {X_train_scaled.shape} float32") + print(f"X_test.npy: {X_test_scaled.shape} float32") + print(f"y_train.npy: {y_train.shape} float32") + print(f"y_test.npy: {y_test.shape} float32") + + # Print some statistics for verification + print("\nData statistics:") + print(f"X_train range: [{X_train_scaled.min():.3f}, {X_train_scaled.max():.3f}]") + print(f"y_train range: [{y_train.min():.3f}, {y_train.max():.3f}]") + print(f"y_train mean: {y_train.mean():.3f}") + + return X_train_scaled, X_test_scaled, y_train, y_test + + + +if __name__ == "__main__": + # Prepare the data + X_train, X_test, y_train, y_test = prepare_california_housing_data() + + print("\nData preparation complete!") + print("Files ready for MicroPython testing:") + print("- X_train.npy, X_test.npy, y_train.npy, y_test.npy") diff --git a/requirements.txt b/requirements.txt index 3a89384..9c11383 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ emlearn>=0.21.2 scikit-learn>=1.0.0 ar>=1.0.0 pyelftools>=0.31 +setuptools>=71.0.0 diff --git a/src/emlearn_iir_q15/iir_filter.c b/src/emlearn_iir_q15/iir_filter.c index a918029..382fa2d 100644 --- a/src/emlearn_iir_q15/iir_filter.c +++ b/src/emlearn_iir_q15/iir_filter.c @@ -5,8 +5,9 @@ #include -// memset is used by some standard C constructs -#if !defined(__linux__) +// memset is used by some standard C constructs. Missing on some platforms, but not all... +// Unfortunately cannot use weak symbols with mpy_ld +#if !(defined(__linux__) || defined(__riscv__) || defined(__riscv)) void *memcpy(void *dst, const void *src, size_t n) { return mp_fun_table.memmove_(dst, src, n); } diff --git a/src/emlearn_linreg/Makefile b/src/emlearn_linreg/Makefile new file mode 100644 index 0000000..686c587 --- /dev/null +++ b/src/emlearn_linreg/Makefile @@ -0,0 +1,37 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../micropython + +# Architecture to build for (x86, x64, armv6m, armv7m, xtensa, xtensawin) +ARCH = x64 + +# The ABI version for .mpy files +MPY_ABI_VERSION := 6.3 + +# Location of emlearn library +EMLEARN_DIR := $(shell python3 -c "import emlearn; print(emlearn.includedir)") + +# enable linking of libm etc +LINK_RUNTIME=1 + +DIST_DIR := ../../dist/$(ARCH)_$(MPY_ABI_VERSION) + +# Name of module +MOD = emlearn_linreg + +# Source files (.c or .py) +SRC = linreg.c linreg.py + +# Include to get the rules for compiling and linking the module +include $(MPY_DIR)/py/dynruntime.mk + +# Releases +DIST_FILE = $(DIST_DIR)/$(MOD).mpy +$(DIST_DIR): + mkdir -p $@ + +$(DIST_FILE): $(MOD).mpy $(DIST_DIR) + cp $< $@ + +CFLAGS += -I$(EMLEARN_DIR) -Wno-unused-function + +dist: $(DIST_FILE) diff --git a/src/emlearn_linreg/eml_linreg.c b/src/emlearn_linreg/eml_linreg.c new file mode 100644 index 0000000..69da4ea --- /dev/null +++ b/src/emlearn_linreg/eml_linreg.c @@ -0,0 +1,96 @@ + +#include + +// ElasticNet implementation (embedded from linreg.c) +typedef struct { + float* weights; + float* weight_gradients; + float bias; + uint16_t n_features; + float l1_ratio; + float alpha; + float learning_rate; +} elastic_net_model_t; + +// Soft thresholding function for L1 penalty +static float soft_threshold(float x, float threshold) { + if (x > threshold) { + return x - threshold; + } else if (x < -threshold) { + return x + threshold; + } else { + return 0.0f; + } +} + +// Calculate prediction for a single sample +static float predict_sample(const elastic_net_model_t* model, const float* features) { + float prediction = model->bias; + for (uint16_t i = 0; i < model->n_features; i++) { + prediction += model->weights[i] * features[i]; + } + return prediction; +} + +// Single iteration of gradient descent +static void elastic_net_iterate(elastic_net_model_t* model, + const float* X, + const float* y, + uint16_t n_samples) { + + // Initialize gradients buffer to zero + memset(model->weight_gradients, 0, model->n_features * sizeof(float)); + float bias_gradient = 0.0f; + + // Forward pass and gradient calculation + for (uint16_t i = 0; i < n_samples; i++) { + // Calculate prediction + float prediction = predict_sample(model, &X[i * model->n_features]); + + // Calculate error + float error = prediction - y[i]; + + // Accumulate gradients + bias_gradient += error; + for (uint16_t j = 0; j < model->n_features; j++) { + model->weight_gradients[j] += error * X[i * model->n_features + j]; + } + } + + // Average gradients + bias_gradient /= n_samples; + for (uint16_t j = 0; j < model->n_features; j++) { + model->weight_gradients[j] /= n_samples; + } + + // Update weights with regularization + for (uint16_t j = 0; j < model->n_features; j++) { + // Add L2 penalty to gradient + float l2_penalty = model->alpha * (1.0f - model->l1_ratio) * model->weights[j]; + + // Update weight + float new_weight = model->weights[j] - model->learning_rate * (model->weight_gradients[j] + l2_penalty); + + // Apply L1 penalty via soft thresholding + float l1_penalty = model->alpha * model->l1_ratio * model->learning_rate; + model->weights[j] = soft_threshold(new_weight, l1_penalty); + } + + // Update bias (no regularization on bias) + model->bias -= model->learning_rate * bias_gradient; +} + +// Calculate mean squared error +static float elastic_net_mse(const elastic_net_model_t* model, + const float* X, + const float* y, + uint16_t n_samples) { + + float mse = 0.0f; + for (uint16_t i = 0; i < n_samples; i++) { + float prediction = predict_sample(model, &X[i * model->n_features]); + float error = y[i] - prediction; + mse += error * error; + } + return mse / n_samples; +} diff --git a/src/emlearn_linreg/linreg.c b/src/emlearn_linreg/linreg.c new file mode 100644 index 0000000..1bb4a5e --- /dev/null +++ b/src/emlearn_linreg/linreg.c @@ -0,0 +1,303 @@ +// Include the header file to get access to the MicroPython API +#include "py/dynruntime.h" + +#include + +#include "eml_linreg.c" + +// memset/memcpy for compatibility +#if !defined(__linux__) +void *memcpy(void *dst, const void *src, size_t n) { + return mp_fun_table.memmove_(dst, src, n); +} +void *memset(void *s, int c, size_t n) { + return mp_fun_table.memset_(s, c, n); +} +#endif + +// MicroPython type for ElasticNet model +typedef struct _mp_obj_elasticnet_model_t { + mp_obj_base_t base; + elastic_net_model_t model; +} mp_obj_elasticnet_model_t; + +mp_obj_full_type_t elasticnet_model_type; + +// Create a new instance +static mp_obj_t elasticnet_model_new(size_t n_args, const mp_obj_t *args) { + // Args: n_features, alpha, l1_ratio, learning_rate + if (n_args != 4) { + mp_raise_ValueError(MP_ERROR_TEXT("Expected 4 arguments: n_features, alpha, l1_ratio, learning_rate")); + } + + mp_int_t n_features = mp_obj_get_int(args[0]); + float alpha = mp_obj_get_float_to_f(args[1]); + float l1_ratio = mp_obj_get_float_to_f(args[2]); + float learning_rate = mp_obj_get_float_to_f(args[3]); + + // Allocate space + mp_obj_elasticnet_model_t *o = \ + mp_obj_malloc(mp_obj_elasticnet_model_t, (mp_obj_type_t *)&elasticnet_model_type); + + elastic_net_model_t *self = &o->model; + memset(self, 0, sizeof(elastic_net_model_t)); + + // Configure model + self->n_features = n_features; + self->alpha = alpha; + self->l1_ratio = l1_ratio; + self->learning_rate = learning_rate; + self->bias = 0.0f; + + // Allocate weight buffers + self->weights = (float *)m_malloc(sizeof(float) * n_features); + self->weight_gradients = (float *)m_malloc(sizeof(float) * n_features); + + // Initialize weights to zero + memset(self->weights, 0, n_features * sizeof(float)); + + return MP_OBJ_FROM_PTR(o); +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(elasticnet_model_new_obj, 4, 4, elasticnet_model_new); + +// Delete an instance +static mp_obj_t elasticnet_model_del(mp_obj_t self_obj) { + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(self_obj); + elastic_net_model_t *self = &o->model; + + // Free allocated memory + m_free(self->weights); + m_free(self->weight_gradients); + + return mp_const_none; +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_1(elasticnet_model_del_obj, elasticnet_model_del); + +// Single training iteration +static mp_obj_t elasticnet_model_step(size_t n_args, const mp_obj_t *args) { + // Args: self, X, y + if (n_args != 3) { + mp_raise_ValueError(MP_ERROR_TEXT("Expected 3 arguments: self, X, y")); + } + + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(args[0]); + elastic_net_model_t *self = &o->model; + + // Extract X buffer + mp_buffer_info_t X_bufinfo; + mp_get_buffer_raise(args[1], &X_bufinfo, MP_BUFFER_READ); + if (X_bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("X expecting float32 array")); + } + const float *X = X_bufinfo.buf; + const int X_len = X_bufinfo.len / sizeof(float); + + // Extract y buffer + mp_buffer_info_t y_bufinfo; + mp_get_buffer_raise(args[2], &y_bufinfo, MP_BUFFER_READ); + if (y_bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("y expecting float32 array")); + } + const float *y = y_bufinfo.buf; + const int y_len = y_bufinfo.len / sizeof(float); + + // Validate dimensions + if (X_len != y_len * self->n_features) { + mp_raise_ValueError(MP_ERROR_TEXT("X and y dimensions don't match")); + } + + const uint16_t n_samples = y_len; + + // Perform single iteration + elastic_net_iterate(self, X, y, n_samples); + + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(elasticnet_model_step_obj, 3, 3, elasticnet_model_step); + +// Predict using the model +static mp_obj_t elasticnet_model_predict(mp_obj_fun_bc_t *self_obj, + size_t n_args, size_t n_kw, mp_obj_t *args) { + // Check number of arguments is valid + mp_arg_check_num(n_args, n_kw, 2, 2, false); + + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(args[0]); + elastic_net_model_t *self = &o->model; + + // Extract buffer pointer and verify typecode + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); + if (bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("expecting float32 array")); + } + const float *features = bufinfo.buf; + const int n_features = bufinfo.len / sizeof(float); + + if (n_features != self->n_features) { + mp_raise_ValueError(MP_ERROR_TEXT("Feature count mismatch")); + } + + // Make prediction + float prediction = predict_sample(self, features); + + return mp_obj_new_float_from_f(prediction); +} + +// Get model weights +static mp_obj_t elasticnet_model_get_weights(mp_obj_t self_obj, mp_obj_t out_obj) { + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(self_obj); + elastic_net_model_t *self = &o->model; + + // Extract buffer pointer and verify typecode + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(out_obj, &bufinfo, MP_BUFFER_WRITE); + if (bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("expecting float32 array")); + } + float *weights = bufinfo.buf; + const int n_features = bufinfo.len / sizeof(float); + + if (n_features != self->n_features) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer is wrong size")); + } + + memcpy(weights, self->weights, sizeof(float) * n_features); + + return mp_const_none; +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_2(elasticnet_model_get_weights_obj, elasticnet_model_get_weights); + +// Get number of features +static mp_obj_t elasticnet_model_get_n_features(mp_obj_t self_obj) { + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(self_obj); + elastic_net_model_t *self = &o->model; + + return mp_obj_new_int(self->n_features); +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_1(elasticnet_model_get_n_features_obj, elasticnet_model_get_n_features); + +// Get model bias +static mp_obj_t elasticnet_model_get_bias(mp_obj_t self_obj) { + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(self_obj); + elastic_net_model_t *self = &o->model; + + return mp_obj_new_float_from_f(self->bias); +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_1(elasticnet_model_get_bias_obj, elasticnet_model_get_bias); + + +// Set model bias +static mp_obj_t elasticnet_model_set_bias(mp_obj_t self_obj, mp_obj_t bias_obj) { + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(self_obj); + elastic_net_model_t *self = &o->model; + + float bias = mp_obj_get_float_to_f(bias_obj); + self->bias = bias; + + return mp_const_none; +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_2(elasticnet_model_set_bias_obj, elasticnet_model_set_bias); + +// Set model weights +static mp_obj_t elasticnet_model_set_weights(mp_obj_t self_obj, mp_obj_t weights_obj) { + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(self_obj); + elastic_net_model_t *self = &o->model; + + // Extract buffer pointer and verify typecode + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(weights_obj, &bufinfo, MP_BUFFER_READ); + if (bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("expecting float32 array")); + } + const float *weights = bufinfo.buf; + const int n_features = bufinfo.len / sizeof(float); + + if (n_features != self->n_features) { + mp_raise_ValueError(MP_ERROR_TEXT("Weight array size mismatch")); + } + + memcpy(self->weights, weights, sizeof(float) * n_features); + + return mp_const_none; +} +// Define a Python reference to the function above +static MP_DEFINE_CONST_FUN_OBJ_2(elasticnet_model_set_weights_obj, elasticnet_model_set_weights); + + +// Calculate MSE from X and y (saves memory by not storing predictions) +static mp_obj_t elasticnet_model_score_mse(size_t n_args, const mp_obj_t *args) { + // Args: self, X, y + if (n_args != 3) { + mp_raise_ValueError(MP_ERROR_TEXT("Expected 3 arguments: self, X, y")); + } + + mp_obj_elasticnet_model_t *o = MP_OBJ_TO_PTR(args[0]); + elastic_net_model_t *self = &o->model; + + // Extract X buffer + mp_buffer_info_t X_bufinfo; + mp_get_buffer_raise(args[1], &X_bufinfo, MP_BUFFER_READ); + if (X_bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("X expecting float32 array")); + } + const float *X = X_bufinfo.buf; + const int X_len = X_bufinfo.len / sizeof(float); + + // Extract y buffer + mp_buffer_info_t y_bufinfo; + mp_get_buffer_raise(args[2], &y_bufinfo, MP_BUFFER_READ); + if (y_bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("y expecting float32 array")); + } + const float *y = y_bufinfo.buf; + const int y_len = y_bufinfo.len / sizeof(float); + + // Validate dimensions + if (X_len != y_len * self->n_features) { + mp_raise_ValueError(MP_ERROR_TEXT("X and y dimensions don't match")); + } + + const uint16_t n_samples = y_len; + float mse = elastic_net_mse(self, X, y, n_samples); + + return mp_obj_new_float_from_f(mse); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(elasticnet_model_score_mse_obj, 3, 3, elasticnet_model_score_mse); + +// Module setup +mp_map_elem_t elasticnet_model_locals_dict_table[10]; +static MP_DEFINE_CONST_DICT(elasticnet_model_locals_dict, elasticnet_model_locals_dict_table); + +// Module setup entrypoint +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // This must be first, it sets up the globals dict and other things + MP_DYNRUNTIME_INIT_ENTRY + + mp_store_global(MP_QSTR_new, MP_OBJ_FROM_PTR(&elasticnet_model_new_obj)); + + elasticnet_model_type.base.type = (void*)&mp_fun_table.type_type; + elasticnet_model_type.flags = MP_TYPE_FLAG_ITER_IS_CUSTOM; + elasticnet_model_type.name = MP_QSTR_elasticnet; + + // methods + elasticnet_model_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_predict), MP_DYNRUNTIME_MAKE_FUNCTION(elasticnet_model_predict) }; + elasticnet_model_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_step), MP_OBJ_FROM_PTR(&elasticnet_model_step_obj) }; + elasticnet_model_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___del__), MP_OBJ_FROM_PTR(&elasticnet_model_del_obj) }; + elasticnet_model_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get_weights), MP_OBJ_FROM_PTR(&elasticnet_model_get_weights_obj) }; + elasticnet_model_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get_bias), MP_OBJ_FROM_PTR(&elasticnet_model_get_bias_obj) }; + elasticnet_model_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get_n_features), MP_OBJ_FROM_PTR(&elasticnet_model_get_n_features_obj) }; + elasticnet_model_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_score_mse), MP_OBJ_FROM_PTR(&elasticnet_model_score_mse_obj) }; + elasticnet_model_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_set_bias), MP_OBJ_FROM_PTR(&elasticnet_model_set_bias_obj) }; + elasticnet_model_locals_dict_table[8] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_set_weights), MP_OBJ_FROM_PTR(&elasticnet_model_set_weights_obj) }; + + MP_OBJ_TYPE_SET_SLOT(&elasticnet_model_type, locals_dict, (void*)&elasticnet_model_locals_dict, 9); + + // This must be last, it restores the globals dict + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/src/emlearn_linreg/linreg.py b/src/emlearn_linreg/linreg.py new file mode 100644 index 0000000..1f18736 --- /dev/null +++ b/src/emlearn_linreg/linreg.py @@ -0,0 +1,55 @@ + + +log_prefix = 'emlearn_linreg:' + +def train(model, X_train, y_train, + max_iterations=100, + tolerance=1e-6, + check_interval=10, + divergence_factor=10.0, + score_limit=None, + verbose=0, + ): + """ + Simple training loop + """ + prev_mse = float('inf') + + for iteration in range(max_iterations): + # Perform one gradient descent step + model.step(X_train, y_train) + + # Only check progress at intervals + if iteration % check_interval != 0: + continue + + # Calculate current MSE + current_mse = model.score_mse(X_train, y_train) + change = abs(prev_mse - current_mse) + + if verbose >= 2: + print(log_prefix, f'Iteration {iteration} mse={current_mse}') + + # Check convergence + converged = change < tolerance and iteration > check_interval * 2 + + if score_limit is not None: + converged = converged or current_mse <= score_limit + + # Check divergence + diverged = current_mse > prev_mse * divergence_factor or not (current_mse == current_mse) # NaN check + + if converged: + if verbose >= 1: + print(log_prefix, f"Converged at iteration {iteration}") + break + + if diverged: + if verbose >= 1: + print(log_prefix, f"Diverged at iteration {iteration}") + break + + prev_mse = current_mse + + return iteration, prev_mse + diff --git a/src/emlearn_trees/trees.c b/src/emlearn_trees/trees.c index 6e13a30..c7a49bd 100644 --- a/src/emlearn_trees/trees.c +++ b/src/emlearn_trees/trees.c @@ -15,7 +15,7 @@ #ifdef MICROPY_ENABLE_DYNRUNTIME // memset is used by some standard C constructs -#if !defined(__linux__) +#if !defined(__linux__) && !defined(__APPLE__) void *memcpy(void *dst, const void *src, size_t n) { return mp_fun_table.memmove_(dst, src, n); } diff --git a/test_california_sklearn.py b/test_california_sklearn.py new file mode 100644 index 0000000..2ef5fd2 --- /dev/null +++ b/test_california_sklearn.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +""" +Reference implementation using scikit-learn ElasticNet for comparison with MicroPython module. +Loads the same .npy files and computes MSE for benchmarking. +""" + +import numpy as np +from sklearn.linear_model import ElasticNet +from sklearn.metrics import mean_squared_error, r2_score +import time + +def load_data(): + """Load the preprocessed California housing data.""" + print("Loading data...") + X_train = np.load('X_train.npy') + X_test = np.load('X_test.npy') + y_train = np.load('y_train.npy') + y_test = np.load('y_test.npy') + + print(f"Train set: X={X_train.shape}, y={y_train.shape}") + print(f"Test set: X={X_test.shape}, y={y_test.shape}") + print(f"Data types: X={X_train.dtype}, y={y_train.dtype}") + + return X_train, X_test, y_train, y_test + +def test_elasticnet_configurations(): + """Test different ElasticNet configurations to find good baselines.""" + + X_train, X_test, y_train, y_test = load_data() + + # Test configurations: (alpha, l1_ratio, description) + configs = [ + (0.0, 0.0, "No regularization (OLS)"), + (0.01, 0.0, "Ridge (alpha=0.01)"), + (0.01, 1.0, "LASSO (alpha=0.01)"), + (0.01, 0.5, "ElasticNet (alpha=0.01, l1_ratio=0.5)"), + (0.001, 0.5, "ElasticNet (alpha=0.001, l1_ratio=0.5)"), + (0.1, 0.5, "ElasticNet (alpha=0.1, l1_ratio=0.5)"), + ] + + print("\n" + "="*70) + print("ElasticNet Configuration Comparison") + print("="*70) + print(f"{'Configuration':<35} {'Train MSE':<12} {'Test MSE':<12} {'R²':<8} {'Time':<8}") + print("-"*70) + + results = [] + + for alpha, l1_ratio, description in configs: + start_time = time.time() + + # Create and train model + if alpha == 0.0: + # Use regular linear regression for no regularization + from sklearn.linear_model import LinearRegression + model = LinearRegression() + else: + model = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, max_iter=2000, random_state=42) + + model.fit(X_train, y_train) + + # Make predictions + y_train_pred = model.predict(X_train) + y_test_pred = model.predict(X_test) + + # Calculate metrics + train_mse = mean_squared_error(y_train, y_train_pred) + test_mse = mean_squared_error(y_test, y_test_pred) + test_r2 = r2_score(y_test, y_test_pred) + + elapsed_time = time.time() - start_time + + print(f"{description:<35} {train_mse:<12.6f} {test_mse:<12.6f} {test_r2:<8.3f} {elapsed_time:<8.3f}") + + results.append({ + 'config': description, + 'alpha': alpha, + 'l1_ratio': l1_ratio, + 'train_mse': train_mse, + 'test_mse': test_mse, + 'r2': test_r2, + 'time': elapsed_time, + 'model': model + }) + + return results + +def detailed_analysis_best_model(results): + """Perform detailed analysis on the best performing model.""" + + # Find best model by test MSE + best_result = min(results, key=lambda x: x['test_mse']) + print(f"\n" + "="*50) + print("Detailed Analysis - Best Model") + print("="*50) + print(f"Best configuration: {best_result['config']}") + print(f"Alpha: {best_result['alpha']}, L1 ratio: {best_result['l1_ratio']}") + print(f"Test MSE: {best_result['test_mse']:.6f}") + print(f"Test RMSE: {np.sqrt(best_result['test_mse']):.6f}") + print(f"Test R²: {best_result['r2']:.6f}") + + model = best_result['model'] + + # Load data again for detailed analysis + X_train, X_test, y_train, y_test = load_data() + + # Show coefficients (if available) + if hasattr(model, 'coef_'): + print(f"\nModel coefficients:") + feature_names = ['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', + 'Population', 'AveOccup', 'Latitude', 'Longitude'] + for i, (name, coef) in enumerate(zip(feature_names, model.coef_)): + print(f" {name:12}: {coef:8.4f}") + + if hasattr(model, 'intercept_'): + print(f" {'Intercept':12}: {model.intercept_:8.4f}") + + # Count non-zero coefficients + non_zero = np.sum(np.abs(model.coef_) > 1e-6) + print(f"\nSparsity: {non_zero}/{len(model.coef_)} non-zero coefficients") + + # Sample predictions + print(f"\nSample predictions (first 10 test samples):") + y_test_pred = model.predict(X_test) + print(f"{'Actual':<10} {'Predicted':<10} {'Error':<10}") + print("-"*30) + for i in range(min(10, len(y_test))): + actual = y_test[i] + predicted = y_test_pred[i] + error = abs(actual - predicted) + print(f"{actual:<10.3f} {predicted:<10.3f} {error:<10.3f}") + + return best_result + +def compare_with_micropython_format(): + """Create a comparison that matches the MicroPython module format.""" + + print(f"\n" + "="*50) + print("MicroPython Module Comparison Format") + print("="*50) + + X_train, X_test, y_train, y_test = load_data() + + # Test with parameters similar to what MicroPython module might use + configs_mp = [ + (0.01, 0.5, "emlearn_linreg equivalent 1"), + (0.001, 0.5, "emlearn_linreg equivalent 2"), + (0.1, 0.5, "emlearn_linreg equivalent 3"), + ] + + for alpha, l1_ratio, description in configs_mp: + print(f"\nTesting: {description}") + print(f"Parameters: alpha={alpha}, l1_ratio={l1_ratio}") + + # Train model + model = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, max_iter=2000, random_state=42) + model.fit(X_train, y_train) + + # Test on small subset (like MicroPython test) + n_small = 100 + X_small = X_train[:n_small] + y_small = y_train[:n_small] + + small_mse = mean_squared_error(y_small, model.predict(X_small)) + full_train_mse = mean_squared_error(y_train, model.predict(X_train)) + test_mse = mean_squared_error(y_test, model.predict(X_test)) + + print(f" Small subset MSE (100 samples): {small_mse:.6f}") + print(f" Full training MSE: {full_train_mse:.6f}") + print(f" Test MSE: {test_mse:.6f}") + + # Show first sample prediction for debugging + first_pred = model.predict(X_test[:1])[0] + first_actual = y_test[0] + print(f" First test sample: actual={first_actual:.3f}, predicted={first_pred:.3f}") + + # Show learned parameters + print(f" Learned bias: {model.intercept_:.6f}") + print(f" Weight range: [{model.coef_.min():.6f}, {model.coef_.max():.6f}]") + print(f" Non-zero weights: {np.sum(np.abs(model.coef_) > 1e-6)}/{len(model.coef_)}") + +def create_reference_outputs(): + """Create reference outputs for validating MicroPython implementation.""" + + print(f"\n" + "="*50) + print("Reference Outputs for MicroPython Validation") + print("="*50) + + X_train, X_test, y_train, y_test = load_data() + + # Use specific parameters for reference + alpha, l1_ratio = 0.01, 0.5 + + print(f"Reference model: alpha={alpha}, l1_ratio={l1_ratio}") + + model = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, max_iter=2000, random_state=42) + model.fit(X_train, y_train) + + # Save reference results + reference_data = { + 'alpha': alpha, + 'l1_ratio': l1_ratio, + 'intercept': model.intercept_, + 'coefficients': model.coef_, + 'train_mse': mean_squared_error(y_train, model.predict(X_train)), + 'test_mse': mean_squared_error(y_test, model.predict(X_test)), + 'first_test_prediction': model.predict(X_test[:1])[0], + 'first_test_actual': y_test[0] + } + + print(f"Intercept: {reference_data['intercept']:.8f}") + print(f"Coefficients: {reference_data['coefficients']}") + print(f"Training MSE: {reference_data['train_mse']:.8f}") + print(f"Test MSE: {reference_data['test_mse']:.8f}") + print(f"First test prediction: {reference_data['first_test_prediction']:.8f}") + print(f"First test actual: {reference_data['first_test_actual']:.8f}") + + # Save to file for MicroPython comparison + np.savez('reference_results.npz', **reference_data) + print(f"\nReference results saved to 'reference_results.npz'") + + return reference_data + +def main(): + """Main function to run all comparisons.""" + + print("Scikit-learn ElasticNet Reference Implementation") + print("=" * 60) + + try: + # Test different configurations + results = test_elasticnet_configurations() + + # Detailed analysis of best model + best_result = detailed_analysis_best_model(results) + + # Compare with MicroPython format + compare_with_micropython_format() + + # Create reference outputs + reference_data = create_reference_outputs() + + print(f"\n" + "="*60) + print("Summary") + print("="*60) + print(f"Best overall performance: {best_result['config']}") + print(f"Best test MSE: {best_result['test_mse']:.6f}") + print(f"Target for MicroPython module: MSE < {best_result['test_mse']:.3f}") + print("\nFiles created:") + print("- reference_results.npz (for MicroPython validation)") + + except FileNotFoundError as e: + print(f"Error: {e}") + print("Make sure the .npy files are in the current directory.") + print("Run prepare_housing_data.py first to create the data files.") + except Exception as e: + print(f"Unexpected error: {e}") + import traceback + traceback.print_exc() + +if __name__ == "__main__": + main() diff --git a/tests/test_linreg.py b/tests/test_linreg.py new file mode 100644 index 0000000..d785388 --- /dev/null +++ b/tests/test_linreg.py @@ -0,0 +1,16 @@ + +import emlearn_linreg +import array + +model = emlearn_linreg.new(4, 0.1, 0.5, 0.01) + +# Training data (float32 arrays) +X = array.array('f', [1,2,3,4, 2,3,4,5]) # flattened +y = array.array('f', [10, 15]) + +# Train +emlearn_linreg.train(model, X, y, max_iterations=100, tolerance=1e-6, verbose=0) + +# Predict +prediction = model.predict(array.array('f', [1,2,3,4])) +print(prediction) diff --git a/tests/test_linreg_california.py b/tests/test_linreg_california.py new file mode 100644 index 0000000..e6766b0 --- /dev/null +++ b/tests/test_linreg_california.py @@ -0,0 +1,139 @@ +# MicroPython test script for ElasticNet module with California housing dataset +import emlearn_linreg +import npyfile +import array +import gc +import time + +def load_npy_as_array(filename, dtype='f'): + """Load .npy file and convert to MicroPython array.""" + print(f"Loading {filename}...") + shape, data = npyfile.load(filename) + print(f"Shape: {shape}, Data type: {type(data)}") + + return shape, data + + +def compare_regularization(): + """Compare different regularization settings.""" + print("\n=== Regularization Comparison ===") + + # Load small dataset for quick comparison + X_shape, X_data = load_npy_as_array('X_train.npy') + y_shape, y_data = load_npy_as_array('y_train.npy') + + n_features = X_shape[1] + n_samples = min(500, y_shape[0]) + + X_subset = array.array('f', X_data[:n_samples * n_features]) + y_subset = array.array('f', y_data[:n_samples]) + + configs = [ + ("No regularization", 0.0, 0.0), + ("Ridge (L2)", 0.01, 0.0), + ("LASSO (L1)", 0.01, 1.0), + ("ElasticNet", 0.01, 0.5), + ] + + for name, alpha, l1_ratio in configs: + print(f"\nTesting {name} (alpha={alpha}, l1_ratio={l1_ratio}):") + + model = emlearn_linreg.new(n_features, alpha, l1_ratio, 0.01) + emlearn_linreg.train(model, X_subset, y_subset, + max_iterations=100, check_interval=50, verbose=1) + + # Calculate predictions and MSE + mse = model.score_mse(X_subset, y_subset) + + weights = array.array('f', [0.0] * n_features) + model.get_weights(weights) + + # Count non-zero weights (sparsity) + non_zero = sum(1 for w in weights if abs(w) > 1e-6) + print(f" MSE: {mse:.6f}") + print(f" Non-zero weights: {non_zero}/{n_features}") + print(f" Max weight magnitude: {max(abs(w) for w in weights):.6f}") + +def test_elasticnet_full(): + """Test with full dataset.""" + print("\n=== Full Dataset Test ===") + + # Load full datasets + X_train_shape, X_train_data = load_npy_as_array('X_train.npy') + y_train_shape, y_train_data = load_npy_as_array('y_train.npy') + X_test_shape, X_test_data = load_npy_as_array('X_test.npy') + y_test_shape, y_test_data = load_npy_as_array('y_test.npy') + + n_features = X_train_shape[1] + n_train = y_train_shape[0] + n_test = y_test_shape[0] + + print(f"Train set: {n_train} samples") + print(f"Test set: {n_test} samples") + print(f"Features: {n_features}") + + # Create model with different hyperparameters for full dataset + print("Creating ElasticNet model...") + model = emlearn_linreg.new(n_features, 0.001, 0.5, 0.01) # Lower learning rate for stability + + train_start = time.ticks_ms() + # Train on full dataset + print("Training on full dataset...") + stop_iter, stop_mse = emlearn_linreg.train(model, + X_train_data, y_train_data, + max_iterations=2000, check_interval=50, + verbose=2, tolerance=0.001, score_limit=0.60, + ) + train_duration = time.ticks_diff(time.ticks_ms(), train_start) + print('Train time (ms)', train_duration, 'per iter', train_duration/stop_iter) + + # Get final parameters + weights = array.array('f', [0.0] * n_features) + model.get_weights(weights) + bias = model.get_bias() + + print(f"Final weights: {list(weights)}") + print(f"Final bias: {bias}") + + # Evaluate on train set + print("Calculating training MSE...") + train_mse = model.score_mse(X_train_data, y_train_data) + print(f"Training MSE: {train_mse}") + + # Evaluate on test set + print("Calculating test MSE...") + test_mse = model.score_mse(X_test_data, y_test_data) + print(f"Test MSE: {test_mse}") + + # Make some sample predictions + print("\nSample predictions:") + for i in range(min(5, n_test)): + start_idx = i * n_features + end_idx = start_idx + n_features + test_features = array.array('f', X_test_data[start_idx:end_idx]) + prediction = model.predict(test_features) + actual = y_test_data[i] + print(f"Sample {i}: predicted={prediction:.3f}, actual={actual:.3f}, error={abs(prediction-actual):.3f}") + + return model + +def main(): + """Main test function.""" + print("ElasticNet MicroPython Module Test") + print("==================================") + + try: + # Compare regularization approaches + #compare_regularization() + gc.collect() + + test_elasticnet_full() + print("\n=== All Tests Completed Successfully! ===") + + except Exception as e: + print(f"Error during testing: {e}") + import sys + sys.print_exception(e) + +if __name__ == "__main__": + main()