From d3f3e3ed7e13a45334a9a5aadb28e53d7cff9712 Mon Sep 17 00:00:00 2001
From: Lev Kokotov <hi@levthe.dev>
Date: Thu, 10 Aug 2023 18:04:28 -0700
Subject: [PATCH 1/2] Build everything

---
 .github/workflows/package-extension.yml       | 86 ++++++++++++++++++-
 .gitignore                                    |  4 +-
 packages/postgresml-dashboard/DEBIAN/control  |  9 ++
 packages/postgresml-dashboard/DEBIAN/postinst |  7 ++
 packages/postgresml-dashboard/DEBIAN/postrm   |  4 +
 packages/postgresml-dashboard/DEBIAN/prerm    |  4 +
 packages/postgresml-dashboard/build.sh        | 39 +++++++++
 .../etc/pgml-dashboard/.gitkeep               |  0
 .../etc/systemd/system/pgml-dashboard.service | 21 +++++
 .../postgresml-dashboard/usr/bin/.gitkeep     |  0
 .../usr/share/pgml-dashboard/.gitkeep         |  0
 .../var/lib/pgml-dashboard/.gitkeep           |  0
 packages/postgresml/postgresml/DEBIAN/control | 12 +++
 .../postgresml/postgresml/DEBIAN/postinst     | 18 ++++
 packages/postgresml/postgresml/DEBIAN/postrm  |  6 ++
 packages/postgresml/postgresml/DEBIAN/prerm   |  6 ++
 packages/postgresml/postgresml/build.sh       | 53 ++++++++++++
 .../postgresml/etc/postgresml/pgml.conf       |  2 +
 .../postgresml/var/lib/postgresml/.gitkeep    |  0
 19 files changed, 268 insertions(+), 3 deletions(-)
 create mode 100644 packages/postgresml-dashboard/DEBIAN/control
 create mode 100644 packages/postgresml-dashboard/DEBIAN/postinst
 create mode 100644 packages/postgresml-dashboard/DEBIAN/postrm
 create mode 100644 packages/postgresml-dashboard/DEBIAN/prerm
 create mode 100644 packages/postgresml-dashboard/build.sh
 create mode 100644 packages/postgresml-dashboard/etc/pgml-dashboard/.gitkeep
 create mode 100644 packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service
 create mode 100644 packages/postgresml-dashboard/usr/bin/.gitkeep
 create mode 100644 packages/postgresml-dashboard/usr/share/pgml-dashboard/.gitkeep
 create mode 100644 packages/postgresml-dashboard/var/lib/pgml-dashboard/.gitkeep
 create mode 100644 packages/postgresml/postgresml/DEBIAN/control
 create mode 100755 packages/postgresml/postgresml/DEBIAN/postinst
 create mode 100644 packages/postgresml/postgresml/DEBIAN/postrm
 create mode 100644 packages/postgresml/postgresml/DEBIAN/prerm
 create mode 100644 packages/postgresml/postgresml/build.sh
 create mode 100644 packages/postgresml/postgresml/etc/postgresml/pgml.conf
 create mode 100644 packages/postgresml/postgresml/var/lib/postgresml/.gitkeep

diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
index 142809ed8..3914932a6 100644
--- a/.github/workflows/package-extension.yml
+++ b/.github/workflows/package-extension.yml
@@ -5,10 +5,13 @@ on:
     inputs:
       packageVersion:
         default: "2.7.4"
+      pythonVersion:
+        default: "3.10"
 
 jobs:
-  build:
+  extension:
     strategy:
+      fail-fast: false # Let the other job finish
       matrix:
         os: ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-16vcpu-ubuntu-2204-arm"]
     runs-on: ${{ matrix.os }}
@@ -147,3 +150,84 @@ jobs:
             postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb \
             --codename $(lsb_release -cs)
         done
+  package:
+    needs: extension
+    strategy:
+      max-parallel: 1
+      fail-fast: false # Let the other job finish, or they can lock each other out
+      matrix:
+        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        TZ: Etc/UTC
+      run: |
+        curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
+        sudo gem install deb-s3-0.11.4.gem
+        dpkg-deb --version
+        sudo apt install -y python3-pip python3 python3-virtualenv
+    - name: Build and release package
+      env:
+        AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+      run: |
+        if [[ $(arch) == "x86_64" ]]; then
+          export ARCH=amd64
+        else
+          export ARCH=arm64
+        fi
+
+        for pg in {11..15}; do
+          bash packages/postgresml/build.sh ${pg} v${{ inputs.packageVersion }} ${{ inputs.pythonVersion }}
+
+          deb-s3 upload \
+            --lock \
+            --bucket apt.postgresml.org \
+            postgresml-${pg}-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
+            --codename $(lsb_release -cs)
+
+          rm -f postgresml-${pg}-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb 
+        done
+  dashboard:
+    strategy:
+      fail-fast: false # Let the other job finish
+      matrix:
+        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
+        
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        TZ: Etc/UTC
+      run: |
+        curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
+        sudo gem install deb-s3-0.11.4.gem
+        dpkg-deb --version
+    - name: Build and release package
+      env:
+        AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+      run: |
+        if [[ $(arch) == "x86_64" ]]; then
+          export ARCH=amd64
+        else
+          export ARCH=arm64
+        fi
+
+        bash packages/postgresml-dashboard/build.sh ${{ inputs.packageVersion }}
+
+        deb-s3 upload \
+          --lock \
+          --bucket apt.postgresml.org \
+          postgresml-dashboard-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
+          --codename $(lsb_release -cs)
diff --git a/.gitignore b/.gitignore
index 8872fd6dd..74e0096c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,11 +15,11 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+/lib
 lib64/
 parts/
 sdist/
-var/
+/var
 wheels/
 share/python-wheels/
 *.egg-info/
diff --git a/packages/postgresml-dashboard/DEBIAN/control b/packages/postgresml-dashboard/DEBIAN/control
new file mode 100644
index 000000000..d1a103387
--- /dev/null
+++ b/packages/postgresml-dashboard/DEBIAN/control
@@ -0,0 +1,9 @@
+Package: postgresml-dashboard
+Version: ${PACKAGE_VERSION}
+Section: database
+Priority: optional
+Architecture: ${ARCH}
+Depends: openssl
+Maintainer: PostgresML <team@postgresml.org>
+Homepage: https://postgresml.org
+Description: PostgresML Dashboard
diff --git a/packages/postgresml-dashboard/DEBIAN/postinst b/packages/postgresml-dashboard/DEBIAN/postinst
new file mode 100644
index 000000000..f1a300c9b
--- /dev/null
+++ b/packages/postgresml-dashboard/DEBIAN/postinst
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+systemctl daemon-reload
+systemctl enable pgml-dashboard
+systemctl start pgml-dashboard
+
+echo "ROCKET_SECRET_KEY=$(openssl rand -hex 32)" >> /etc/pgml-dashboard/.env
diff --git a/packages/postgresml-dashboard/DEBIAN/postrm b/packages/postgresml-dashboard/DEBIAN/postrm
new file mode 100644
index 000000000..4f9dd3c06
--- /dev/null
+++ b/packages/postgresml-dashboard/DEBIAN/postrm
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+systemctl daemon-reload
+rm /etc/pgml-dashboard/.env
diff --git a/packages/postgresml-dashboard/DEBIAN/prerm b/packages/postgresml-dashboard/DEBIAN/prerm
new file mode 100644
index 000000000..dc5251c90
--- /dev/null
+++ b/packages/postgresml-dashboard/DEBIAN/prerm
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+systemctl stop pgml-dashboard
+systemctl disable pgml-dashboard
diff --git a/packages/postgresml-dashboard/build.sh b/packages/postgresml-dashboard/build.sh
new file mode 100644
index 000000000..8580d1024
--- /dev/null
+++ b/packages/postgresml-dashboard/build.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e
+
+dir="/tmp/postgresml-dashboard"
+deb_dir="$dir/deb-build"
+source_dir="$dir/source"
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+export PACKAGE_VERSION=${1:-"2.7.4"}
+export GITHUB_STARS=$(curl -s "https://api.github.com/repos/postgresml/postgresml" | grep stargazers_count | cut -d : -f 2 | tr -d " " | tr -d ",")
+if [[ $(arch) == "x86_64" ]]; then
+  export ARCH=amd64
+else
+  export ARCH=arm64
+fi
+
+rm -rf "$dir"
+mkdir -p "$deb_dir"
+
+cp -R ${SCRIPT_DIR}/* "$deb_dir"
+rm "$deb_dir/build.sh"
+
+( cd pgml-dashboard && \
+  cargo build --release && \
+  cp target/release/pgml-dashboard "$deb_dir/usr/bin/pgml-dashboard" && \
+  cp -R content "$deb_dir/usr/share/pgml-dashboard/dashboard-content" && \
+  cp -R static "$deb_dir/usr/share/pgml-dashboard/dashboard-static" )
+
+(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control"
+(cat ${SCRIPT_DIR}/etc/systemd/system/pgml-dashboard.service | envsubst) > "$deb_dir/etc/systemd/system/pgml-dashboard.service"
+
+chmod 755 ${deb_dir}/DEBIAN/post*
+chmod 755 ${deb_dir}/DEBIAN/pre*
+
+dpkg-deb \
+  --root-owner-group \
+  --build "$deb_dir" \
+  postgresml-dashboard-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb
+
+rm -rf "$dir"
diff --git a/packages/postgresml-dashboard/etc/pgml-dashboard/.gitkeep b/packages/postgresml-dashboard/etc/pgml-dashboard/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service b/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service
new file mode 100644
index 000000000..2e130814c
--- /dev/null
+++ b/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=PostgresML Dashboard: inspect models, write notebooks, and more.
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Environment=RUST_LOG=info
+Environment=DASHBOARD_STATIC_DIRECTORY=/usr/share/pgml-dashboard/dashboard-static
+Environment=DASHBOARD_CONTENT_DIRECTORY=/usr/share/pgml-dashboard/dashboard-content
+Environment=ROCKET_ADDRESS=0.0.0.0
+Environment=GITHUB_STARS=${GITHUB_STARS}
+Environment=SEARCH_INDEX_DIRECTORY=/var/lib/pgml-dashboard/search-index
+
+Type=simple
+Restart=always
+RestartSec=1
+ExecStart=/usr/bin/pgml-dashboard
+WorkingDirectory=/etc/pgml-dashboard
+
+[Install]
+WantedBy=multi-user.target
diff --git a/packages/postgresml-dashboard/usr/bin/.gitkeep b/packages/postgresml-dashboard/usr/bin/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/postgresml-dashboard/usr/share/pgml-dashboard/.gitkeep b/packages/postgresml-dashboard/usr/share/pgml-dashboard/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/postgresml-dashboard/var/lib/pgml-dashboard/.gitkeep b/packages/postgresml-dashboard/var/lib/pgml-dashboard/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/postgresml/postgresml/DEBIAN/control b/packages/postgresml/postgresml/DEBIAN/control
new file mode 100644
index 000000000..a2dd92c38
--- /dev/null
+++ b/packages/postgresml/postgresml/DEBIAN/control
@@ -0,0 +1,12 @@
+Package: postgresml-${PGVERSION}
+Version: ${PACKAGE_VERSION}
+Section: database
+Priority: optional
+Architecture: ${ARCH}
+Depends: postgresql-${PGVERSION}, postgresql-common, python${PYTHON_VERSION}, python${PYTHON_VERSION}-dev, python3-pip, python3-virtualenv, postgresql-pgml-${PGVERSION}, build-essential
+Maintainer: PostgresML <team@postgresml.org>
+Homepage: https://postgresml.org
+Description: PostgresML - Generative AI and Simple ML inside PostgreSQL
+ PostgresML is an AI application database.
+ Download open source models from Huggingface, or train your own, to create and index LLM embeddings,
+ generate text, or make online predictions using only SQL.
diff --git a/packages/postgresml/postgresml/DEBIAN/postinst b/packages/postgresml/postgresml/DEBIAN/postinst
new file mode 100755
index 000000000..bfdab863d
--- /dev/null
+++ b/packages/postgresml/postgresml/DEBIAN/postinst
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+#
+#
+set -e
+
+# Install config
+ln -sf /etc/postgresml/pgml.conf /etc/postgresql/${PGVERSION}/main/conf.d/01-pgml.conf
+
+# Setup virtualenv
+virtualenv /var/lib/postgresml/pgml-venv
+source "/var/lib/postgresml/pgml-venv/bin/activate"
+python -m pip install -r "/etc/postgresml/requirements.txt"
+python -m pip install -r "/etc/postgresml/requirements-xformers.txt" --no-dependencies
+deactivate
+
+# Restart Postgres
+service postgresql restart
diff --git a/packages/postgresml/postgresml/DEBIAN/postrm b/packages/postgresml/postgresml/DEBIAN/postrm
new file mode 100644
index 000000000..17d80f70b
--- /dev/null
+++ b/packages/postgresml/postgresml/DEBIAN/postrm
@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+#
+#
+set -e
+service postgresql restart
diff --git a/packages/postgresml/postgresml/DEBIAN/prerm b/packages/postgresml/postgresml/DEBIAN/prerm
new file mode 100644
index 000000000..0031d3b5d
--- /dev/null
+++ b/packages/postgresml/postgresml/DEBIAN/prerm
@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+#
+set -e
+
+rm /etc/postgresql/${PGVERSION}/main/conf.d/01-pgml.conf
diff --git a/packages/postgresml/postgresml/build.sh b/packages/postgresml/postgresml/build.sh
new file mode 100644
index 000000000..bca9fe54e
--- /dev/null
+++ b/packages/postgresml/postgresml/build.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+#
+#
+set -e
+
+deb_dir="/tmp/postgresml/deb-build"
+major=${1:-"14"}
+
+export PGVERSION=${major}
+export PACKAGE_VERSION=${1:-"2.7.4"}
+export PYTHON_VERSION=${2:-"3.10"}
+
+if [[ $(arch) == "x86_64" ]]; then
+  export ARCH=amd64
+  export PGVECTOR_MARCH=skylake
+else
+  export ARCH=arm64
+  export PGVECTOR_MARCH=native
+fi
+
+rm -rf "$deb_dir"
+mkdir -p "$deb_dir"
+
+cp -R packages/postgresml/* "$deb_dir"
+rm "$deb_dir/build.sh"
+
+(cat packages/postgresml/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control"
+(cat packages/postgresml/DEBIAN/postinst | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postinst"
+(cat packages/postgresml/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm"
+(cat packages/postgresml/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm"
+
+cp pgml-extension/requirements.txt "$deb_dir/etc/postgresml/requirements.txt"
+cp pgml-extension/requirements-xformers.txt "$deb_dir/etc/postgresml/requirements-xformers.txt"
+
+virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml/pgml-venv"
+source "$deb_dir/var/lib/postgresml/pgml-venv/bin/activate"
+
+python -m pip install -r "${deb_dir}/etc/postgresml/requirements.txt"
+python -m pip install -r "${deb_dir}/etc/postgresml/requirements-xformers.txt" --no-dependencies
+
+deactivate
+
+chmod 755 ${deb_dir}/DEBIAN/post*
+chmod 755 ${deb_dir}/DEBIAN/pre*
+
+dpkg-deb \
+  --root-owner-group \
+  -z1 \
+  --build "$deb_dir" \
+  postgresml-${PGVERSION}-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb
+
+rm -rf "$deb_dir"
diff --git a/packages/postgresml/postgresml/etc/postgresml/pgml.conf b/packages/postgresml/postgresml/etc/postgresml/pgml.conf
new file mode 100644
index 000000000..c47da6382
--- /dev/null
+++ b/packages/postgresml/postgresml/etc/postgresml/pgml.conf
@@ -0,0 +1,2 @@
+pgml.venv = '/var/lib/postgresml/pgml-venv'
+shared_preload_libraries = 'pgml,pg_stat_statements'
diff --git a/packages/postgresml/postgresml/var/lib/postgresml/.gitkeep b/packages/postgresml/postgresml/var/lib/postgresml/.gitkeep
new file mode 100644
index 000000000..e69de29bb

From 34949ca1a4cbfc97cfafd1d22dd1db97f6a16579 Mon Sep 17 00:00:00 2001
From: Lev Kokotov <hi@levthe.dev>
Date: Thu, 10 Aug 2023 18:07:39 -0700
Subject: [PATCH 2/2] i said everything

---
 .github/workflows/package-extension.yml | 33 ++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
index 3914932a6..1f2b1855b 100644
--- a/.github/workflows/package-extension.yml
+++ b/.github/workflows/package-extension.yml
@@ -182,22 +182,21 @@ jobs:
         fi
 
         for pg in {11..15}; do
-          bash packages/postgresml/build.sh ${pg} v${{ inputs.packageVersion }} ${{ inputs.pythonVersion }}
+          bash packages/postgresml/build.sh ${pg} v${{ inputs.packageVersion }}-1 ${{ inputs.pythonVersion }}
 
           deb-s3 upload \
             --lock \
             --bucket apt.postgresml.org \
-            postgresml-${pg}-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
+            postgresml-${pg}-${{ inputs.packageVersion }}-1-ubuntu22.04-${ARCH}.deb \
             --codename $(lsb_release -cs)
 
-          rm -f postgresml-${pg}-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb 
+          rm -f postgresml-${pg}-${{ inputs.packageVersion }}-1-ubuntu22.04-${ARCH}.deb 
         done
   dashboard:
     strategy:
       fail-fast: false # Let the other job finish
       matrix:
         os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
-        
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
@@ -231,3 +230,29 @@ jobs:
           --bucket apt.postgresml.org \
           postgresml-dashboard-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
           --codename $(lsb_release -cs)
+  publish-extension-docker-image:
+    needs: package
+    strategy:
+      matrix:
+        os: ["buildjet-4vcpu-ubuntu-2204"]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: pgml-extension/docker/postgresml
+    steps:
+      - uses: actions/checkout@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        run: |
+          docker run --privileged --rm tonistiigi/binfmt --install all
+          docker buildx create --use --name mybuilder
+          docker buildx build  \
+            --platform linux/amd64,linux/arm64 \
+            --tag ghcr.io/postgresml/postgresml:${{ inputs.version }} \
+            --build-arg POSTGRESML_VERSION=${{ inputs.postgresmlVersion }} \
+            --push .