postgresml · levkk · Aug 11, 2023 · Aug 11, 2023 · Aug 11, 2023
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
@@ -5,10 +5,13 @@ on:
     inputs:
       packageVersion:
         default: "2.7.4"
+      pythonVersion:
+        default: "3.10"
 
 jobs:
-  build:
+  extension:
     strategy:
+      fail-fast: false # Let the other job finish
       matrix:
         os: ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-16vcpu-ubuntu-2204-arm"]
     runs-on: ${{ matrix.os }}
@@ -147,3 +150,109 @@ jobs:
             postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb \
             --codename $(lsb_release -cs)
         done
+  package:
+    needs: extension
+    strategy:
+      max-parallel: 1
+      fail-fast: false # Let the other job finish, or they can lock each other out
+      matrix:
+        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        TZ: Etc/UTC
+      run: |
+        curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
+        sudo gem install deb-s3-0.11.4.gem
+        dpkg-deb --version
+        sudo apt install -y python3-pip python3 python3-virtualenv
+    - name: Build and release package
+      env:
+        AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+      run: |
+        if [[ $(arch) == "x86_64" ]]; then
+          export ARCH=amd64
+        else
+          export ARCH=arm64
+        fi
+
+        for pg in {11..15}; do
+          bash packages/postgresml/build.sh ${pg} v${{ inputs.packageVersion }}-1 ${{ inputs.pythonVersion }}
+
+          deb-s3 upload \
+            --lock \
+            --bucket apt.postgresml.org \
+            postgresml-${pg}-${{ inputs.packageVersion }}-1-ubuntu22.04-${ARCH}.deb \
+            --codename $(lsb_release -cs)
+
+          rm -f postgresml-${pg}-${{ inputs.packageVersion }}-1-ubuntu22.04-${ARCH}.deb 
+        done
+  dashboard:
+    strategy:
+      fail-fast: false # Let the other job finish
+      matrix:
+        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        TZ: Etc/UTC
+      run: |
+        curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
+        sudo gem install deb-s3-0.11.4.gem
+        dpkg-deb --version
+    - name: Build and release package
+      env:
+        AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+      run: |
+        if [[ $(arch) == "x86_64" ]]; then
+          export ARCH=amd64
+        else
+          export ARCH=arm64
+        fi
+
+        bash packages/postgresml-dashboard/build.sh ${{ inputs.packageVersion }}
+
+        deb-s3 upload \
+          --lock \
+          --bucket apt.postgresml.org \
+          postgresml-dashboard-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
+          --codename $(lsb_release -cs)
+  publish-extension-docker-image:
+    needs: package
+    strategy:
+      matrix:
+        os: ["buildjet-4vcpu-ubuntu-2204"]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: pgml-extension/docker/postgresml
+    steps:
+      - uses: actions/checkout@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        run: |
+          docker run --privileged --rm tonistiigi/binfmt --install all
+          docker buildx create --use --name mybuilder
+          docker buildx build  \
+            --platform linux/amd64,linux/arm64 \
+            --tag ghcr.io/postgresml/postgresml:${{ inputs.version }} \
+            --build-arg POSTGRESML_VERSION=${{ inputs.postgresmlVersion }} \
+            --push .
diff --git a/.gitignore b/.gitignore
@@ -15,11 +15,11 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+/lib
 lib64/
 parts/
 sdist/
-var/
+/var
 wheels/
 share/python-wheels/
 *.egg-info/

diff --git a/packages/postgresml-dashboard/DEBIAN/control b/packages/postgresml-dashboard/DEBIAN/control
@@ -0,0 +1,9 @@
+Package: postgresml-dashboard
+Version: ${PACKAGE_VERSION}
+Section: database
+Priority: optional
+Architecture: ${ARCH}
+Depends: openssl
+Maintainer: PostgresML <team@postgresml.org>
+Homepage: https://postgresml.org
+Description: PostgresML Dashboard
diff --git a/packages/postgresml-dashboard/DEBIAN/postinst b/packages/postgresml-dashboard/DEBIAN/postinst
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+systemctl daemon-reload
+systemctl enable pgml-dashboard
+systemctl start pgml-dashboard
+
+echo "ROCKET_SECRET_KEY=$(openssl rand -hex 32)" >> /etc/pgml-dashboard/.env
diff --git a/packages/postgresml-dashboard/DEBIAN/postrm b/packages/postgresml-dashboard/DEBIAN/postrm
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+systemctl daemon-reload
+rm /etc/pgml-dashboard/.env
diff --git a/packages/postgresml-dashboard/DEBIAN/prerm b/packages/postgresml-dashboard/DEBIAN/prerm
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+systemctl stop pgml-dashboard
+systemctl disable pgml-dashboard
diff --git a/packages/postgresml-dashboard/build.sh b/packages/postgresml-dashboard/build.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e
+
+dir="/tmp/postgresml-dashboard"
+deb_dir="$dir/deb-build"
+source_dir="$dir/source"
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+export PACKAGE_VERSION=${1:-"2.7.4"}
+export GITHUB_STARS=$(curl -s "https://api.github.com/repos/postgresml/postgresml" | grep stargazers_count | cut -d : -f 2 | tr -d " " | tr -d ",")
+if [[ $(arch) == "x86_64" ]]; then
+  export ARCH=amd64
+else
+  export ARCH=arm64
+fi
+
+rm -rf "$dir"
+mkdir -p "$deb_dir"
+
+cp -R ${SCRIPT_DIR}/* "$deb_dir"
+rm "$deb_dir/build.sh"
+
+( cd pgml-dashboard && \
+  cargo build --release && \
+  cp target/release/pgml-dashboard "$deb_dir/usr/bin/pgml-dashboard" && \
+  cp -R content "$deb_dir/usr/share/pgml-dashboard/dashboard-content" && \
+  cp -R static "$deb_dir/usr/share/pgml-dashboard/dashboard-static" )
+
+(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control"
+(cat ${SCRIPT_DIR}/etc/systemd/system/pgml-dashboard.service | envsubst) > "$deb_dir/etc/systemd/system/pgml-dashboard.service"
+
+chmod 755 ${deb_dir}/DEBIAN/post*
+chmod 755 ${deb_dir}/DEBIAN/pre*
+
+dpkg-deb \
+  --root-owner-group \
+  --build "$deb_dir" \
+  postgresml-dashboard-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb
+
+rm -rf "$dir"
diff --git a/packages/postgresml-dashboard/etc/pgml-dashboard/.gitkeep b/packages/postgresml-dashboard/etc/pgml-dashboard/.gitkeep
diff --git a/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service b/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=PostgresML Dashboard: inspect models, write notebooks, and more.
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Environment=RUST_LOG=info
+Environment=DASHBOARD_STATIC_DIRECTORY=/usr/share/pgml-dashboard/dashboard-static
+Environment=DASHBOARD_CONTENT_DIRECTORY=/usr/share/pgml-dashboard/dashboard-content
+Environment=ROCKET_ADDRESS=0.0.0.0
+Environment=GITHUB_STARS=${GITHUB_STARS}
+Environment=SEARCH_INDEX_DIRECTORY=/var/lib/pgml-dashboard/search-index
+
+Type=simple
+Restart=always
+RestartSec=1
+ExecStart=/usr/bin/pgml-dashboard
+WorkingDirectory=/etc/pgml-dashboard
+
+[Install]
+WantedBy=multi-user.target
diff --git a/packages/postgresml-dashboard/usr/bin/.gitkeep b/packages/postgresml-dashboard/usr/bin/.gitkeep
diff --git a/packages/postgresml-dashboard/usr/share/pgml-dashboard/.gitkeep b/packages/postgresml-dashboard/usr/share/pgml-dashboard/.gitkeep
diff --git a/packages/postgresml-dashboard/var/lib/pgml-dashboard/.gitkeep b/packages/postgresml-dashboard/var/lib/pgml-dashboard/.gitkeep
diff --git a/packages/postgresml/postgresml/DEBIAN/control b/packages/postgresml/postgresml/DEBIAN/control
@@ -0,0 +1,12 @@
+Package: postgresml-${PGVERSION}
+Version: ${PACKAGE_VERSION}
+Section: database
+Priority: optional
+Architecture: ${ARCH}
+Depends: postgresql-${PGVERSION}, postgresql-common, python${PYTHON_VERSION}, python${PYTHON_VERSION}-dev, python3-pip, python3-virtualenv, postgresql-pgml-${PGVERSION}, build-essential
+Maintainer: PostgresML <team@postgresml.org>
+Homepage: https://postgresml.org
+Description: PostgresML - Generative AI and Simple ML inside PostgreSQL
+ PostgresML is an AI application database.
+ Download open source models from Huggingface, or train your own, to create and index LLM embeddings,
+ generate text, or make online predictions using only SQL.
diff --git a/packages/postgresml/postgresml/DEBIAN/postinst b/packages/postgresml/postgresml/DEBIAN/postinst
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+#
+#
+set -e
+
+# Install config
+ln -sf /etc/postgresml/pgml.conf /etc/postgresql/${PGVERSION}/main/conf.d/01-pgml.conf
+
+# Setup virtualenv
+virtualenv /var/lib/postgresml/pgml-venv
+source "/var/lib/postgresml/pgml-venv/bin/activate"
+python -m pip install -r "/etc/postgresml/requirements.txt"
+python -m pip install -r "/etc/postgresml/requirements-xformers.txt" --no-dependencies
+deactivate
+
+# Restart Postgres
+service postgresql restart
diff --git a/packages/postgresml/postgresml/DEBIAN/postrm b/packages/postgresml/postgresml/DEBIAN/postrm
@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+#
+#
+set -e
+service postgresql restart
diff --git a/packages/postgresml/postgresml/DEBIAN/prerm b/packages/postgresml/postgresml/DEBIAN/prerm
@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+#
+set -e
+
+rm /etc/postgresql/${PGVERSION}/main/conf.d/01-pgml.conf
diff --git a/packages/postgresml/postgresml/build.sh b/packages/postgresml/postgresml/build.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+#
+#
+set -e
+
+deb_dir="/tmp/postgresml/deb-build"
+major=${1:-"14"}
+
+export PGVERSION=${major}
+export PACKAGE_VERSION=${1:-"2.7.4"}
+export PYTHON_VERSION=${2:-"3.10"}
+
+if [[ $(arch) == "x86_64" ]]; then
+  export ARCH=amd64
+  export PGVECTOR_MARCH=skylake
+else
+  export ARCH=arm64
+  export PGVECTOR_MARCH=native
+fi
+
+rm -rf "$deb_dir"
+mkdir -p "$deb_dir"
+
+cp -R packages/postgresml/* "$deb_dir"
+rm "$deb_dir/build.sh"
+
+(cat packages/postgresml/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control"
+(cat packages/postgresml/DEBIAN/postinst | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postinst"
+(cat packages/postgresml/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm"
+(cat packages/postgresml/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm"
+
+cp pgml-extension/requirements.txt "$deb_dir/etc/postgresml/requirements.txt"
+cp pgml-extension/requirements-xformers.txt "$deb_dir/etc/postgresml/requirements-xformers.txt"
+
+virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml/pgml-venv"
+source "$deb_dir/var/lib/postgresml/pgml-venv/bin/activate"
+
+python -m pip install -r "${deb_dir}/etc/postgresml/requirements.txt"
+python -m pip install -r "${deb_dir}/etc/postgresml/requirements-xformers.txt" --no-dependencies
+
+deactivate
+
+chmod 755 ${deb_dir}/DEBIAN/post*
+chmod 755 ${deb_dir}/DEBIAN/pre*
+
+dpkg-deb \
+  --root-owner-group \
+  -z1 \
+  --build "$deb_dir" \
+  postgresml-${PGVERSION}-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb
+
+rm -rf "$deb_dir"
diff --git a/packages/postgresml/postgresml/etc/postgresml/pgml.conf b/packages/postgresml/postgresml/etc/postgresml/pgml.conf
@@ -0,0 +1,2 @@
+pgml.venv = '/var/lib/postgresml/pgml-venv'
+shared_preload_libraries = 'pgml,pg_stat_statements'
diff --git a/packages/postgresml/postgresml/var/lib/postgresml/.gitkeep b/packages/postgresml/postgresml/var/lib/postgresml/.gitkeep
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,11 +15,11 @@ dist/ @@
     downloads/
     eggs/
     .eggs/
-    lib/
+    /lib
     lib64/
     parts/
     sdist/
-    var/
+    /var
     wheels/
     share/python-wheels/
     *.egg-info/
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		pgml.venv = '/var/lib/postgresml/pgml-venv'
		shared_preload_libraries = 'pgml,pg_stat_statements'