Skip to content

Commit 8fcb39a

Browse files
levkklevkk
authored
Speed up releae (#923)
1 parent 58f9f0e commit 8fcb39a

File tree

26 files changed

+346
-175
lines changed

26 files changed

+346
-175
lines changed

‎.github/workflows/package-extension.yml renamed to ‎.github/workflows/ubuntu-packages-and-docker-image.yml

Lines changed: 26 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
1-
name: package extension (deb)
1+
name: Ubuntu packages and Docker image
22

33
on:
44
workflow_dispatch:
55
inputs:
66
packageVersion:
77
default: "2.7.4"
8-
pythonVersion:
9-
default: "3.10"
10-
118
jobs:
12-
extension:
9+
#
10+
# PostgresML extension.
11+
#
12+
postgresml-pgml:
1313
strategy:
1414
fail-fast: false # Let the other job finish
1515
matrix:
16-
os: ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-16vcpu-ubuntu-2204-arm"]
16+
os: ["buildjet-16vcpu-ubuntu-2204", "buildjet-16vcpu-ubuntu-2204-arm"]
1717
runs-on: ${{ matrix.os }}
1818
defaults:
1919
run:
@@ -131,68 +131,32 @@ jobs:
131131
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
132132
AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
133133
run: |
134-
for pg in {11..15}; do
135-
export PACKAGE_VERSION=${{ inputs.packageVersion }}
136-
export PGVERSION=${pg}
137-
138-
if [[ $(arch) == "x86_64" ]]; then
139-
export ARCH=amd64
140-
else
141-
export ARCH=arm64
142-
fi
143-
144-
mkdir -p target/release/pgml-pg${pg}/DEBIAN
145-
(cat control | envsubst) > target/release/pgml-pg${pg}/DEBIAN/control
146-
dpkg-deb --root-owner-group --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb
134+
bash packages/postgresql-pgml/release.sh ${{ inputs.packageVersion }}
147135
148-
deb-s3 upload \
149-
--bucket apt.postgresml.org \
150-
postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb \
151-
--codename $(lsb_release -cs)
152-
done
153-
package:
154-
needs: extension
136+
#
137+
# PostgresML meta package which installs
138+
# all the necessary dependencies.
139+
#
140+
postgresml:
141+
needs: postgresml-pgml
155142
strategy:
156-
max-parallel: 1
157-
fail-fast: false # Let the other job finish, or they can lock each other out
158143
matrix:
159-
os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
144+
os: ["ubuntu-22.04"]
160145
runs-on: ${{ matrix.os }}
161146
steps:
162147
- uses: actions/checkout@v3
163-
- name: Install dependencies
164-
env:
165-
DEBIAN_FRONTEND: noninteractive
166-
TZ: Etc/UTC
167-
run: |
168-
curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
169-
sudo gem install deb-s3-0.11.4.gem
170-
dpkg-deb --version
171-
sudo apt install -y python3-pip python3 python3-virtualenv
172148
- name: Build and release package
173149
env:
174150
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
175151
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
176152
AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
177153
run: |
178-
if [[ $(arch) == "x86_64" ]]; then
179-
export ARCH=amd64
180-
else
181-
export ARCH=arm64
182-
fi
154+
bash packages/postgresml/release.sh ${{ inputs.packageVersion }}
183155
184-
for pg in {11..15}; do
185-
bash packages/postgresml/build.sh ${pg} v${{ inputs.packageVersion }}-1 ${{ inputs.pythonVersion }}
186-
187-
deb-s3 upload \
188-
--lock \
189-
--bucket apt.postgresml.org \
190-
postgresml-${pg}-${{ inputs.packageVersion }}-1-ubuntu22.04-${ARCH}.deb \
191-
--codename $(lsb_release -cs)
192-
193-
rm -f postgresml-${pg}-${{ inputs.packageVersion }}-1-ubuntu22.04-${ARCH}.deb
194-
done
195-
dashboard:
156+
#
157+
# PostgresML dashboard.
158+
#
159+
postgresml-dashboard:
196160
strategy:
197161
fail-fast: false # Let the other job finish
198162
matrix:
@@ -203,35 +167,21 @@ jobs:
203167
- uses: actions-rs/toolchain@v1
204168
with:
205169
toolchain: stable
206-
- name: Install dependencies
207-
env:
208-
DEBIAN_FRONTEND: noninteractive
209-
TZ: Etc/UTC
210-
run: |
211-
curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
212-
sudo gem install deb-s3-0.11.4.gem
213-
dpkg-deb --version
214170
- name: Build and release package
215171
env:
216172
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
217173
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
218174
AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
219175
run: |
220-
if [[ $(arch) == "x86_64" ]]; then
221-
export ARCH=amd64
222-
else
223-
export ARCH=arm64
224-
fi
225-
226-
bash packages/postgresml-dashboard/build.sh ${{ inputs.packageVersion }}
176+
bash packages/postgresml-dashboard/release.sh ${{ inputs.packageVersion }}
227177
228-
deb-s3 upload \
229-
--lock \
230-
--bucket apt.postgresml.org \
231-
postgresml-dashboard-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
232-
--codename $(lsb_release -cs)
178+
#
179+
# PostgresML Docker image.
180+
#
233181
publish-extension-docker-image:
234-
needs: package
182+
needs:
183+
- postgresml
184+
- postgresml-dashboard
235185
strategy:
236186
matrix:
237187
os: ["buildjet-4vcpu-ubuntu-2204"]

‎.github/workflows/ubuntu-postgresml-python-package.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Ubuntu PostgresML Python package
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
packageVersion:
7+
default: "2.7.4"
8+
9+
jobs:
10+
postgresml-python:
11+
strategy:
12+
matrix:
13+
os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-4vcpu-ubuntu-2204-arm"]
14+
runs-on: ${{ matrix.os }}
15+
steps:
16+
- uses: actions/checkout@v3
17+
- name: Build and release package
18+
env:
19+
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
20+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
21+
AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
22+
run: |
23+
bash packages/postgresml-python/release.sh ${{ inputs.packageVersion }}

‎packages/postgresml-dashboard/build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ mkdir -p "$deb_dir"
1919
cp -R ${SCRIPT_DIR}/* "$deb_dir"
2020
rm "$deb_dir/build.sh"
2121

22-
( cd pgml-dashboard && \
22+
( cd ${SCRIPT_DIR}/../../pgml-dashboard && \
2323
cargo build --release && \
2424
cp target/release/pgml-dashboard "$deb_dir/usr/bin/pgml-dashboard" && \
2525
cp -R content "$deb_dir/usr/share/pgml-dashboard/dashboard-content" && \

‎packages/postgresml-dashboard/release.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
5+
package_version="$1"
6+
7+
if [[ -z "$package_version" ]]; then
8+
echo "Usage: $0 <package version, e.g. 2.7.4>"
9+
exit 1
10+
fi
11+
12+
if [[ $(arch) == "x86_64" ]]; then
13+
export ARCH=amd64
14+
else
15+
export ARCH=arm64
16+
fi
17+
18+
if ! which dpkg-deb; then
19+
curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
20+
sudo gem install deb-s3-0.11.4.gem
21+
dpkg-deb --version
22+
fi
23+
24+
function package_name() {
25+
echo "postgresml-dashboard-${package_version}-ubuntu22.04-${ARCH}.deb"
26+
}
27+
28+
bash ${SCRIPT_DIR}/build.sh "$package_version"
29+
30+
if [[ ! -f $(package_name) ]]; then
31+
echo "File $(package_name) doesn't exist"
32+
exit 1
33+
fi
34+
35+
deb-s3 upload \
36+
--lock \
37+
--bucket apt.postgresml.org \
38+
$(package_name) \
39+
--codename $(lsb_release -cs)

‎packages/postgresml-python/DEBIAN/control

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Package: postgresml-python
2+
Version: ${PACKAGE_VERSION}
3+
Section: database
4+
Priority: optional
5+
Architecture: ${ARCH}
6+
Depends: python${PYTHON_VERSION}, python${PYTHON_VERSION}-dev, python3-pip, python3-virtualenv, build-essential
7+
Maintainer: PostgresML <team@postgresml.org>
8+
Homepage: https://postgresml.org
9+
Description: PostgresML - Generative AI and Simple ML inside PostgreSQL (virtual environment)
10+
PostgresML is an AI application database.
11+
Download open source models from Huggingface, or train your own, to create and index LLM embeddings,
12+
generate text, or make online predictions using only SQL.
13+
This package installs the Python dependencies necessary for using PostgresML.

‎packages/postgresml-python/DEBIAN/postinst

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
#
3+
#
4+
#
5+
set -e
6+
7+
# Setup virtualenv
8+
virtualenv /var/lib/postgresml-python/pgml-venv
9+
source "/var/lib/postgresml-python/pgml-venv/bin/activate"
10+
python -m pip install -r "/etc/postgresml-python/requirements.txt"
11+
python -m pip install -r "/etc/postgresml-python/requirements-xformers.txt" --no-dependencies
12+
deactivate

‎packages/postgresml-python/DEBIAN/postrm

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
set -e
3+

‎packages/postgresml-python/DEBIAN/prerm

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
#
3+
#

‎packages/postgresml-python/build.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
#
3+
#
4+
#
5+
set -e
6+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
7+
deb_dir="/tmp/postgresml-python/deb-build"
8+
major=${1:-"14"}
9+
10+
export PACKAGE_VERSION=${1:-"2.7.4"}
11+
export PYTHON_VERSION=${2:-"3.10"}
12+
13+
if [[ $(arch) == "x86_64" ]]; then
14+
export ARCH=amd64
15+
else
16+
export ARCH=arm64
17+
fi
18+
19+
rm -rf "$deb_dir"
20+
mkdir -p "$deb_dir"
21+
22+
cp -R ${SCRIPT_DIR}/* "$deb_dir"
23+
rm "$deb_dir/build.sh"
24+
25+
(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control"
26+
(cat ${SCRIPT_DIR}/DEBIAN/postinst | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postinst"
27+
(cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm"
28+
(cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm"
29+
30+
cp ${SCRIPT_DIR}/../../pgml-extension/requirements.txt "$deb_dir/etc/postgresml-python/requirements.txt"
31+
cp ${SCRIPT_DIR}/../../pgml-extension/requirements-xformers.txt "$deb_dir/etc/postgresml-python/requirements-xformers.txt"
32+
33+
virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml-python/pgml-venv"
34+
source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate"
35+
36+
python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt"
37+
python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-xformers.txt" --no-dependencies
38+
39+
deactivate
40+
41+
chmod 755 ${deb_dir}/DEBIAN/post*
42+
chmod 755 ${deb_dir}/DEBIAN/pre*
43+
44+
dpkg-deb \
45+
--root-owner-group \
46+
-z1 \
47+
--build "$deb_dir" \
48+
postgresml-python-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb
49+
50+
rm -rf "$deb_dir"

‎packages/postgresml/postgresml/var/lib/postgresml/.gitkeep renamed to ‎packages/postgresml-python/etc/postgresml-python/.gitkeep

File renamed without changes.

0 commit comments

Comments
 (0)