Skip to content

Silas final pipelines #934

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 35 commits into from
Aug 21, 2023
Merged

Silas final pipelines #934

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
6c14956
Remote save
SilasMarvin Jul 14, 2023
3b2e3b5
Working remote embeddings
SilasMarvin Jul 17, 2023
f1d6bf7
Compiling
SilasMarvin Jul 19, 2023
22f280e
Commit before moving everything to lazy
SilasMarvin Jul 19, 2023
ec090ca
Working lazy python
SilasMarvin Jul 21, 2023
58c01a3
Commit before moving adjusting Javascript macros
SilasMarvin Jul 21, 2023
9e7b146
Working javascript sdk
SilasMarvin Jul 22, 2023
abb4f5e
Working javascript sdk
SilasMarvin Jul 26, 2023
76ccf3a
The start of working pipelines
SilasMarvin Jul 28, 2023
cfcc66b
Working pipelines in python
SilasMarvin Aug 3, 2023
a9dcbc9
Uncomment
SilasMarvin Aug 3, 2023
8b48750
Added to_dict function
SilasMarvin Aug 3, 2023
6e3f1e6
Small changes and prep for progress bars
SilasMarvin Aug 4, 2023
5365557
Working progress bars and many other small but exciting things
SilasMarvin Aug 4, 2023
66476ff
Prepping to push to test pypi
SilasMarvin Aug 7, 2023
f2613d7
Prepping for javascript
SilasMarvin Aug 8, 2023
92c9623
Improvments to javascript and updates to the python sdk deploy script
SilasMarvin Aug 8, 2023
8a4e3cf
Prepping for real tests
SilasMarvin Aug 8, 2023
12bb3a8
Updated sql
SilasMarvin Aug 9, 2023
2b5b68b
Python examples translated to use pipelines
SilasMarvin Aug 9, 2023
447fc80
Mostly cleaned up and documented crate, and cleaned up python README …
SilasMarvin Aug 10, 2023
333c5e6
Ready for test deployments
SilasMarvin Aug 10, 2023
11bcce2
Updated manual build file for python
SilasMarvin Aug 10, 2023
845bf02
Build fast
SilasMarvin Aug 11, 2023
4904a1a
Small tweaks
SilasMarvin Aug 11, 2023
64dc7e2
Prepping for another test release
SilasMarvin Aug 11, 2023
c3b274c
Prepping to expand query_builder
SilasMarvin Aug 11, 2023
cb143a5
Massive cleanups to macros
SilasMarvin Aug 11, 2023
c66b07b
Massive cleanups to macros
SilasMarvin Aug 11, 2023
b7d4c2d
Ready to release
SilasMarvin Aug 11, 2023
a2c87b1
Formatting
SilasMarvin Aug 13, 2023
dd9c3ab
Renamed files
SilasMarvin Aug 21, 2023
5568608
Added removed file
SilasMarvin Aug 21, 2023
4a2e98d
Removed unnecessary file
SilasMarvin Aug 21, 2023
e673af4
Updated sdk version to 0.9
SilasMarvin Aug 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
The start of working pipelines
  • Loading branch information
@SilasMarvin
SilasMarvin committed Aug 21, 2023
commit 76ccf3aa6353dbfc8602c35e87e67743543de586
243 changes: 123 additions & 120 deletions pgml-sdks/rust/pgml-macros/src/python.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,37 +69,91 @@ pub fn generate_python_derive(parsed: DeriveInput) -> proc_macro::TokenStream {
let expanded = quote! {
#[cfg(feature = "python")]
#[pyo3::pyclass(name = #wrapped_type_name)]
#[derive(Debug, Clone)]
#[derive(Clone, Debug)]
pub struct #name_ident {
pub wrapped: std::sync::Arc<tokio::sync::Mutex<#wrapped_type_ident>>
pub wrapped: std::boxed::Box<#wrapped_type_ident>
}

#[cfg(feature = "python")]
impl From<#wrapped_type_ident> for #name_ident {
fn from(w: #wrapped_type_ident) -> Self {
Self {
wrapped: std::sync::Arc::new(tokio::sync::Mutex::new(w)),
wrapped: std::boxed::Box::new(w),
}
}
}

// #[cfg(feature = "python")]
// impl From<#name_ident> for #wrapped_type_ident {
// fn from(w: #name_ident) -> Self {
// // The verbose typing here is necessary
// // let inner: tokio::sync::Mutex<Self> = std::sync::Arc::into_inner(w.wrapped).expect("My guess is this is it");
// // inner.into_inner()
//
// use std::ops::DerefMut;
// let mut wrapped = w.wrapped.blocking_lock();
// let wrapped = wrapped.deref_mut();
// let wrapped = wrapped.to_owned();
// wrapped
//
// // let wrapped = (*w.wrapped).into_inner();
// // wrapped
// }
// }
#[cfg(feature = "python")]
impl CustomInto<#wrapped_type_ident> for #name_ident {
fn custom_into(self) -> #wrapped_type_ident {
*self.wrapped
}
}

#[cfg(feature = "python")]
impl CustomInto<&'static mut #wrapped_type_ident> for &mut #name_ident {
fn custom_into(self) -> &'static mut #wrapped_type_ident {
// This is how we get around the liftime checker
unsafe {
let ptr = &*self.wrapped as *const #wrapped_type_ident;
let ptr = ptr as *mut #wrapped_type_ident;
let boxed = Box::from_raw(ptr);
Box::leak(boxed)
}
}
}

#[cfg(feature = "python")]
impl CustomInto<&'static #wrapped_type_ident> for &mut #name_ident {
fn custom_into(self) -> &'static #wrapped_type_ident {
// This is how we get around the liftime checker
unsafe {
let ptr = &*self.wrapped as *const #wrapped_type_ident;
let ptr = ptr as *mut #wrapped_type_ident;
let boxed = Box::from_raw(ptr);
Box::leak(boxed)
}
}
}

#[cfg(feature = "python")]
impl CustomInto<#wrapped_type_ident> for &mut #name_ident {
fn custom_into(self) -> #wrapped_type_ident {
// This is how we get around the liftime checker
unsafe {
let ptr = &*self.wrapped as *const #wrapped_type_ident;
let ptr = ptr as *mut #wrapped_type_ident;
let boxed = Box::from_raw(ptr);
Box::leak(boxed).to_owned()
}
}
}

#[cfg(feature = "python")]
impl CustomInto<&'static #wrapped_type_ident> for &#name_ident {
fn custom_into(self) -> &'static #wrapped_type_ident {
// This is how we get around the liftime checker
unsafe {
let ptr = &*self.wrapped as *const #wrapped_type_ident;
let ptr = ptr as *mut #wrapped_type_ident;
let boxed = Box::from_raw(ptr);
Box::leak(boxed)
}
}
}

#[cfg(feature = "python")]
impl CustomInto<#wrapped_type_ident> for &#name_ident {
fn custom_into(self) -> #wrapped_type_ident {
// This is how we get around the liftime checker
unsafe {
let ptr = &*self.wrapped as *const #wrapped_type_ident;
let ptr = ptr as *mut #wrapped_type_ident;
let boxed = Box::from_raw(ptr);
Box::leak(boxed).to_owned()
}
}
}

#[cfg(feature = "python")]
impl pyo3::IntoPy<pyo3::PyObject> for #wrapped_type_ident {
Expand Down Expand Up @@ -162,6 +216,23 @@ pub fn generate_python_methods(
.as_ref()
.is_some_and(|r| r.to_string().replace("mut", "").trim() == "self");

let some_wrapper_type = match method.receiver.as_ref() {
Some(r) => {
let st = r.to_string();
Some(if st.contains("&") {
let st = st.replace("self", &wrapped_type_ident.to_string());
let s = syn::parse_str::<syn::Type>(&st).expect(&format!(
"Error converting self type to necessary syn type: {:?}",
r
));
s.to_token_stream()
} else {
quote! { #wrapped_type_ident }
})
}
None => None,
};

let signature = quote! {
pub fn #method_ident<'a>(#(#method_arguments),*) -> #output_type
};
Expand All @@ -174,7 +245,13 @@ pub fn generate_python_methods(
let mut p3 = method
.method_arguments
.iter()
.map(|a| format!("{}: {}", a.0.replace("mut", "").trim(), get_python_type(&a.1)))
.map(|a| {
format!(
"{}: {}",
a.0.replace("mut", "").trim(),
get_python_type(&a.1)
)
})
.collect::<Vec<String>>();
p3.insert(0, "self".to_string());
let p3 = p3.join(", ");
Expand Down Expand Up @@ -228,37 +305,15 @@ pub fn generate_python_methods(
#method_ident(#(#wrapper_arguments),*)
};

let middle = if does_take_ownership_of_self {
if method.is_async {
quote! {
{
use std::ops::DerefMut;
let mut wrapped = wrapped.lock().await;
let wrapped = wrapped.deref_mut();
let wrapped = wrapped.to_owned();
wrapped.#middle.await
}
}
} else {
quote! {
{
use std::ops::DerefMut;
let mut wrapped = wrapped.blocking_lock();
let wrapped = wrapped.deref_mut();
let wrapped = wrapped.to_owned();
wrapped.#middle
}
let middle = if method.is_async {
quote! {
{
wrapped.#middle.await
}
}
} else {
if method.is_async {
quote! {
wrapped.lock().await.#middle.await
}
} else {
quote! {
wrapped.blocking_lock().#middle
}
quote! {
wrapped.#middle
}
};

Expand All @@ -284,17 +339,17 @@ pub fn generate_python_methods(
};
let middle = if method.is_async {
quote! {
let wrapped = self.wrapped.clone();
let mut wrapped: #some_wrapper_type = self.custom_into();
#prepared_wrapper_arguments
pyo3_asyncio::tokio::future_into_py(py, async move {
use std::ops::DerefMut;
#prepared_wrapper_arguments
#middle
Ok(x)
})
}
} else {
quote! {
let wrapped = self.wrapped.clone();
let mut wrapped: #some_wrapper_type = self.custom_into();
use std::ops::DerefMut;
#prepared_wrapper_arguments
#middle
Expand Down Expand Up @@ -385,79 +440,26 @@ pub fn get_method_wrapper_arguments_python(
fn convert_method_wrapper_arguments(
name_ident: syn::Ident,
ty: &SupportedType,
is_async: bool,
_is_async: bool,
) -> (
proc_macro2::TokenStream,
proc_macro2::TokenStream,
proc_macro2::TokenStream,
) {
match ty {
SupportedType::Reference(r) => {
let (d, w, p) = convert_method_wrapper_arguments(name_ident.clone(), &r.ty, is_async);
match *r.ty.clone() {
SupportedType::Database
| SupportedType::Collection
| SupportedType::Model
| SupportedType::QueryBuilder
| SupportedType::QueryRunner
| SupportedType::Splitter => {
let p = if is_async {
quote! {
let mut #name_ident = #name_ident.wrapped.lock().await;
let #name_ident = #name_ident.deref_mut();
}
} else {
quote! {
let mut #name_ident = #name_ident.wrapped.blocking_lock();
let #name_ident = #name_ident.deref_mut();
}
};
(d, w, p)
}
_ => {
if r.mutable {
(d, quote! { &mut #w}, p)
} else {
(d, quote! { & #w}, p)
}
}
}
}
SupportedType::str => (
quote! {#name_ident: String},
quote! { #name_ident},
quote! {},
),
_ => {
let t = ty
.to_type(Some("Python"))
.expect("Could not parse type in convert_method_type in python.rs");
let p = match ty {
SupportedType::Database
| SupportedType::Collection
| SupportedType::Model
| SupportedType::QueryBuilder
| SupportedType::QueryRunner
| SupportedType::Splitter => {
if is_async {
quote! {
let mut #name_ident = #name_ident.wrapped.lock().await;
let #name_ident = #name_ident.deref_mut();
let #name_ident = #name_ident.to_owned();
}
} else {
quote! {
let mut #name_ident = #name_ident.wrapped.blocking_lock();
let #name_ident = #name_ident.deref_mut();
let #name_ident = #name_ident.to_owned();
}
}
}
_ => quote! {},
};
(quote! { #name_ident : #t}, quote! {#name_ident}, p)
}
}
let pyo3_type = ty
.to_type(Some("Python"))
.expect("Could not parse type in convert_method_wrapper_arguments in python.rs");
let normal_type = ty
.to_type(None)
.expect("Could not parse type in convert_method_wrapper_arguments in python.rs");

(
quote! { #name_ident: #pyo3_type },
quote! { #name_ident },
quote! {
let #name_ident: #normal_type = #name_ident.custom_into();
},
)
}

fn convert_output_type_convert_from_python(
Expand Down Expand Up @@ -541,6 +543,7 @@ fn get_type_for_optional(ty: &SupportedType) -> String {
SupportedType::i64 | SupportedType::u64 => 1.to_string(),
SupportedType::f64 => 1.0.to_string(),
SupportedType::Json => "{}".to_string(),
SupportedType::Model | SupportedType::Splitter => "None".to_string(),
_ => panic!("Type not yet supported for optional python stub: {:?}", ty),
}
}
Loading