|
## Provider Endpoints
Groq: api.groq.com/openai
Hyperbolic: https://api.hyperbolic.xyz/v1
Mistral: api.mistral.ai
OpenAI: api.openai.com
OpenRouter: openrouter.ai/api
Together: api.together.xyz
xAI: https://api.x.ai/v1
Cerebras: https://api.cerebras.ai/v1
SambaNova: https://api.sambanova.ai/v1
FIreworks: https://api.fireworks.ai/inference/v1
Gemini: https://generativelanguage.googleapis.com/v1beta/openai
# Example Code
## Basic
```python
import requests, time
Keys = {
'mistral':'abc',
'openrouter':'sk-or-v1-abc',
'groq':'abc',
'openai': 'sk-abc'}
Endpoints = {
'mistral': 'https://api.mistral.ai/v1',
'openrouter': 'https://openrouter.ai/api/v1',
'groq': 'https://api.groq.com/openai/v1',
'openai': 'https://api.openai.com/v1'}
navigateKeys = lambda f, D, L: D if L == [] else f(f, D[L[0]], L[1:])
mask = lambda D, E: {k: D[k] if k not in E else E[k] for k in list(D.keys())
+list(E.keys())}
makeHeaders = lambda model: {'Authorization': f"Bearer {Keys[model['provider']]}",
'Content-Type': 'application/json'}
def makeResponse(model, data):
r = requests.post(Endpoints[model['provider']] + '/' * bool(model['mode']) +
model['mode'] + '/completions', headers=makeHeaders(model), json=mask({'model':
model['name']}, data))
return navigateKeys(navigateKeys, r.json(), (lambda x: ['choices', 0, 'text']
if model['mode'] == '' else ['choices', 0, 'message', 'content'])(0))
# print(f'Ran at {time.time()}')
r = makeResponse(#{'name': 'codestral-latest', 'mode': 'fim', 'provider':
'mistral'},
{'name': 'gpt-3.5-turbo-instruct', 'mode': '', 'provider': 'openai'},
{'prompt':'What did Sally', 'suffix':' his dog like that?',
'temperature':0.7, 'max_tokens': 200})
print(r)
```
## Claude 3.5 Sonnet
Chat with Prefilling
```python
import requests, json, sys
APIKey = "sk-or-v1-abc"
URL = "https://openrouter.ai/api/v1" + "/chat/completions"
model = "anthropic/claude-3.5-sonnet"
headers = {'Authorization': f"Bearer {APIKey}", 'Content-Type': 'application/json'}
data = {'model':model, 'stream': True, 'messages': [{'role': 'user', 'content': '''
Can you write a Python script to go through an arbitrary file, find all closed XML
blocks like `<obj type="c">...</obj>`, turn them all into dictionaries like
`{"tag": "obj", "type": "c"}`, and then give each dictionary a 'parent' pointer and
'children' list of pointers to the others in a way that represents the tree
structure of tag inclusions?
Making the dictionaries should be straightforward, but for formatting them into a
tree, I think you can do the following:
go through all the block <starts> and </ends> in order and construct a stack S:
when you encounter the nth opening <t_n>, initialize its dictionary D_n, do
D_n.parent = S.head and S.children.append(D_n), and then S.push(D_n);
whenever you encounter a closing </t>, assert t == S.head.tag (since otherwise the
XML is misformatted, right?) and then S.pop().
If this seems right, can you go ahead and implement it?
'''.strip()}]}
response = requests.post(URL, headers=headers, json=data, stream=True).iter_lines()
for chunk in response:
if chunk and b'data: {' in chunk:
print(json.loads(chunk[6:])['choices'][0]['delta']['content'], end='')
print()
```
## Groq
```python
import requests
APIKey = "abc"
URL = "https://api.groq.com/openai/v1" + "/chat/completions"
Model = "llama3-70b-8192"
headers = {'Authorization': f"Bearer {APIKey}",
'Content-Type': 'application/json'}
data = {'model': Model,
'stream': False,
'messages': [
{'role': 'user', 'content': 'What does kubernetes actually do?'}
]}
response = requests.post(URL, headers=headers, json=data)
print(response.json()['choices'][0]['message']['content'])
```
# Misc.
