@@ -138,3 +138,143 @@ jobs:
138138 else
139139 echo "Quick benchmark not found, skipping"
140140 fi
141+
142+ # Real API benchmark - only runs when API keys are available
143+ # Triggered manually or on main branch with secrets
144+ real-api-benchmark :
145+ runs-on : ubuntu-latest
146+ timeout-minutes : 15
147+ if : github.event_name == 'workflow_dispatch' || (github.ref == 'refs/heads/main' && github.event_name == 'push')
148+
149+ steps :
150+ - uses : actions/checkout@v4
151+
152+ - name : Set up Python
153+ uses : actions/setup-python@v5
154+ with :
155+ python-version : ' 3.11'
156+ cache : ' pip'
157+
158+ - name : Install package
159+ run : |
160+ pip install -e .
161+ pip install pytest
162+
163+ - name : Check API Key Availability
164+ id : check_keys
165+ run : |
166+ if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then
167+ echo "has_openai=true" >> $GITHUB_OUTPUT
168+ echo "✅ OpenAI API key available"
169+ else
170+ echo "has_openai=false" >> $GITHUB_OUTPUT
171+ echo "⚠️ OpenAI API key not available, skipping real API tests"
172+ fi
173+
174+ - name : Run Real API Benchmark
175+ if : steps.check_keys.outputs.has_openai == 'true'
176+ env :
177+ OPENAI_API_KEY : ${{ secrets.OPENAI_API_KEY }}
178+ run : |
179+ python -c "
180+ import time
181+ import os
182+
183+ # Verify API key is set
184+ if not os.environ.get('OPENAI_API_KEY'):
185+ print('❌ OPENAI_API_KEY not set')
186+ exit(1)
187+
188+ from praisonaiagents import Agent
189+
190+ # Create a simple agent
191+ agent = Agent(
192+ name='BenchmarkAgent',
193+ instructions='You are a helpful assistant. Be concise.',
194+ llm='gpt-4o-mini',
195+ output='silent'
196+ )
197+
198+ # Warmup run
199+ print('Running warmup...')
200+ agent.chat('Say hello in one word')
201+
202+ # Benchmark runs
203+ print('Running benchmark (3 iterations)...')
204+ times = []
205+ for i in range(3):
206+ start = time.perf_counter()
207+ response = agent.chat('What is 2+2? Answer with just the number.')
208+ elapsed = time.perf_counter() - start
209+ times.append(elapsed)
210+ print(f' Run {i+1}: {elapsed:.2f}s')
211+
212+ avg = sum(times) / len(times)
213+ print(f'')
214+ print(f'Real API Benchmark Results:')
215+ print(f' Average response time: {avg:.2f}s')
216+ print(f' Min: {min(times):.2f}s')
217+ print(f' Max: {max(times):.2f}s')
218+
219+ # Verify response quality
220+ if '4' in response:
221+ print('✅ Response quality check passed')
222+ else:
223+ print(f'⚠️ Unexpected response: {response}')
224+ "
225+
226+ - name : Run Multi-Agent Benchmark
227+ if : steps.check_keys.outputs.has_openai == 'true'
228+ env :
229+ OPENAI_API_KEY : ${{ secrets.OPENAI_API_KEY }}
230+ run : |
231+ python -c "
232+ import time
233+ import os
234+
235+ from praisonaiagents import Agent, Agents, Task
236+
237+ # Create agents
238+ researcher = Agent(
239+ name='Researcher',
240+ instructions='You research topics. Be brief.',
241+ llm='gpt-4o-mini',
242+ output='silent'
243+ )
244+
245+ writer = Agent(
246+ name='Writer',
247+ instructions='You write summaries. Be brief.',
248+ llm='gpt-4o-mini',
249+ output='silent'
250+ )
251+
252+ # Create tasks
253+ task1 = Task(
254+ description='List 2 facts about Python programming',
255+ agent=researcher,
256+ expected_output='2 brief facts'
257+ )
258+
259+ task2 = Task(
260+ description='Summarize the facts in one sentence',
261+ agent=writer,
262+ expected_output='One sentence summary'
263+ )
264+
265+ # Run workflow
266+ print('Running multi-agent workflow...')
267+ start = time.perf_counter()
268+
269+ agents = Agents(agents=[researcher, writer], tasks=[task1, task2])
270+ result = agents.start()
271+
272+ elapsed = time.perf_counter() - start
273+
274+ print(f'')
275+ print(f'Multi-Agent Benchmark Results:')
276+ print(f' Total workflow time: {elapsed:.2f}s')
277+ print(f' Agents: 2')
278+ print(f' Tasks: 2')
279+ print('✅ Multi-agent workflow completed successfully')
280+ "
0 commit comments