Visitar URL original
feat(firebaseai): make Live API working with developer API (#17503) · firebase/flutterfire@467eaa1 · GitHub
Skip to content

Commit 467eaa1

Browse files
cynthiajoangoogle-labs-jules[bot]
cynthiajoan
and
google-labs-jules[bot]
authored
feat(firebaseai): make Live API working with developer API (#17503)
* add developer api to live * make the whole process going through * feat(firebase_ai): handle unknown parts when parsing content * tweak the content test * remove the extra exception * Make Live API sending tool response for function calling * update bidi model for googleAI in example * fix analyzer and test * more test fixing --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
1 parent 0c3ccd3 commit 467eaa1

11 files changed

Lines changed: 128 additions & 66 deletions

File tree

‎packages/firebase_ai/firebase_ai/example/lib/main.dart‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -154,10 +154,6 @@ class HomeScreen extends StatefulWidget {
154154

155155
class _HomeScreenState extends State<HomeScreen> {
156156
void _onItemTapped(int index) {
157-
if (index == 9 && !widget.useVertexBackend) {
158-
// Live Stream feature only works with Vertex AI now.
159-
return;
160-
}
161157
widget.onSelectedIndexChanged(index);
162158
}
163159

@@ -192,12 +188,12 @@ class _HomeScreenState extends State<HomeScreen> {
192188
case 8:
193189
return VideoPage(title: 'Video Prompt', model: currentModel);
194190
case 9:
195-
if (useVertexBackend) {
196-
return BidiPage(title: 'Live Stream', model: currentModel);
197-
} else {
198-
// Fallback to the first page in case of an unexpected index
199-
return ChatPage(title: 'Chat', model: currentModel);
200-
}
191+
return BidiPage(
192+
title: 'Live Stream',
193+
model: currentModel,
194+
useVertexBackend: useVertexBackend,
195+
);
196+
201197
default:
202198
// Fallback to the first page in case of an unexpected index
203199
return ChatPage(title: 'Chat', model: currentModel);
@@ -270,61 +266,58 @@ class _HomeScreenState extends State<HomeScreen> {
270266
unselectedItemColor: widget.useVertexBackend
271267
? Theme.of(context).colorScheme.onSurface.withValues(alpha: 0.7)
272268
: Colors.grey,
273-
items: <BottomNavigationBarItem>[
274-
const BottomNavigationBarItem(
269+
items: const <BottomNavigationBarItem>[
270+
BottomNavigationBarItem(
275271
icon: Icon(Icons.chat),
276272
label: 'Chat',
277273
tooltip: 'Chat',
278274
),
279-
const BottomNavigationBarItem(
275+
BottomNavigationBarItem(
280276
icon: Icon(Icons.mic),
281277
label: 'Audio',
282278
tooltip: 'Audio Prompt',
283279
),
284-
const BottomNavigationBarItem(
280+
BottomNavigationBarItem(
285281
icon: Icon(Icons.numbers),
286282
label: 'Tokens',
287283
tooltip: 'Token Count',
288284
),
289-
const BottomNavigationBarItem(
285+
BottomNavigationBarItem(
290286
icon: Icon(Icons.functions),
291287
label: 'Functions',
292288
tooltip: 'Function Calling',
293289
),
294-
const BottomNavigationBarItem(
290+
BottomNavigationBarItem(
295291
icon: Icon(Icons.image),
296292
label: 'Image',
297293
tooltip: 'Image Prompt',
298294
),
299-
const BottomNavigationBarItem(
295+
BottomNavigationBarItem(
300296
icon: Icon(Icons.image_search),
301297
label: 'Imagen',
302298
tooltip: 'Imagen Model',
303299
),
304-
const BottomNavigationBarItem(
300+
BottomNavigationBarItem(
305301
icon: Icon(Icons.schema),
306302
label: 'Schema',
307303
tooltip: 'Schema Prompt',
308304
),
309-
const BottomNavigationBarItem(
305+
BottomNavigationBarItem(
310306
icon: Icon(Icons.edit_document),
311307
label: 'Document',
312308
tooltip: 'Document Prompt',
313309
),
314-
const BottomNavigationBarItem(
310+
BottomNavigationBarItem(
315311
icon: Icon(Icons.video_collection),
316312
label: 'Video',
317313
tooltip: 'Video Prompt',
318314
),
319315
BottomNavigationBarItem(
320316
icon: Icon(
321317
Icons.stream,
322-
color: widget.useVertexBackend ? null : Colors.grey,
323318
),
324319
label: 'Live',
325-
tooltip: widget.useVertexBackend
326-
? 'Live Stream'
327-
: 'Live Stream (Currently Disabled)',
320+
tooltip: 'Live Stream',
328321
),
329322
],
330323
currentIndex: widget.selectedIndex,

‎packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,16 @@ import '../utils/audio_output.dart';
2222
import '../widgets/message_widget.dart';
2323

2424
class BidiPage extends StatefulWidget {
25-
const BidiPage({super.key, required this.title, required this.model});
25+
const BidiPage({
26+
super.key,
27+
required this.title,
28+
required this.model,
29+
required this.useVertexBackend,
30+
});
2631

2732
final String title;
2833
final GenerativeModel model;
34+
final bool useVertexBackend;
2935

3036
@override
3137
State<BidiPage> createState() => _BidiPageState();
@@ -64,13 +70,21 @@ class _BidiPageState extends State<BidiPage> {
6470
);
6571

6672
// ignore: deprecated_member_use
67-
_liveModel = FirebaseAI.vertexAI().liveGenerativeModel(
68-
model: 'gemini-2.0-flash-exp',
69-
liveGenerationConfig: config,
70-
tools: [
71-
Tool.functionDeclarations([lightControlTool]),
72-
],
73-
);
73+
_liveModel = widget.useVertexBackend
74+
? FirebaseAI.vertexAI().liveGenerativeModel(
75+
model: 'gemini-2.0-flash-exp',
76+
liveGenerationConfig: config,
77+
tools: [
78+
Tool.functionDeclarations([lightControlTool]),
79+
],
80+
)
81+
: FirebaseAI.googleAI().liveGenerativeModel(
82+
model: 'gemini-live-2.5-flash-preview',
83+
liveGenerationConfig: config,
84+
tools: [
85+
Tool.functionDeclarations([lightControlTool]),
86+
],
87+
);
7488
_initAudio();
7589
}
7690

@@ -389,9 +403,13 @@ class _BidiPageState extends State<BidiPage> {
389403
brightness: brightness,
390404
colorTemperature: color,
391405
);
392-
await _session.send(
393-
input: Content.functionResponse(functionCall.name, functionResult),
394-
);
406+
await _session.sendToolResponse([
407+
FunctionResponse(
408+
functionCall.name,
409+
functionResult,
410+
id: functionCall.id,
411+
),
412+
]);
395413
} else {
396414
throw UnimplementedError(
397415
'Function not declared to the model: ${functionCall.name}',

‎packages/firebase_ai/firebase_ai/lib/src/base_model.dart‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ enum Task {
5656

5757
abstract interface class _ModelUri {
5858
String get baseAuthority;
59+
String get apiVersion;
5960
Uri taskUri(Task task);
6061
({String prefix, String name}) get model;
6162
}
@@ -96,6 +97,9 @@ final class _VertexUri implements _ModelUri {
9697
@override
9798
String get baseAuthority => _baseAuthority;
9899

100+
@override
101+
String get apiVersion => _apiVersion;
102+
99103
@override
100104
Uri taskUri(Task task) {
101105
return _projectUri.replace(
@@ -135,6 +139,9 @@ final class _GoogleAIUri implements _ModelUri {
135139
@override
136140
String get baseAuthority => _baseAuthority;
137141

142+
@override
143+
String get apiVersion => _apiVersion;
144+
138145
@override
139146
Uri taskUri(Task task) => _baseUri.replace(
140147
pathSegments: _baseUri.pathSegments

‎packages/firebase_ai/firebase_ai/lib/src/content.dart‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,9 @@ final class Content {
4848
static Content model(Iterable<Part> parts) => Content('model', [...parts]);
4949

5050
/// Return a [Content] with [FunctionResponse].
51-
static Content functionResponse(String name, Map<String, Object?> response) =>
52-
Content('function', [FunctionResponse(name, response)]);
51+
static Content functionResponse(String name, Map<String, Object?> response,
52+
{String? id}) =>
53+
Content('function', [FunctionResponse(name, response, id: id)]);
5354

5455
/// Return a [Content] with multiple [FunctionResponse].
5556
static Content functionResponses(Iterable<FunctionResponse> responses) =>

‎packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,14 +175,11 @@ class FirebaseAI extends FirebasePluginPlatform {
175175
List<Tool>? tools,
176176
Content? systemInstruction,
177177
}) {
178-
if (!_useVertexBackend) {
179-
throw FirebaseAISdkException(
180-
'LiveGenerativeModel is currently only supported with the VertexAI backend.');
181-
}
182178
return createLiveGenerativeModel(
183179
app: app,
184180
location: location,
185181
model: model,
182+
useVertexBackend: _useVertexBackend,
186183
liveGenerationConfig: liveGenerationConfig,
187184
tools: tools,
188185
systemInstruction: systemInstruction,

‎packages/firebase_ai/firebase_ai/lib/src/live_api.dart‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,15 @@ class LiveClientToolResponse {
234234
final List<FunctionResponse>? functionResponses;
235235
// ignore: public_member_api_docs
236236
Map<String, dynamic> toJson() => {
237-
'functionResponses': functionResponses?.map((e) => e.toJson()).toList(),
237+
'toolResponse': {
238+
'functionResponses': functionResponses
239+
?.map((e) => {
240+
'name': e.name,
241+
'response': e.response,
242+
if (e.id != null) 'id': e.id,
243+
})
244+
.toList(),
245+
},
238246
};
239247
}
240248

‎packages/firebase_ai/firebase_ai/lib/src/live_model.dart‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
part of 'base_model.dart';
1616

1717
const _apiUrl = 'ws/google.firebase.vertexai';
18-
const _apiUrlSuffix = 'LlmBidiService/BidiGenerateContent/locations';
18+
const _apiUrlSuffixVertexAI = 'LlmBidiService/BidiGenerateContent/locations';
19+
const _apiUrlSuffixGoogleAI = 'GenerativeService/BidiGenerateContent';
1920

2021
/// A live, generative AI model for real-time interaction.
2122
///
@@ -32,36 +33,56 @@ final class LiveGenerativeModel extends BaseModel {
3233
{required String model,
3334
required String location,
3435
required FirebaseApp app,
36+
required bool useVertexBackend,
3537
FirebaseAppCheck? appCheck,
3638
FirebaseAuth? auth,
3739
LiveGenerationConfig? liveGenerationConfig,
3840
List<Tool>? tools,
3941
Content? systemInstruction})
4042
: _app = app,
4143
_location = location,
44+
_useVertexBackend = useVertexBackend,
4245
_appCheck = appCheck,
4346
_auth = auth,
4447
_liveGenerationConfig = liveGenerationConfig,
4548
_tools = tools,
4649
_systemInstruction = systemInstruction,
4750
super._(
4851
serializationStrategy: VertexSerialization(),
49-
modelUri: _VertexUri(
50-
model: model,
51-
app: app,
52-
location: location,
53-
),
52+
modelUri: useVertexBackend
53+
? _VertexUri(
54+
model: model,
55+
app: app,
56+
location: location,
57+
)
58+
: _GoogleAIUri(
59+
model: model,
60+
app: app,
61+
),
5462
);
55-
static const _apiVersion = 'v1beta';
5663

5764
final FirebaseApp _app;
5865
final String _location;
66+
final bool _useVertexBackend;
5967
final FirebaseAppCheck? _appCheck;
6068
final FirebaseAuth? _auth;
6169
final LiveGenerationConfig? _liveGenerationConfig;
6270
final List<Tool>? _tools;
6371
final Content? _systemInstruction;
6472

73+
String _vertexAIUri() => 'wss://${_modelUri.baseAuthority}/'
74+
'$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixVertexAI/'
75+
'$_location?key=${_app.options.apiKey}';
76+
77+
String _vertexAIModelString() => 'projects/${_app.options.projectId}/'
78+
'locations/$_location/publishers/google/models/${model.name}';
79+
80+
String _googleAIUri() => 'wss://${_modelUri.baseAuthority}/'
81+
'$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixGoogleAI?key=${_app.options.apiKey}';
82+
83+
String _googleAIModelString() =>
84+
'projects/${_app.options.projectId}/models/${model.name}';
85+
6586
/// Establishes a connection to a live generation service.
6687
///
6788
/// This function handles the WebSocket connection setup and returns an [LiveSession]
@@ -70,11 +91,9 @@ final class LiveGenerativeModel extends BaseModel {
7091
/// Returns a [Future] that resolves to an [LiveSession] object upon successful
7192
/// connection.
7293
Future<LiveSession> connect() async {
73-
final uri = 'wss://${_modelUri.baseAuthority}/'
74-
'$_apiUrl.$_apiVersion.$_apiUrlSuffix/'
75-
'$_location?key=${_app.options.apiKey}';
76-
final modelString = 'projects/${_app.options.projectId}/'
77-
'locations/$_location/publishers/google/models/${model.name}';
94+
final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri();
95+
final modelString =
96+
_useVertexBackend ? _vertexAIModelString() : _googleAIModelString();
7897

7998
final setupJson = {
8099
'setup': {
@@ -96,6 +115,7 @@ final class LiveGenerativeModel extends BaseModel {
96115
await ws.ready;
97116

98117
ws.sink.add(request);
118+
99119
return LiveSession(ws);
100120
}
101121
}
@@ -105,6 +125,7 @@ LiveGenerativeModel createLiveGenerativeModel({
105125
required FirebaseApp app,
106126
required String location,
107127
required String model,
128+
required bool useVertexBackend,
108129
FirebaseAppCheck? appCheck,
109130
FirebaseAuth? auth,
110131
LiveGenerationConfig? liveGenerationConfig,
@@ -117,6 +138,7 @@ LiveGenerativeModel createLiveGenerativeModel({
117138
appCheck: appCheck,
118139
auth: auth,
119140
location: location,
141+
useVertexBackend: useVertexBackend,
120142
liveGenerationConfig: liveGenerationConfig,
121143
tools: tools,
122144
systemInstruction: systemInstruction,

‎packages/firebase_ai/firebase_ai/lib/src/live_session.dart‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,18 @@ class LiveSession {
6161
? LiveClientContent(turns: [input], turnComplete: turnComplete)
6262
: LiveClientContent(turnComplete: turnComplete);
6363
var clientJson = jsonEncode(clientMessage.toJson());
64+
_ws.sink.add(clientJson);
65+
}
6466

67+
/// Sends tool responses for function calling to the server.
68+
///
69+
/// [functionResponses] (optional): The list of function responses.
70+
Future<void> sendToolResponse(
71+
List<FunctionResponse>? functionResponses) async {
72+
final toolResponse =
73+
LiveClientToolResponse(functionResponses: functionResponses);
74+
_checkWsStatus();
75+
var clientJson = jsonEncode(toolResponse.toJson());
6576
_ws.sink.add(clientJson);
6677
}
6778

‎packages/firebase_ai/firebase_ai/test/live_test.dart‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -154,15 +154,17 @@ void main() {
154154
final response = FunctionResponse('test', {});
155155
final message = LiveClientToolResponse(functionResponses: [response]);
156156
expect(message.toJson(), {
157-
'functionResponses': [
158-
{
159-
'functionResponse': {'name': 'test', 'response': {}}
160-
}
161-
]
157+
'toolResponse': {
158+
'functionResponses': [
159+
{'name': 'test', 'response': {}}
160+
]
161+
}
162162
});
163163

164164
final message2 = LiveClientToolResponse();
165-
expect(message2.toJson(), {'functionResponses': null});
165+
expect(message2.toJson(), {
166+
'toolResponse': {'functionResponses': null}
167+
});
166168
});
167169

168170
test('parseServerMessage parses serverContent message correctly', () {

0 commit comments

Comments
 (0)