Visitar URL original
Import metrics_center (#268) · creatorpiyush/packages@2cba790 · GitHub
Skip to content

Commit 2cba790

Browse files
authored
Import metrics_center (flutter#268)
1 parent e17e625 commit 2cba790

25 files changed

Lines changed: 1932 additions & 1 deletion

‎analysis_options.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ linter:
3838
- always_specify_types
3939
- annotate_overrides
4040
# - avoid_annotating_with_dynamic # conflicts with always_specify_types
41-
- avoid_as
41+
# - avoid_as # conflicts with NNBD
4242
- avoid_bool_literals_in_conditional_expressions
4343
# - avoid_catches_without_on_clauses # we do this commonly
4444
# - avoid_catching_errors # we do this commonly

‎packages/metrics_center/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
secret
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# 0.0.4+1
2+
3+
- Moved to the `flutter/packages` repository

‎packages/metrics_center/LICENSE‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
Copyright 2014 The Flutter Authors. All rights reserved.
2+
3+
Redistribution and use in source and binary forms, with or without modification,
4+
are permitted provided that the following conditions are met:
5+
6+
* Redistributions of source code must retain the above copyright
7+
notice, this list of conditions and the following disclaimer.
8+
* Redistributions in binary form must reproduce the above
9+
copyright notice, this list of conditions and the following
10+
disclaimer in the documentation and/or other materials provided
11+
with the distribution.
12+
* Neither the name of Google Inc. nor the names of its
13+
contributors may be used to endorse or promote products derived
14+
from this software without specific prior written permission.
15+
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
23+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎packages/metrics_center/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Metrics Center
2+
3+
Metrics center is a minimal set of code and services to support multiple perf
4+
metrics generators (e.g., Cocoon device lab, Cirrus bots, LUCI bots, Firebase
5+
Test Lab) and destinations (e.g., old Cocoon perf dashboard, Skia perf
6+
dashboard). The work and maintenance it requires is very close to that of just
7+
supporting a single generator and destination (e.g., engine bots to Skia perf),
8+
and the small amount of extra work is designed to make it easy to support more
9+
generators and destinations in the future.
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
export 'src/common.dart';
6+
export 'src/flutter.dart';
7+
export 'src/google_benchmark.dart';
8+
export 'src/skiaperf.dart';
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:collection';
6+
import 'dart:convert';
7+
8+
import 'package:crypto/crypto.dart';
9+
import 'package:equatable/equatable.dart';
10+
11+
import 'package:googleapis_auth/auth.dart';
12+
import 'package:googleapis_auth/auth_io.dart';
13+
import 'package:http/http.dart';
14+
15+
/// Common format of a metric data point.
16+
class MetricPoint extends Equatable {
17+
/// Creates a new data point.
18+
MetricPoint(
19+
this.value,
20+
Map<String, String> tags,
21+
) : _tags = SplayTreeMap<String, String>.from(tags);
22+
23+
/// Can store integer values.
24+
final double value;
25+
26+
/// Test name, unit, timestamp, configs, git revision, ..., in sorted order.
27+
UnmodifiableMapView<String, String> get tags =>
28+
UnmodifiableMapView<String, String>(_tags);
29+
30+
/// Unique identifier for updating existing data point.
31+
///
32+
/// We shouldn't have to worry about hash collisions until we have about
33+
/// 2^128 points.
34+
///
35+
/// This id should stay constant even if the [tags.keys] are reordered.
36+
/// (Because we are using an ordered SplayTreeMap to generate the id.)
37+
String get id => sha256.convert(utf8.encode('$_tags')).toString();
38+
39+
@override
40+
String toString() {
41+
return 'MetricPoint(value=$value, tags=$_tags)';
42+
}
43+
44+
final SplayTreeMap<String, String> _tags;
45+
46+
@override
47+
List<Object> get props => <Object>[value, tags];
48+
}
49+
50+
/// Interface to write [MetricPoint].
51+
abstract class MetricDestination {
52+
/// Insert new data points or modify old ones with matching id.
53+
Future<void> update(List<MetricPoint> points);
54+
}
55+
56+
/// Create `AuthClient` in case we only have an access token without the full
57+
/// credentials json. It's currently the case for Chrmoium LUCI bots.
58+
AuthClient authClientFromAccessToken(String token, List<String> scopes) {
59+
final DateTime anHourLater = DateTime.now().add(const Duration(hours: 1));
60+
final AccessToken accessToken =
61+
AccessToken('Bearer', token, anHourLater.toUtc());
62+
final AccessCredentials accessCredentials =
63+
AccessCredentials(accessToken, null, scopes);
64+
return authenticatedClient(Client(), accessCredentials);
65+
}
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
/// Strings used for MetricPoint tag keys
6+
const String kGithubRepoKey = 'gitRepo';
7+
8+
/// Strings used for MetricPoint tag keys
9+
const String kGitRevisionKey = 'gitRevision';
10+
11+
/// Strings used for MetricPoint tag keys
12+
const String kUnitKey = 'unit';
13+
14+
/// Strings used for MetricPoint tag keys
15+
const String kNameKey = 'name';
16+
17+
/// Strings used for MetricPoint tag keys
18+
const String kSubResultKey = 'subResult';
19+
20+
/// Flutter repo name.
21+
const String kFlutterFrameworkRepo = 'flutter/flutter';
22+
23+
/// Engine repo name.
24+
const String kFlutterEngineRepo = 'flutter/engine';
25+
26+
/// The key for the GCP project id in the credentials json.
27+
const String kProjectId = 'project_id';
28+
29+
/// Timeline key in JSON.
30+
const String kSourceTimeMicrosName = 'sourceTimeMicros';
31+
32+
/// The size of 500 is currently limited by Google datastore. It cannot write
33+
/// more than 500 entities in a single call.
34+
const int kMaxBatchSize = 500;
35+
36+
/// The prod bucket name for Flutter's instance of Skia Perf.
37+
const String kBucketName = 'flutter-skia-perf-prod';
38+
39+
/// The test bucket name for Flutter's instance of Skia Perf.
40+
const String kTestBucketName = 'flutter-skia-perf-test';
41+
42+
/// JSON key for Skia Perf's git hash entry.
43+
const String kSkiaPerfGitHashKey = 'gitHash';
44+
45+
/// JSON key for Skia Perf's results entry.
46+
const String kSkiaPerfResultsKey = 'results';
47+
48+
/// JSON key for Skia Perf's value entry.
49+
const String kSkiaPerfValueKey = 'value';
50+
51+
/// JSON key for Skia Perf's options entry.
52+
const String kSkiaPerfOptionsKey = 'options';
53+
54+
/// JSON key for Skia Perf's default config entry.
55+
const String kSkiaPerfDefaultConfig = 'default';
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'common.dart';
6+
import 'constants.dart';
7+
import 'legacy_datastore.dart';
8+
import 'legacy_flutter.dart';
9+
10+
/// Convenient class to capture the benchmarks in the Flutter engine repo.
11+
class FlutterEngineMetricPoint extends MetricPoint {
12+
/// Creates a metric point for the Flutter engine repository.
13+
///
14+
/// The `name`, `value`, and `gitRevision` parameters must not be null.
15+
FlutterEngineMetricPoint(
16+
String name,
17+
double value,
18+
String gitRevision, {
19+
Map<String, String> moreTags = const <String, String>{},
20+
}) : super(
21+
value,
22+
<String, String>{
23+
kNameKey: name,
24+
kGithubRepoKey: kFlutterEngineRepo,
25+
kGitRevisionKey: gitRevision,
26+
}..addAll(moreTags),
27+
);
28+
}
29+
30+
/// All Flutter performance metrics (framework, engine, ...) should be written
31+
/// to this destination.
32+
class FlutterDestination extends MetricDestination {
33+
// TODO(liyuqian): change the implementation of this class (without changing
34+
// its public APIs) to remove `LegacyFlutterDestination` and directly use
35+
// `SkiaPerfDestination` once the migration is fully done.
36+
FlutterDestination._(this._legacyDestination);
37+
38+
/// Creates a [FlutterDestination] from service account JSON.
39+
static Future<FlutterDestination> makeFromCredentialsJson(
40+
Map<String, dynamic> json) async {
41+
final LegacyFlutterDestination legacyDestination =
42+
LegacyFlutterDestination(await datastoreFromCredentialsJson(json));
43+
return FlutterDestination._(legacyDestination);
44+
}
45+
46+
/// Creates a [FlutterDestination] from an OAuth access token.
47+
static FlutterDestination makeFromAccessToken(
48+
String accessToken, String projectId) {
49+
final LegacyFlutterDestination legacyDestination = LegacyFlutterDestination(
50+
datastoreFromAccessToken(accessToken, projectId));
51+
return FlutterDestination._(legacyDestination);
52+
}
53+
54+
@override
55+
Future<void> update(List<MetricPoint> points) async {
56+
await _legacyDestination.update(points);
57+
}
58+
59+
final LegacyFlutterDestination _legacyDestination;
60+
}
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'package:googleapis/storage/v1.dart';
6+
import 'package:googleapis_auth/auth.dart';
7+
8+
/// Global (in terms of earth) mutex using Google Cloud Storage.
9+
class GcsLock {
10+
/// Create a lock with an authenticated client and a GCS bucket name.
11+
///
12+
/// The client is used to communicate with Google Cloud Storage APIs.
13+
GcsLock(this._client, this._bucketName)
14+
: assert(_client != null),
15+
assert(_bucketName != null) {
16+
_api = StorageApi(_client);
17+
}
18+
19+
/// Create a temporary lock file in GCS, and use it as a mutex mechanism to
20+
/// run a piece of code exclusively.
21+
///
22+
/// There must be no existing lock file with the same name in order to
23+
/// proceed. If multiple [GcsLock]s with the same `bucketName` and
24+
/// `lockFileName` try [protectedRun] simultaneously, only one will proceed
25+
/// and create the lock file. All others will be blocked.
26+
///
27+
/// When [protectedRun] finishes, the lock file is deleted, and other blocked
28+
/// [protectedRun] may proceed.
29+
///
30+
/// If the lock file is stuck (e.g., `_unlock` is interrupted unexpectedly),
31+
/// one may need to manually delete the lock file from GCS to unblock any
32+
/// [protectedRun] that may depend on it.
33+
Future<void> protectedRun(String lockFileName, Future<void> f()) async {
34+
await _lock(lockFileName);
35+
try {
36+
await f();
37+
} catch (e, stacktrace) {
38+
print(stacktrace);
39+
rethrow;
40+
} finally {
41+
await _unlock(lockFileName);
42+
}
43+
}
44+
45+
Future<void> _lock(String lockFileName) async {
46+
final Object object = Object();
47+
object.bucket = _bucketName;
48+
object.name = lockFileName;
49+
final Media content = Media(const Stream<List<int>>.empty(), 0);
50+
51+
Duration waitPeriod = const Duration(milliseconds: 10);
52+
bool locked = false;
53+
while (!locked) {
54+
try {
55+
await _api.objects.insert(object, _bucketName,
56+
ifGenerationMatch: '0', uploadMedia: content);
57+
locked = true;
58+
} on DetailedApiRequestError catch (e) {
59+
if (e.status == 412) {
60+
// Status 412 means that the lock file already exists. Wait until
61+
// that lock file is deleted.
62+
await Future<void>.delayed(waitPeriod);
63+
waitPeriod *= 2;
64+
if (waitPeriod >= _kWarningThreshold) {
65+
print(
66+
'The lock is waiting for a long time: $waitPeriod. '
67+
'If the lock file $lockFileName in bucket $_bucketName '
68+
'seems to be stuck (i.e., it was created a long time ago and '
69+
'no one seems to be owning it currently), delete it manually '
70+
'to unblock this.',
71+
);
72+
}
73+
} else {
74+
rethrow;
75+
}
76+
}
77+
}
78+
}
79+
80+
Future<void> _unlock(String lockFileName) async {
81+
await _api.objects.delete(_bucketName, lockFileName);
82+
}
83+
84+
StorageApi _api;
85+
86+
final String _bucketName;
87+
final AuthClient _client;
88+
89+
static const Duration _kWarningThreshold = Duration(seconds: 10);
90+
}

0 commit comments

Comments
 (0)