Repository navigation
Expand file tree
/
Copy pathrun-with-viztracer
More file actions
executable file
·69 lines (54 loc) · 1.96 KB
/
Copy pathrun-with-viztracer
File metadata and controls
executable file
·69 lines (54 loc) · 1.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -o pipefail
set -e
FWDIR="$(cd "`dirname $0`"; pwd)"
# Function to display help message
function usage {
cat <<EOF
Usage:
$(basename $0) your_original_commands
To view the profiling results, run:
vizviewer pyspark_*.json
Environment:
If SPARK_VIZTRACER_OUTPUT_DIR is set, the output will be saved to the directory.
Otherwise, it will be saved to the current directory.
Requirements:
- viztracer must be installed (pip install viztracer)
Check the following documentation for more information on using viztracer:
https://viztracer.readthedocs.io/en/latest/
Examples:
- Start pyspark shell
$0 bin/pyspark --conf spark.driver.memory=16g
- Start pyspark shell in Connect mode
$0 bin/pyspark --remote local
EOF
exit 0
}
# Check for help flags or no arguments
if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
usage
fi
if ! hash viztracer 2>/dev/null; then
echo "Error: viztracer is not installed. Please install it using 'pip install viztracer'."
exit 1
fi
export PYTHONPATH="$FWDIR/conf_viztracer:$PYTHONPATH"
export SPARK_CONF_DIR="$FWDIR/conf_viztracer"
export SPARK_VIZTRACER_OUTPUT_DIR="${SPARK_VIZTRACER_OUTPUT_DIR:-"$(pwd)"}"
exec "$@"