Sandbox::https://github.com/apache/spark/blob/master/dev/lint-python

Repository navigation

History

executable file

325 lines (274 loc) · 8.36 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements. See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License. You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# define test binaries + versions

RUFF_BUILD="ruff"

MINIMUM_RUFF="0.14.0"

MINIMUM_MYPY="1.19.1"

MYPY_BUILD="mypy"

PYTEST_BUILD="pytest"

PYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-python3}"

function exit_with_usage {

set +x

echo "lint-python - linter for Python"

echo ""

echo "usage:"

cl_options="[--compile] [--custom-pyspark-error] [--ruff] [--mypy] [--mypy-examples] [--mypy-data]"

echo "lint-python $cl_options"

echo ""

exit 1

}

# Parse arguments

while (( "$#" )); do

case $1 in

--compile)

COMPILE_TEST=true

;;

--custom-pyspark-error)

PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true

;;

--ruff)

RUFF_TEST=true

;;

--mypy)

MYPY_TEST=true

;;

--mypy-examples)

MYPY_EXAMPLES_TEST=true

;;

--mypy-data)

MYPY_DATA_TEST=true

;;

echo "Error: $1 is not supported"

exit_with_usage

;;

esac

shift

done

if [[ -z "$COMPILE_TEST$PYSPARK_CUSTOM_ERRORS_CHECK_TEST$RUFF_TEST$MYPY_TEST$MYPY_EXAMPLES_TEST$MYPY_DATA_TEST" ]]; then

COMPILE_TEST=true

PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true

RUFF_TEST=true

MYPY_TEST=true

MYPY_EXAMPLES_TEST=true

MYPY_DATA_TEST=true

function satisfies_min_version {

local provided_version="$1"

local expected_version="$2"

echo "$(

"$PYTHON_EXECUTABLE" << EOM

try:

from setuptools.extern.packaging import version

except ModuleNotFoundError:

from packaging import version

print(version.parse('$provided_version') >= version.parse('$expected_version'))

EOM

}

function compile_python_test {

local COMPILE_STATUS=

local COMPILE_REPORT=

if [[ ! "$1" ]]; then

echo "No python files found! Something is very wrong -- exiting."

exit 1;

# compileall: https://docs.python.org/3/library/compileall.html

echo "starting python compilation test..."

COMPILE_REPORT=$( ("$PYTHON_EXECUTABLE" -B -mcompileall -q -l -x "[/\\\\][.]git" $1) 2>&1)

COMPILE_STATUS=$?

if [ $COMPILE_STATUS -ne 0 ]; then

echo "Python compilation failed with the following errors:"

echo "$COMPILE_REPORT"

echo "$COMPILE_STATUS"

exit "$COMPILE_STATUS"

else

echo "python compilation succeeded."

echo

}

function mypy_annotation_test {

local MYPY_REPORT=

local MYPY_STATUS=

echo "starting mypy annotations test..."

MYPY_REPORT=$( ($MYPY_BUILD \

--python-executable $PYTHON_EXECUTABLE \

--namespace-packages \

--config-file python/mypy.ini \

--cache-dir /tmp/.mypy_cache/ \

python/pyspark) 2>&1)

MYPY_STATUS=$?

if [ "$MYPY_STATUS" -ne 0 ]; then

echo "annotations failed mypy checks:"

echo "$MYPY_REPORT"

echo "$MYPY_STATUS"

exit "$MYPY_STATUS"

else

echo "annotations passed mypy checks."

echo

}

function mypy_data_test {

local PYTEST_REPORT=

local PYTEST_STATUS=

echo "starting mypy data test..."

$PYTHON_EXECUTABLE -c "import importlib.util; import sys; \

sys.exit(0 if importlib.util.find_spec('pytest_mypy_plugins') else 1)"

if [ $? -ne 0 ]; then

echo "pytest-mypy-plugins missing. Skipping for now."

return

PYTEST_REPORT=$( (MYPYPATH=python $PYTEST_BUILD \

-o "python_files=test_*.yml" \

--rootdir python \

--mypy-only-local-stub \

--mypy-ini-file python/mypy.ini \

python/pyspark/tests/typing \

python/pyspark/sql/tests/typing \

python/pyspark/ml/tests/typing \

) 2>&1)

PYTEST_STATUS=$?

if [ "$PYTEST_STATUS" -ne 0 ]; then

echo "annotations failed data checks:"

echo "$PYTEST_REPORT"

echo "$PYTEST_STATUS"

exit "$PYTEST_STATUS"

else

echo "annotations passed data checks."

echo

}

function mypy_examples_test {

local MYPY_REPORT=

local MYPY_STATUS=

echo "starting mypy examples test..."

MYPY_REPORT=$( (MYPYPATH=python $MYPY_BUILD \

--python-executable $PYTHON_EXECUTABLE \

--namespace-packages \

--config-file python/mypy.ini \

--exclude "mllib/*" \

examples/src/main/python/) 2>&1)

MYPY_STATUS=$?

if [ "$MYPY_STATUS" -ne 0 ]; then

echo "examples failed mypy checks:"

echo "$MYPY_REPORT"

echo "$MYPY_STATUS"

exit "$MYPY_STATUS"

else

echo "examples passed mypy checks."

echo

}

function mypy_test {

if ! hash "$MYPY_BUILD" 2> /dev/null; then

echo "The $MYPY_BUILD command was not found. Skipping for now."

return

_MYPY_VERSION=($($MYPY_BUILD --version))

MYPY_VERSION="${_MYPY_VERSION[1]}"

EXPECTED_MYPY="$(satisfies_min_version $MYPY_VERSION $MINIMUM_MYPY)"

if [[ "$EXPECTED_MYPY" == "False" ]]; then

echo "The minimum mypy version needs to be $MINIMUM_MYPY. Your current version is $MYPY_VERSION. Skipping for now."

return

if [[ "$MYPY_TEST" == "true" ]]; then

mypy_annotation_test

if [[ "$MYPY_EXAMPLES_TEST" == "true" ]]; then

mypy_examples_test

if [[ "$MYPY_DATA_TEST" == "true" ]]; then

mypy_data_test

}

function ruff_test {

local RUFF_VERSION=

local EXPECTED_RUFF=

local RUFF_REPORT=

local RUFF_STATUS=

if ! hash "$RUFF_BUILD" 2> /dev/null; then

echo "The ruff command was not found. Skipping for now."

return

_RUFF_VERSION=($($RUFF_BUILD --version))

RUFF_VERSION="${_RUFF_VERSION[1]}"

EXPECTED_RUFF="$(satisfies_min_version $RUFF_VERSION $MINIMUM_RUFF)"

if [[ "$EXPECTED_RUFF" == "False" ]]; then

echo "\

The minimum ruff version needs to be $MINIMUM_RUFF. Your current version is $RUFF_VERSION

ruff checks failed."

exit 1

echo "starting $RUFF_BUILD test..."

RUFF_REPORT=$( ($RUFF_BUILD check) 2>&1)

RUFF_STATUS=$?

if [ "$RUFF_STATUS" -ne 0 ]; then

echo "ruff checks failed:"

echo "$RUFF_REPORT"

echo "$RUFF_STATUS"

exit "$RUFF_STATUS"

else

echo "ruff checks passed."

echo

RUFF_REPORT=$( ($RUFF_BUILD format --diff python/pyspark dev python/packaging python/benchmarks) 2>&1)

RUFF_STATUS=$?

if [ "$RUFF_STATUS" -ne 0 ]; then

echo "ruff format checks failed:"

echo "$RUFF_REPORT"

echo "$RUFF_STATUS"

exit "$RUFF_STATUS"

else

echo "ruff format checks passed."

echo

}

function pyspark_custom_errors_check_test {

local PYSPARK_CUSTOM_ERRORS_CHECK_REPORT=

local PYSPARK_CUSTOM_ERRORS_CHECK_STATUS=

echo "starting PySpark custom errors check..."

PYSPARK_CUSTOM_ERRORS_CHECK_REPORT=$( "$PYTHON_EXECUTABLE" dev/check_pyspark_custom_errors.py 2>&1 )

PYSPARK_CUSTOM_ERRORS_CHECK_STATUS=$?

if [ "$PYSPARK_CUSTOM_ERRORS_CHECK_STATUS" -ne 0 ]; then

echo "PySpark custom errors check failed!"

echo "$PYSPARK_CUSTOM_ERRORS_CHECK_REPORT"

exit "$PYSPARK_CUSTOM_ERRORS_CHECK_STATUS"

else

echo "PySpark custom errors check passed."

echo

}

SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"

SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"

pushd "$SPARK_ROOT_DIR" &> /dev/null

PYTHON_SOURCE="$(git ls-files '*.py')"

if [[ "$COMPILE_TEST" == "true" ]]; then

compile_python_test "$PYTHON_SOURCE"

if [[ "$PYSPARK_CUSTOM_ERRORS_CHECK_TEST" == "true" ]]; then

pyspark_custom_errors_check_test

if [[ "$RUFF_TEST" == "true" ]]; then

ruff_test

if [[ "$MYPY_TEST" == "true" ]] || [[ "$MYPY_EXAMPLES_TEST" == "true" ]] || [[ "$MYPY_DATA_TEST" == "true" ]]; then

mypy_test

echo

echo "all lint-python tests passed!"

popd &> /dev/null

FilesExpand file tree

lint-python

Latest commit

History

lint-python

File metadata and controls