spark_test.py 877 B

12345678910111213141516171819202122232425
  1. #
  2. # Copyright 2019 The FATE Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. #
  16. from pyspark import SparkContext
  17. sc = SparkContext('local', 'test')
  18. a = []
  19. for i in range(10):
  20. a.append((i, str(i)))
  21. rdd1 = sc.parallelize(a)
  22. rdd2 = rdd1.mapValues(f=lambda x: x + "1")
  23. for k, v in rdd2.collect():
  24. print(f"{type(k)}: {k} {type(v)} {v}")