方法一:
from pyspark import SparkConf, SparkContext
conf = SparkConf().setAppName("isPrime")
sc = SparkContext(conf=conf)
def isPrime(n):
if n<2:
return False
if n==2:
return True
if not n&1:
return False
for i in range(3, int(n**0.5)+2, 2):
if n%i == 0:
return False
return True
#创建RDD
rdd = sc.parallelize(range(1000))
#过滤
result = rdd.filter(isPrime).collect()
print('='*30)
print(result)
方法二,空间占用大,不推荐:
from pyspark import SparkConf, SparkContext
conf = SparkConf().setAppName("isPrime")
sc = SparkContext(conf=conf)
n = 1000
m = int(n**0.5) + 1
rdd = sc.parallelize(range(2, n))
result = set()
while True:
#获取第一个元素
t = rdd.first()
if t > m:
break
result.add(t)
#对RDD上的所有元素进行过滤、筛选,能被整除的全部过滤掉
rdd = sc.parallelize(rdd.filter(lambda x: x%t != 0).collect())
print(list(result)+rdd.collect())