[Spark] Spark BlockMatrix Multiply 방법

Reference

• https://spark.apache.org/docs/latest/api/java/org/apache/spark/mllib/linalg/distributed/BlockMatrix.html
• https://github.com/apache/spark/blob/master/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala

예제 코드

import org.apache.spark.mllib.linalg.distributed.{CoordinateMatrix, MatrixEntry}

val lr = 1134890
val lc = 1134890
val rc = 1000
val lr_block = 3000
val lc_block = 2000
val rc_block = 1000

// COO 형식의 Left Matrix Data 를 RDD 로 저장
val l_rdd_string =  sc.textFile("s3://unho-spmm/real-matrix/com-youtube.matrix.txt", 40)
// RDD MatrixEntry 생성
val l_rdd_entries = l_rdd_string.map(x=>x.split(" ")).map(x => MatrixEntry(x(0).toInt, x(1).toInt, x(2).toDouble))
// RDD MatrixEntry 를 사용해 CoordinateMatrix 생성
val l_coo_matrix : CoordinateMatrix = new CoordinateMatrix(l_rdd_entries, lr, lc)
// CoordinateMatrix 를 BlockMatrix 로 변환
val l_block_matrix = l_coo_matrix.toBlockMatrix(lr_block, lc_block).cache

// COO 형식의 Right Matrix Data 를 RDD 로 저장
val r_rdd_string =  sc.textFile("s3://unho-spmm/real-matrix/com-youtube-right-matrix-rc-1000-rnnz-322380955.txt", 40)
// RDD MatrixEntry 생성
val r_rdd_entries = r_rdd_string.map(x=>x.split(" ")).map(x => MatrixEntry(x(0).toInt, x(1).toInt, x(2).toDouble))
// RDD MatrixEntry 를 사용해 CoordinateMatrix 생성
val r_coo_matrix : CoordinateMatrix = new CoordinateMatrix(r_rdd_entries, lc, rc)
// CoordinateMatrix 를 BlockMatrix 로 변환
val r_block_matrix = r_coo_matrix.toBlockMatrix(lc_block, rc_block).cache

// BlockMatrix Multiply (Transformation)
val midsplits = 1000
val result_matrix = l_block_matrix.multiply(r_block_matrix, midsplits)

// BlockMatrix Multiply (Action)
val result_matrix_block_count = result_matrix.validate