spark能传递外部命名参数给main函数吗?

时间:2022-08-25 21:28:29

查了资料好像都没有办法。只能通过:

  def main(args: Array[String]): Unit = {
    // 读取参数
    var city = args(0)
    var input = args(1)
    var date = args(2)

下标来获取。不过不确定,去*发帖问下。

https://*.com/questions/46845292/can-spark-submit-with-named-argument/46845525#46845525

确实不能,如果要的话,就需要自己写代码来解析:

object CommandLineUtil {

  def getOpts(args: Array[String], usage: String): collection.mutable.Map[String, String] = {
    if (args.length == 0) {
      log.warn(usage)
      System.exit(1)
    }

    val (opts, vals) = args.partition {
      _.startsWith("-")
    }

    val optsMap = collection.mutable.Map[String, String]()
    opts.map { x =>
      val pair = x.split("=")
      if (pair.length == 2) {
        optsMap += (pair(0).split("-{1,2}")(1) -> pair(1))
      } else {
        log.warn(usage)
        System.exit(1)
      }
    }

    optsMap
  }
}
val usage = "Usage:  [--citys] [--num]"
val optsMap = CommandLineUtil.getOpts(args, usage)
val citysValue = optsMap("citys")
val numValue = optsMap("num")