import large data to JanusGraph vertex label issue

[cvamy11]

mgmt = graph.openManagement();
mgmt.makeVertexLabel("Account").make();
mgmt.makeVertexLabel("Address").make();
mgmt.makeVertexLabel("BankCard").make();
mgmt.makeVertexLabel("Company").make();
mgmt.makeVertexLabel("Device").make();
mgmt.makeVertexLabel("IdCard").make();
mgmt.makeVertexLabel("IMSI").make();
mgmt.makeVertexLabel("Mobile").make();
mgmt.makeVertexLabel("WMAC").make();
mgmt.makeVertexLabel("Password").make();
mgmt.makeVertexLabel("IFAADevice").make();
mgmt.makeVertexLabel("Merchant").make();
mgmt.makeVertexLabel("PubAddress").make();
mgmt.commit();

mgmt = graph.openManagement();
mgmt.makePropertyKey("account_no").dataType(String.class).make();
mgmt.makePropertyKey("acct_name").dataType(String.class).make();
mgmt.makePropertyKey("name").dataType(String.class).make();
mgmt.makePropertyKey("mbl").dataType(String.class).make();
mgmt.makePropertyKey("update_time").dataType(String.class).make();
mgmt.makePropertyKey("id_card").dataType(String.class).make();
mgmt.makePropertyKey("auth_flag").dataType(String.class).make();
mgmt.makePropertyKey("auth_date").dataType(String.class).make();
mgmt.makePropertyKey("register_time").dataType(String.class).make();
mgmt.makePropertyKey("cust_fund").dataType(String.class).make();
mgmt.makePropertyKey("cert_attribution").dataType(String.class).make();
mgmt.makePropertyKey("tags").dataType(String.class).make();
mgmt.makePropertyKey("acct_status").dataType(String.class).make();
mgmt.makePropertyKey("cancel").dataType(String.class).make();
mgmt.makePropertyKey("id_no").dataType(String.class).make();
mgmt.makePropertyKey("address").dataType(String.class).make();
mgmt.makePropertyKey("addr_flag").dataType(String.class).make();
mgmt.makePropertyKey("bank_card_no").dataType(String.class).make();
mgmt.makePropertyKey("net_acct_type").dataType(String.class).make();
mgmt.makePropertyKey("net_cntr_name").dataType(String.class).make();
mgmt.makePropertyKey("bank_code").dataType(String.class).make();
mgmt.makePropertyKey("company").dataType(String.class).make();
mgmt.makePropertyKey("com_addressess").dataType(String.class).make();
mgmt.makePropertyKey("device_id").dataType(String.class).make();
mgmt.makePropertyKey("devalias").dataType(String.class).make();
mgmt.makePropertyKey("terminal").dataType(String.class).make();
mgmt.makePropertyKey("cert_code").dataType(String.class).make();
mgmt.makePropertyKey("user_sex").dataType(String.class).make();
mgmt.makePropertyKey("work_unit").dataType(String.class).make();
mgmt.makePropertyKey("cash_loan_amt").dataType(String.class).make();
mgmt.makePropertyKey("credit_loan_amt").dataType(String.class).make();
mgmt.makePropertyKey("max_overdue_days").dataType(String.class).make();
mgmt.makePropertyKey("overdue_amount").dataType(String.class).make();
mgmt.makePropertyKey("imsi").dataType(String.class).make();
mgmt.makePropertyKey("mobile").dataType(String.class).make();
mgmt.makePropertyKey("mbl_attribution").dataType(String.class).make();
mgmt.makePropertyKey("wmac").dataType(String.class).make();
mgmt.makePropertyKey("ts_pwd").dataType(String.class).make();
mgmt.makePropertyKey("ts_ym").dataType(String.class).make();
mgmt.makePropertyKey("ts_degree").dataType(String.class).make();
mgmt.makePropertyKey("cancel_date").dataType(String.class).make();
mgmt.makePropertyKey("post_addressess").dataType(String.class).make();
mgmt.makePropertyKey("registered_address").dataType(String.class).make();
mgmt.makePropertyKey("ifaa_device_id").dataType(String.class).make();
mgmt.makePropertyKey("biz_no").dataType(String.class).make();
mgmt.makePropertyKey("biz_name").dataType(String.class).make();
mgmt.makePropertyKey("join_date").dataType(String.class).make();
mgmt.makePropertyKey("is_retreat").dataType(String.class).make();
mgmt.makePropertyKey("pub_address").dataType(String.class).make();
mgmt.makePropertyKey("apply_count").dataType(String.class).make()
mgmt.makePropertyKey("last_time").dataType(String.class).make()
mgmt.makePropertyKey("lt1y_flag").dataType(String.class).make()
mgmt.makePropertyKey("pay_count").dataType(String.class).make()
mgmt.makePropertyKey("total_amt").dataType(String.class).make()
mgmt.makePropertyKey("offline_count").dataType(String.class).make()
mgmt.makePropertyKey("online_count").dataType(String.class).make()
mgmt.makePropertyKey("last_amt").dataType(String.class).make()
mgmt.makePropertyKey("phf_work").dataType(String.class).make()
mgmt.makePropertyKey("login_time").dataType(String.class).make()
mgmt.makePropertyKey("login_days").dataType(String.class).make()
mgmt.makePropertyKey("start_year").dataType(String.class).make()
mgmt.makePropertyKey("spouse_cert_code").dataType(String.class).make()
mgmt.makePropertyKey("withdraw_count").dataType(String.class).make()
mgmt.makePropertyKey("count_ttl").dataType(String.class).make()
mgmt.makePropertyKey("amt_ttl").dataType(String.class).make()
mgmt.makePropertyKey("count_1month").dataType(String.class).make()
mgmt.makePropertyKey("amt_1month").dataType(String.class).make()
mgmt.makePropertyKey("invitee_no").dataType(String.class).make()
mgmt.makePropertyKey("buyer_account_no").dataType(String.class).make()
mgmt.makePropertyKey("payer_account_no").dataType(String.class).make()
mgmt.makePropertyKey("count_1week").dataType(String.class).make()
mgmt.makePropertyKey("amt_1week").dataType(String.class).make()
mgmt.makePropertyKey("num_ttl").dataType(String.class).make()
mgmt.makePropertyKey("acct_no_main").dataType(String.class).make()
mgmt.makePropertyKey("acct_no_refer").dataType(String.class).make()
mgmt.makePropertyKey("sign_time").dataType(String.class).make()
mgmt.makePropertyKey("rec_user_id").dataType(String.class).make()
mgmt.makePropertyKey("bank_c").dataType(String.class).make()
mgmt.makePropertyKey("edge_id").dataType(String.class).make()
mgmt.commit() gremlin> g.V().has("account_no", "0011000001478737514").label()
==>Account
gremlin> g.V().has("account_no", "0000000000234354827").label()
==>vertex
gremlin> g.V().has("account_no", "0011000001478715959").label()
==>Account
gremlin> g.V().has("account_no", "0011000001478738371").label()
==>Account
gremlin> g.V().has("account_no", "0011000001337709206").label()
==>vertex
gremlin>
code is there "object Vertex extends Command(description = "Vertex Bulk Import") with Common {
var vertexLabelOpt: Option[String] = opt[Option[String]](name = "vertex-label", description = "Vertex label name")
var vertexColOpt: Option[String] = opt[Option[String]](name = "vertex-col", description = "Vertex property column name")

override def exec(spark: SparkSession): Unit = {
  val vertexLabel = vertexLabelOpt.get
  val vertexCol = vertexColOpt.get

  // Add log to output vertex label and property column
  println(s"===> Starting to import vertices, label: $vertexLabel, property column: $vertexCol")
  val fullHql =
    s"""
       |select
       |  s.*,
       |  m.vertex_id
       |from ($hql) s
       |left join (select * from $vertexMappingTable where vertex_label='$vertexLabel') m
       |on  m.prop_key = '$vertexCol'
       |and m.prop_value = s.`$vertexCol`
       |""".stripMargin
  //and m.vertex_label = '$vertexLabel'
  println(s"Bulk Vertex Import HQL ===> $fullHql")
  val df = spark.sql(fullHql)
  if (df.count() == 0) {
    return
  }
  val columns = df.columns
  println(s"====> Columns: ${columns.mkString(",")}")
  if (!columns.contains(vertexCol))
    throw new IllegalArgumentException(s"Unable to find vertex col [$vertexCol] in HQL columns [$columns].")

  val successCount = genCounter(spark, "success counter")
  val failureCount = genCounter(spark, "failure counter")

  val confMap = loadConfMap()
  import spark.implicits._
  var result =
    df.repartitionByRange(numImportPartitions, df.col(vertexCol))
      .mapPartitions(partitionIterator => {
        // Lazy initialization
        // Cannot use inner anonymous class directly because of Spark serialization
        val partitionExecutor = new PartitionExecutor[Iterator[VertexMapping]](
          confMap,
          successCount,
          failureCount,
          batchSize
        ) {
          override def apply(rows: Iterator[Row]): Iterator[VertexMapping] =
            new Iterator[VertexMapping]() {
              override def hasNext: Boolean = {
                val hasNext = rows.hasNext
                if (!hasNext) commit(true)
                hasNext
              }

              override def next(): VertexMapping = {
                val row = rows.next()
                //println(s"===> row $row")
                val mappingId = Option(row.getAs[Long](VERTEX_ID_COL_NAME))
                val propValue = row.getAs[String](vertexCol)

                // Add log to output mapping ID and property value
                println(s"===> Processing vertex: property value=$propValue, mapping ID=${mappingId.getOrElse("None")}")

                val vertexId =
                  if (mappingId.isDefined) {
                    println(s"===> Using existing vertex ID: ${mappingId.get}")
                    mappingId.get
                  } else
                  {
                    val mayId = g.V().has(vertexCol, propValue).hasLabel(vertexLabel).id()
                    if (mayId.hasNext) {
                      val id = mayId.next()
                      println(s"===> Found existing vertex: ID=$id, label=$vertexLabel, $vertexCol=$propValue")
                      id
                    } else {
                      println(s"===> Creating new vertex: label=$vertexLabel, $vertexCol=$propValue")
                      val newId = tx.addVertex(Array(T.label, vertexLabel): _*).id()
                      println(s"===> New vertex created successfully: ID=$newId")
                      newId
                    }
                  }

                val vertex = g.V(vertexId.asInstanceOf[AnyRef])
                println(s"===> Starting to set vertex properties: ID=$vertexId")
                for (col <- columns if col != VERTEX_ID_COL_NAME) {
                  if (null != row.getAs[String](col)) {
                    println(s"===> Setting property: $col = ${row.getAs[String](col)}")
                    vertex.property(col, row.getAs[String](col))
                  }
                }


                vertex.hasNext // executing
                //println(s"===> saved vertex $vertex")

                batch += 1
                commit(false)

                VertexMapping(vertexId.asInstanceOf[Long], vertexLabel, vertexCol, propValue)
              }
            }
        }

        partitionExecutor.apply(partitionIterator)
      })

  result = result.filter(_ != null).repartition(result.col(VERTEX_ID_COL_NAME))
  // Save to temporary table
  val tempTableName = s"${vertexMappingTable}_temp_${System.currentTimeMillis()}"
  result.distinct().write.mode(SaveMode.Append).saveAsTable(tempTableName)
  // Update mapping table with temporary table
  val appendSql =
    s"""select t1.*
       |from ${tempTableName} t1
       |left join (select * from ${vertexMappingTable} where vertex_label='$vertexLabel') t2
       |on t1.vertex_label=t2.vertex_label and t1.vertex_id=t2.vertex_id
       |where t2.vertex_id is null
       |""".stripMargin
  //s"select t1.* from ${tempTableName} t1 left join ${vertexMappingTable} t2 on t1.vertex_label=t2.vertex_label and t1.vertex_id=t2.vertex_id  where t2.vertex_id is null "
  //spark.sql(appendSql).write.format("Hive").mode(SaveMode.Append).saveAsTable(vertexMappingTable)
  spark.sql(appendSql).write.format("Hive").mode(SaveMode.Append).partitionBy(PARTITION_COLUMN).saveAsTable(vertexMappingTable)
  spark.sql(s"drop table $tempTableName")
  println(
    s"""
       |==========================================================
       |Vertex Bulk Import Metrics
       |success count: ${successCount.value}
       |failure count: ${failureCount.value}
       |==========================================================
       |""".stripMargin)
}

}"
why there is vertex label ?How to avoid or fix vertex label

[porunov]

JanusGraph assigns the default vertex label vertex for any created vertices that were created without explicitly defining the vertex label. As of today you can't create a vertex with the null vertex label.
If you need better clarity about your ingested data I would recommend creating your own base vertex label. You could call it base, general, undefined, or else, and assign such vertex labels explicitly to your vertices.